1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.ipc;
19
20 import static org.junit.Assert.assertEquals;
21
22 import java.io.IOException;
23 import java.nio.ByteBuffer;
24 import java.util.Arrays;
25
26 import org.apache.commons.lang.time.StopWatch;
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.commons.logging.impl.Log4JLogger;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.hbase.Cell;
32 import org.apache.hadoop.hbase.CellScanner;
33 import org.apache.hadoop.hbase.CellUtil;
34 import org.apache.hadoop.hbase.HBaseConfiguration;
35 import org.apache.hadoop.hbase.KeyValue;
36 import org.apache.hadoop.hbase.SmallTests;
37 import org.apache.hadoop.hbase.codec.Codec;
38 import org.apache.hadoop.hbase.codec.KeyValueCodec;
39 import org.apache.hadoop.hbase.io.SizedCellScanner;
40 import org.apache.hadoop.hbase.util.Bytes;
41 import org.apache.hadoop.hbase.util.ClassSize;
42 import org.apache.hadoop.io.compress.CompressionCodec;
43 import org.apache.hadoop.io.compress.DefaultCodec;
44 import org.apache.hadoop.io.compress.GzipCodec;
45 import org.apache.log4j.Level;
46 import org.junit.Before;
47 import org.junit.Test;
48 import org.junit.experimental.categories.Category;
49
50 @Category(SmallTests.class)
51 public class TestIPCUtil {
52
53 public static final Log LOG = LogFactory.getLog(IPCUtil.class);
54
55 IPCUtil util;
56 @Before
57 public void before() {
58 this.util = new IPCUtil(new Configuration());
59 }
60
61 @Test
62 public void testBuildCellBlock() throws IOException {
63 doBuildCellBlockUndoCellBlock(this.util, new KeyValueCodec(), null);
64 doBuildCellBlockUndoCellBlock(this.util, new KeyValueCodec(), new DefaultCodec());
65 doBuildCellBlockUndoCellBlock(this.util, new KeyValueCodec(), new GzipCodec());
66 }
67
68 static void doBuildCellBlockUndoCellBlock(final IPCUtil util,
69 final Codec codec, final CompressionCodec compressor)
70 throws IOException {
71 doBuildCellBlockUndoCellBlock(util, codec, compressor, 10, 1, false);
72 }
73
74 static void doBuildCellBlockUndoCellBlock(final IPCUtil util, final Codec codec,
75 final CompressionCodec compressor, final int count, final int size, final boolean sized)
76 throws IOException {
77 Cell [] cells = getCells(count, size);
78 CellScanner cellScanner = sized? getSizedCellScanner(cells):
79 CellUtil.createCellScanner(Arrays.asList(cells).iterator());
80 ByteBuffer bb = util.buildCellBlock(codec, compressor, cellScanner);
81 cellScanner = util.createCellScanner(codec, compressor, bb.array(), 0, bb.limit());
82 int i = 0;
83 while (cellScanner.advance()) {
84 i++;
85 }
86 assertEquals(count, i);
87 }
88
89 static CellScanner getSizedCellScanner(final Cell [] cells) {
90 int size = -1;
91 for (Cell cell: cells) {
92 size += CellUtil.estimatedSizeOf(cell);
93 }
94 final int totalSize = ClassSize.align(size);
95 final CellScanner cellScanner = CellUtil.createCellScanner(cells);
96 return new SizedCellScanner() {
97 @Override
98 public long heapSize() {
99 return totalSize;
100 }
101
102 @Override
103 public Cell current() {
104 return cellScanner.current();
105 }
106
107 @Override
108 public boolean advance() throws IOException {
109 return cellScanner.advance();
110 }
111 };
112 }
113
114 static Cell [] getCells(final int howMany) {
115 return getCells(howMany, 1024);
116 }
117
118 static Cell [] getCells(final int howMany, final int valueSize) {
119 Cell [] cells = new Cell[howMany];
120 byte [] value = new byte[valueSize];
121 for (int i = 0; i < howMany; i++) {
122 byte [] index = Bytes.toBytes(i);
123 KeyValue kv = new KeyValue(index, Bytes.toBytes("f"), index, value);
124 cells[i] = kv;
125 }
126 return cells;
127 }
128
129 private static final String COUNT = "--count=";
130 private static final String SIZE = "--size=";
131
132
133
134
135
136 private static void usage(final int errCode) {
137 System.out.println("Usage: IPCUtil [options]");
138 System.out.println("Micro-benchmarking how changed sizes and counts work with buffer resizing");
139 System.out.println(" --count Count of Cells");
140 System.out.println(" --size Size of Cell values");
141 System.out.println("Example: IPCUtil --count=1024 --size=1024");
142 System.exit(errCode);
143 }
144
145 private static void timerTests(final IPCUtil util, final int count, final int size,
146 final Codec codec, final CompressionCodec compressor)
147 throws IOException {
148 final int cycles = 1000;
149 StopWatch timer = new StopWatch();
150 timer.start();
151 for (int i = 0; i < cycles; i++) {
152 timerTest(util, timer, count, size, codec, compressor, false);
153 }
154 timer.stop();
155 LOG.info("Codec=" + codec + ", compression=" + compressor + ", sized=" + false +
156 ", count=" + count + ", size=" + size + ", + took=" + timer.getTime() + "ms");
157 timer.reset();
158 timer.start();
159 for (int i = 0; i < cycles; i++) {
160 timerTest(util, timer, count, size, codec, compressor, true);
161 }
162 timer.stop();
163 LOG.info("Codec=" + codec + ", compression=" + compressor + ", sized=" + true +
164 ", count=" + count + ", size=" + size + ", + took=" + timer.getTime() + "ms");
165 }
166
167 private static void timerTest(final IPCUtil util, final StopWatch timer, final int count,
168 final int size, final Codec codec, final CompressionCodec compressor, final boolean sized)
169 throws IOException {
170 doBuildCellBlockUndoCellBlock(util, codec, compressor, count, size, sized);
171 }
172
173
174
175
176
177
178 public static void main(String[] args) throws IOException {
179 int count = 1024;
180 int size = 10240;
181 for (String arg: args) {
182 if (arg.startsWith(COUNT)) {
183 count = Integer.parseInt(arg.replace(COUNT, ""));
184 } else if (arg.startsWith(SIZE)) {
185 size = Integer.parseInt(arg.replace(SIZE, ""));
186 } else {
187 usage(1);
188 }
189 }
190 IPCUtil util = new IPCUtil(HBaseConfiguration.create());
191 ((Log4JLogger)IPCUtil.LOG).getLogger().setLevel(Level.ALL);
192 timerTests(util, count, size, new KeyValueCodec(), null);
193 timerTests(util, count, size, new KeyValueCodec(), new DefaultCodec());
194 timerTests(util, count, size, new KeyValueCodec(), new GzipCodec());
195 }
196 }