1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase;
20
21
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertNotNull;
24 import static org.junit.Assert.assertTrue;
25 import static org.junit.Assert.fail;
26
27 import java.io.ByteArrayInputStream;
28 import java.io.ByteArrayOutputStream;
29 import java.io.DataInputStream;
30 import java.io.DataOutputStream;
31 import java.io.IOException;
32 import java.util.List;
33 import java.util.Map;
34 import java.util.NavigableSet;
35 import java.util.Set;
36
37 import org.apache.hadoop.hbase.client.Get;
38 import org.apache.hadoop.hbase.client.Scan;
39 import org.apache.hadoop.hbase.exceptions.DeserializationException;
40 import org.apache.hadoop.hbase.filter.BinaryComparator;
41 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
42 import org.apache.hadoop.hbase.filter.Filter;
43 import org.apache.hadoop.hbase.filter.PrefixFilter;
44 import org.apache.hadoop.hbase.filter.RowFilter;
45 import org.apache.hadoop.hbase.io.TimeRange;
46 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
47 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
48 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
49 import org.apache.hadoop.hbase.util.Bytes;
50 import org.apache.hadoop.hbase.util.Writables;
51 import org.apache.hadoop.io.DataInputBuffer;
52 import org.junit.Test;
53 import org.junit.experimental.categories.Category;
54
55
56
57
58 @Category(SmallTests.class)
59 public class TestSerialization {
60 @Test public void testKeyValue() throws Exception {
61 final String name = "testKeyValue2";
62 byte[] row = name.getBytes();
63 byte[] fam = "fam".getBytes();
64 byte[] qf = "qf".getBytes();
65 long ts = System.currentTimeMillis();
66 byte[] val = "val".getBytes();
67 KeyValue kv = new KeyValue(row, fam, qf, ts, val);
68 ByteArrayOutputStream baos = new ByteArrayOutputStream();
69 DataOutputStream dos = new DataOutputStream(baos);
70 long l = KeyValue.write(kv, dos);
71 dos.close();
72 byte [] mb = baos.toByteArray();
73 ByteArrayInputStream bais = new ByteArrayInputStream(mb);
74 DataInputStream dis = new DataInputStream(bais);
75 KeyValue deserializedKv = KeyValue.create(dis);
76 assertTrue(Bytes.equals(kv.getBuffer(), deserializedKv.getBuffer()));
77 assertEquals(kv.getOffset(), deserializedKv.getOffset());
78 assertEquals(kv.getLength(), deserializedKv.getLength());
79 }
80
81 @Test public void testCreateKeyValueInvalidNegativeLength() {
82
83 KeyValue kv_0 = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"),
84 Bytes.toBytes("myQualifier"), 12345L, Bytes.toBytes("my12345"));
85
86 KeyValue kv_1 = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"),
87 Bytes.toBytes("myQualifier"), 12345L, Bytes.toBytes("my123"));
88
89 ByteArrayOutputStream baos = new ByteArrayOutputStream();
90 DataOutputStream dos = new DataOutputStream(baos);
91
92 long l = 0;
93 try {
94 l = KeyValue.oswrite(kv_0, dos, false);
95 l += KeyValue.oswrite(kv_1, dos, false);
96 assertEquals(100L, l);
97 } catch (IOException e) {
98 fail("Unexpected IOException" + e.getMessage());
99 }
100
101 ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
102 DataInputStream dis = new DataInputStream(bais);
103
104 try {
105 KeyValue.create(dis);
106 assertTrue(kv_0.equals(kv_1));
107 } catch (Exception e) {
108 fail("Unexpected Exception" + e.getMessage());
109 }
110
111
112 try {
113
114 KeyValue.create(-1, dis);
115 fail("Expected corrupt stream");
116 } catch (Exception e) {
117 assertEquals("Failed read -1 bytes, stream corrupt?", e.getMessage());
118 }
119
120 }
121
122 @Test
123 public void testSplitLogTask() throws DeserializationException {
124 SplitLogTask slt = new SplitLogTask.Unassigned(ServerName.valueOf("mgr,1,1"),
125 RecoveryMode.LOG_REPLAY);
126 byte [] bytes = slt.toByteArray();
127 SplitLogTask sltDeserialized = SplitLogTask.parseFrom(bytes);
128 assertTrue(slt.equals(sltDeserialized));
129 }
130
131 @Test public void testCompareFilter() throws Exception {
132 Filter f = new RowFilter(CompareOp.EQUAL,
133 new BinaryComparator(Bytes.toBytes("testRowOne-2")));
134 byte [] bytes = f.toByteArray();
135 Filter ff = RowFilter.parseFrom(bytes);
136 assertNotNull(ff);
137 }
138
139 @Test public void testTableDescriptor() throws Exception {
140 final String name = "testTableDescriptor";
141 HTableDescriptor htd = createTableDescriptor(name);
142 byte [] mb = Writables.getBytes(htd);
143 HTableDescriptor deserializedHtd =
144 (HTableDescriptor)Writables.getWritable(mb, new HTableDescriptor());
145 assertEquals(htd.getTableName(), deserializedHtd.getTableName());
146 }
147
148
149
150
151
152 @Test public void testRegionInfo() throws Exception {
153 HRegionInfo hri = createRandomRegion("testRegionInfo");
154
155
156 byte [] hrib = hri.toByteArray();
157 HRegionInfo deserializedHri = HRegionInfo.parseFrom(hrib);
158 assertEquals(hri.getEncodedName(), deserializedHri.getEncodedName());
159 assertEquals(hri, deserializedHri);
160
161
162 hrib = hri.toDelimitedByteArray();
163 DataInputBuffer buf = new DataInputBuffer();
164 try {
165 buf.reset(hrib, hrib.length);
166 deserializedHri = HRegionInfo.parseFrom(buf);
167 assertEquals(hri.getEncodedName(), deserializedHri.getEncodedName());
168 assertEquals(hri, deserializedHri);
169 } finally {
170 buf.close();
171 }
172 }
173
174 @Test public void testRegionInfos() throws Exception {
175 HRegionInfo hri = createRandomRegion("testRegionInfos");
176 byte[] triple = HRegionInfo.toDelimitedByteArray(hri, hri, hri);
177 List<HRegionInfo> regions = HRegionInfo.parseDelimitedFrom(triple, 0, triple.length);
178 assertTrue(regions.size() == 3);
179 assertTrue(regions.get(0).equals(regions.get(1)));
180 assertTrue(regions.get(0).equals(regions.get(2)));
181 }
182
183 private HRegionInfo createRandomRegion(final String name) {
184 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
185 String [] families = new String [] {"info", "anchor"};
186 for (int i = 0; i < families.length; i++) {
187 htd.addFamily(new HColumnDescriptor(families[i]));
188 }
189 return new HRegionInfo(htd.getTableName(), HConstants.EMPTY_START_ROW,
190 HConstants.EMPTY_END_ROW);
191 }
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300 @Test public void testGet() throws Exception{
301 byte[] row = "row".getBytes();
302 byte[] fam = "fam".getBytes();
303 byte[] qf1 = "qf1".getBytes();
304
305 long ts = System.currentTimeMillis();
306 int maxVersions = 2;
307
308 Get get = new Get(row);
309 get.addColumn(fam, qf1);
310 get.setTimeRange(ts, ts+1);
311 get.setMaxVersions(maxVersions);
312
313 ClientProtos.Get getProto = ProtobufUtil.toGet(get);
314 Get desGet = ProtobufUtil.toGet(getProto);
315
316 assertTrue(Bytes.equals(get.getRow(), desGet.getRow()));
317 Set<byte[]> set = null;
318 Set<byte[]> desSet = null;
319
320 for(Map.Entry<byte[], NavigableSet<byte[]>> entry :
321 get.getFamilyMap().entrySet()){
322 assertTrue(desGet.getFamilyMap().containsKey(entry.getKey()));
323 set = entry.getValue();
324 desSet = desGet.getFamilyMap().get(entry.getKey());
325 for(byte [] qualifier : set){
326 assertTrue(desSet.contains(qualifier));
327 }
328 }
329
330 assertEquals(get.getMaxVersions(), desGet.getMaxVersions());
331 TimeRange tr = get.getTimeRange();
332 TimeRange desTr = desGet.getTimeRange();
333 assertEquals(tr.getMax(), desTr.getMax());
334 assertEquals(tr.getMin(), desTr.getMin());
335 }
336
337
338 @Test public void testScan() throws Exception {
339
340 byte[] startRow = "startRow".getBytes();
341 byte[] stopRow = "stopRow".getBytes();
342 byte[] fam = "fam".getBytes();
343 byte[] qf1 = "qf1".getBytes();
344
345 long ts = System.currentTimeMillis();
346 int maxVersions = 2;
347
348 Scan scan = new Scan(startRow, stopRow);
349 scan.addColumn(fam, qf1);
350 scan.setTimeRange(ts, ts+1);
351 scan.setMaxVersions(maxVersions);
352
353 ClientProtos.Scan scanProto = ProtobufUtil.toScan(scan);
354 Scan desScan = ProtobufUtil.toScan(scanProto);
355
356 assertTrue(Bytes.equals(scan.getStartRow(), desScan.getStartRow()));
357 assertTrue(Bytes.equals(scan.getStopRow(), desScan.getStopRow()));
358 assertEquals(scan.getCacheBlocks(), desScan.getCacheBlocks());
359 Set<byte[]> set = null;
360 Set<byte[]> desSet = null;
361
362 for(Map.Entry<byte[], NavigableSet<byte[]>> entry :
363 scan.getFamilyMap().entrySet()){
364 assertTrue(desScan.getFamilyMap().containsKey(entry.getKey()));
365 set = entry.getValue();
366 desSet = desScan.getFamilyMap().get(entry.getKey());
367 for(byte[] column : set){
368 assertTrue(desSet.contains(column));
369 }
370
371
372 scan = new Scan(startRow);
373 final String name = "testScan";
374 byte [] prefix = Bytes.toBytes(name);
375 scan.setFilter(new PrefixFilter(prefix));
376 scanProto = ProtobufUtil.toScan(scan);
377 desScan = ProtobufUtil.toScan(scanProto);
378 Filter f = desScan.getFilter();
379 assertTrue(f instanceof PrefixFilter);
380 }
381
382 assertEquals(scan.getMaxVersions(), desScan.getMaxVersions());
383 TimeRange tr = scan.getTimeRange();
384 TimeRange desTr = desScan.getTimeRange();
385 assertEquals(tr.getMax(), desTr.getMax());
386 assertEquals(tr.getMin(), desTr.getMin());
387 }
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566 protected static final int MAXVERSIONS = 3;
567 protected final static byte [] fam1 = Bytes.toBytes("colfamily1");
568 protected final static byte [] fam2 = Bytes.toBytes("colfamily2");
569 protected final static byte [] fam3 = Bytes.toBytes("colfamily3");
570 protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
571
572
573
574
575
576
577
578 protected HTableDescriptor createTableDescriptor(final String name) {
579 return createTableDescriptor(name, MAXVERSIONS);
580 }
581
582
583
584
585
586
587
588
589 protected HTableDescriptor createTableDescriptor(final String name,
590 final int versions) {
591 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
592 htd.addFamily(new HColumnDescriptor(fam1)
593 .setMaxVersions(versions)
594 .setBlockCacheEnabled(false)
595 );
596 htd.addFamily(new HColumnDescriptor(fam2)
597 .setMaxVersions(versions)
598 .setBlockCacheEnabled(false)
599 );
600 htd.addFamily(new HColumnDescriptor(fam3)
601 .setMaxVersions(versions)
602 .setBlockCacheEnabled(false)
603 );
604 return htd;
605 }
606 }