1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase;
20
21 import java.io.IOException;
22 import java.util.Iterator;
23 import java.util.List;
24 import java.util.NavigableMap;
25
26 import junit.framework.AssertionFailedError;
27 import junit.framework.TestCase;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.conf.Configuration;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.hbase.client.Delete;
35 import org.apache.hadoop.hbase.client.Durability;
36 import org.apache.hadoop.hbase.client.Get;
37 import org.apache.hadoop.hbase.client.HTable;
38 import org.apache.hadoop.hbase.client.Put;
39 import org.apache.hadoop.hbase.client.Result;
40 import org.apache.hadoop.hbase.client.ResultScanner;
41 import org.apache.hadoop.hbase.client.Scan;
42 import org.apache.hadoop.hbase.client.Table;
43 import org.apache.hadoop.hbase.regionserver.HRegion;
44 import org.apache.hadoop.hbase.regionserver.InternalScanner;
45 import org.apache.hadoop.hbase.regionserver.Region;
46 import org.apache.hadoop.hbase.util.Bytes;
47 import org.apache.hadoop.hbase.util.FSTableDescriptors;
48 import org.apache.hadoop.hbase.util.FSUtils;
49 import org.apache.hadoop.hdfs.MiniDFSCluster;
50
51
52
53
54
55
56 public abstract class HBaseTestCase extends TestCase {
57 private static final Log LOG = LogFactory.getLog(HBaseTestCase.class);
58
59 protected final static byte [] fam1 = Bytes.toBytes("colfamily11");
60 protected final static byte [] fam2 = Bytes.toBytes("colfamily21");
61 protected final static byte [] fam3 = Bytes.toBytes("colfamily31");
62
63 protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
64
65 private boolean localfs = false;
66 protected static Path testDir = null;
67 protected FileSystem fs = null;
68 protected HRegion meta = null;
69 protected static final char FIRST_CHAR = 'a';
70 protected static final char LAST_CHAR = 'z';
71 protected static final String PUNCTUATION = "~`@#$%^&*()-_+=:;',.<>/?[]{}|";
72 protected static final byte [] START_KEY_BYTES = {FIRST_CHAR, FIRST_CHAR, FIRST_CHAR};
73 protected String START_KEY = new String(START_KEY_BYTES, HConstants.UTF8_CHARSET);
74 protected static final int MAXVERSIONS = 3;
75
76 protected final HBaseTestingUtility testUtil = new HBaseTestingUtility();
77
78 public volatile Configuration conf = HBaseConfiguration.create();
79 public final FSTableDescriptors fsTableDescriptors;
80 {
81 try {
82 fsTableDescriptors = new FSTableDescriptors(conf);
83 } catch (IOException e) {
84 throw new RuntimeException("Failed to init descriptors", e);
85 }
86 }
87
88
89 public HBaseTestCase() {
90 super();
91 }
92
93
94
95
96 public HBaseTestCase(String name) {
97 super(name);
98 }
99
100
101
102
103
104 @Override
105 protected void setUp() throws Exception {
106 super.setUp();
107 localfs =
108 (conf.get("fs.defaultFS", "file:///").compareTo("file:///") == 0);
109
110 if (fs == null) {
111 this.fs = FileSystem.get(conf);
112 }
113 try {
114 if (localfs) {
115 this.testDir = getUnitTestdir(getName());
116 if (fs.exists(testDir)) {
117 fs.delete(testDir, true);
118 }
119 } else {
120 this.testDir = FSUtils.getRootDir(conf);
121 }
122 } catch (Exception e) {
123 LOG.fatal("error during setup", e);
124 throw e;
125 }
126 }
127
128 @Override
129 protected void tearDown() throws Exception {
130 try {
131 if (localfs) {
132 if (this.fs.exists(testDir)) {
133 this.fs.delete(testDir, true);
134 }
135 }
136 } catch (Exception e) {
137 LOG.fatal("error during tear down", e);
138 }
139 super.tearDown();
140 }
141
142
143
144
145
146
147 protected Path getUnitTestdir(String testName) {
148 return testUtil.getDataTestDir(testName);
149 }
150
151
152
153
154
155
156
157
158
159
160
161 public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
162 byte [] endKey)
163 throws IOException {
164 return createNewHRegion(desc, startKey, endKey, this.conf);
165 }
166
167 public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
168 byte [] endKey, Configuration conf)
169 throws IOException {
170 HRegionInfo hri = new HRegionInfo(desc.getTableName(), startKey, endKey);
171 return HRegion.createHRegion(hri, testDir, conf, desc);
172 }
173
174 protected HRegion openClosedRegion(final HRegion closedRegion)
175 throws IOException {
176 return HRegion.openHRegion(closedRegion, null);
177 }
178
179
180
181
182
183
184
185 protected HTableDescriptor createTableDescriptor(final String name) {
186 return createTableDescriptor(name, MAXVERSIONS);
187 }
188
189
190
191
192
193
194
195
196 protected HTableDescriptor createTableDescriptor(final String name,
197 final int versions) {
198 return createTableDescriptor(name, HColumnDescriptor.DEFAULT_MIN_VERSIONS,
199 versions, HConstants.FOREVER, HColumnDescriptor.DEFAULT_KEEP_DELETED);
200 }
201
202
203
204
205
206
207
208
209 protected HTableDescriptor createTableDescriptor(final String name,
210 final int minVersions, final int versions, final int ttl, KeepDeletedCells keepDeleted) {
211 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
212 for (byte[] cfName : new byte[][]{ fam1, fam2, fam3 }) {
213 htd.addFamily(new HColumnDescriptor(cfName)
214 .setMinVersions(minVersions)
215 .setMaxVersions(versions)
216 .setKeepDeletedCells(keepDeleted)
217 .setBlockCacheEnabled(false)
218 .setTimeToLive(ttl)
219 );
220 }
221 return htd;
222 }
223
224
225
226
227
228
229
230
231
232
233
234 public static long addContent(final Region r, final byte [] columnFamily, final byte[] column)
235 throws IOException {
236 byte [] startKey = r.getRegionInfo().getStartKey();
237 byte [] endKey = r.getRegionInfo().getEndKey();
238 byte [] startKeyBytes = startKey;
239 if (startKeyBytes == null || startKeyBytes.length == 0) {
240 startKeyBytes = START_KEY_BYTES;
241 }
242 return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily), Bytes.toString(column),
243 startKeyBytes, endKey, -1);
244 }
245
246 public static long addContent(final Region r, final byte [] columnFamily) throws IOException {
247 return addContent(r, columnFamily, null);
248 }
249
250
251
252
253
254
255
256
257
258
259
260 public static long addContent(final Incommon updater,
261 final String columnFamily) throws IOException {
262 return addContent(updater, columnFamily, START_KEY_BYTES, null);
263 }
264
265 public static long addContent(final Incommon updater, final String family,
266 final String column) throws IOException {
267 return addContent(updater, family, column, START_KEY_BYTES, null);
268 }
269
270
271
272
273
274
275
276
277
278
279
280
281
282 public static long addContent(final Incommon updater, final String columnFamily,
283 final byte [] startKeyBytes, final byte [] endKey)
284 throws IOException {
285 return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1);
286 }
287
288 public static long addContent(final Incommon updater, final String family, String column,
289 final byte [] startKeyBytes, final byte [] endKey) throws IOException {
290 return addContent(updater, family, column, startKeyBytes, endKey, -1);
291 }
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306 public static long addContent(final Incommon updater,
307 final String columnFamily,
308 final String column,
309 final byte [] startKeyBytes, final byte [] endKey, final long ts)
310 throws IOException {
311 long count = 0;
312
313
314
315
316 char secondCharStart = (char)startKeyBytes[1];
317 char thirdCharStart = (char)startKeyBytes[2];
318 EXIT: for (char c = (char)startKeyBytes[0]; c <= LAST_CHAR; c++) {
319 for (char d = secondCharStart; d <= LAST_CHAR; d++) {
320 for (char e = thirdCharStart; e <= LAST_CHAR; e++) {
321 byte [] t = new byte [] {(byte)c, (byte)d, (byte)e};
322 if (endKey != null && endKey.length > 0
323 && Bytes.compareTo(endKey, t) <= 0) {
324 break EXIT;
325 }
326 try {
327 Put put;
328 if(ts != -1) {
329 put = new Put(t, ts);
330 } else {
331 put = new Put(t);
332 }
333 try {
334 StringBuilder sb = new StringBuilder();
335 if (column != null && column.contains(":")) {
336 sb.append(column);
337 } else {
338 if (columnFamily != null) {
339 sb.append(columnFamily);
340 if (!columnFamily.endsWith(":")) {
341 sb.append(":");
342 }
343 if (column != null) {
344 sb.append(column);
345 }
346 }
347 }
348 byte[][] split =
349 KeyValue.parseColumn(Bytes.toBytes(sb.toString()));
350 if(split.length == 1) {
351 put.add(split[0], new byte[0], t);
352 } else {
353 put.add(split[0], split[1], t);
354 }
355 put.setDurability(Durability.SKIP_WAL);
356 updater.put(put);
357 count++;
358 } catch (RuntimeException ex) {
359 ex.printStackTrace();
360 throw ex;
361 } catch (IOException ex) {
362 ex.printStackTrace();
363 throw ex;
364 }
365 } catch (RuntimeException ex) {
366 ex.printStackTrace();
367 throw ex;
368 } catch (IOException ex) {
369 ex.printStackTrace();
370 throw ex;
371 }
372 }
373
374 thirdCharStart = FIRST_CHAR;
375 }
376 secondCharStart = FIRST_CHAR;
377 }
378 return count;
379 }
380
381
382
383
384 public interface FlushCache {
385
386
387
388 void flushcache() throws IOException;
389 }
390
391
392
393
394
395
396
397 public interface Incommon {
398
399
400
401
402
403
404 void delete(Delete delete, boolean writeToWAL)
405 throws IOException;
406
407
408
409
410
411 void put(Put put) throws IOException;
412
413 Result get(Get get) throws IOException;
414
415
416
417
418
419
420
421
422
423 ScannerIncommon getScanner(
424 byte[] family, byte[][] qualifiers, byte[] firstRow, long ts
425 )
426 throws IOException;
427 }
428
429
430
431
432 public static class HRegionIncommon implements Incommon, FlushCache {
433 final HRegion region;
434
435
436
437
438 public HRegionIncommon(final HRegion HRegion) {
439 this.region = HRegion;
440 }
441
442 public HRegionIncommon(final Region region) {
443 this.region = (HRegion)region;
444 }
445
446 public void put(Put put) throws IOException {
447 region.put(put);
448 }
449
450 public void delete(Delete delete, boolean writeToWAL)
451 throws IOException {
452 this.region.delete(delete);
453 }
454
455 public Result get(Get get) throws IOException {
456 return region.get(get);
457 }
458
459 public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
460 byte [] firstRow, long ts)
461 throws IOException {
462 Scan scan = new Scan(firstRow);
463 if(qualifiers == null || qualifiers.length == 0) {
464 scan.addFamily(family);
465 } else {
466 for(int i=0; i<qualifiers.length; i++){
467 scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
468 }
469 }
470 scan.setTimeRange(0, ts);
471 return new
472 InternalScannerIncommon(region.getScanner(scan));
473 }
474
475 public void flushcache() throws IOException {
476 this.region.flush(true);
477 }
478 }
479
480
481
482
483 public static class HTableIncommon implements Incommon {
484 final Table table;
485
486
487
488
489 public HTableIncommon(final Table table) {
490 super();
491 this.table = table;
492 }
493
494 public void put(Put put) throws IOException {
495 table.put(put);
496 }
497
498
499 public void delete(Delete delete, boolean writeToWAL)
500 throws IOException {
501 this.table.delete(delete);
502 }
503
504 public Result get(Get get) throws IOException {
505 return table.get(get);
506 }
507
508 public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
509 byte [] firstRow, long ts)
510 throws IOException {
511 Scan scan = new Scan(firstRow);
512 if(qualifiers == null || qualifiers.length == 0) {
513 scan.addFamily(family);
514 } else {
515 for(int i=0; i<qualifiers.length; i++){
516 scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
517 }
518 }
519 scan.setTimeRange(0, ts);
520 return new
521 ClientScannerIncommon(table.getScanner(scan));
522 }
523 }
524
525 public interface ScannerIncommon
526 extends Iterable<Result> {
527 boolean next(List<Cell> values)
528 throws IOException;
529
530 void close() throws IOException;
531 }
532
533 public static class ClientScannerIncommon implements ScannerIncommon {
534 ResultScanner scanner;
535 public ClientScannerIncommon(ResultScanner scanner) {
536 this.scanner = scanner;
537 }
538
539 @Override
540 public boolean next(List<Cell> values)
541 throws IOException {
542 Result results = scanner.next();
543 if (results == null) {
544 return false;
545 }
546 values.clear();
547 values.addAll(results.listCells());
548 return true;
549 }
550
551 public void close() throws IOException {
552 scanner.close();
553 }
554
555 public Iterator<Result> iterator() {
556 return scanner.iterator();
557 }
558 }
559
560 public static class InternalScannerIncommon implements ScannerIncommon {
561 InternalScanner scanner;
562
563 public InternalScannerIncommon(InternalScanner scanner) {
564 this.scanner = scanner;
565 }
566
567 @Override
568 public boolean next(List<Cell> results)
569 throws IOException {
570 return scanner.next(results);
571 }
572
573 @Override
574 public void close() throws IOException {
575 scanner.close();
576 }
577
578 @Override
579 public Iterator<Result> iterator() {
580 throw new UnsupportedOperationException();
581 }
582 }
583
584 protected void assertResultEquals(final HRegion region, final byte [] row,
585 final byte [] family, final byte [] qualifier, final long timestamp,
586 final byte [] value)
587 throws IOException {
588 Get get = new Get(row);
589 get.setTimeStamp(timestamp);
590 Result res = region.get(get);
591 NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map =
592 res.getMap();
593 byte [] res_value = map.get(family).get(qualifier).get(timestamp);
594
595 if (value == null) {
596 assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
597 " at timestamp " + timestamp, null, res_value);
598 } else {
599 if (res_value == null) {
600 fail(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
601 " at timestamp " + timestamp + "\" was expected to be \"" +
602 Bytes.toStringBinary(value) + " but was null");
603 }
604 if (res_value != null) {
605 assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
606 " at timestamp " +
607 timestamp, value, new String(res_value));
608 }
609 }
610 }
611
612
613
614
615
616
617 public static void shutdownDfs(MiniDFSCluster cluster) {
618 if (cluster != null) {
619 LOG.info("Shutting down Mini DFS ");
620 try {
621 cluster.shutdown();
622 } catch (Exception e) {
623
624
625
626 }
627 try {
628 FileSystem fs = cluster.getFileSystem();
629 if (fs != null) {
630 LOG.info("Shutting down FileSystem");
631 fs.close();
632 }
633 FileSystem.closeAll();
634 } catch (IOException e) {
635 LOG.error("error closing file system", e);
636 }
637 }
638 }
639
640
641
642
643
644
645 protected void createMetaRegion() throws IOException {
646 FSTableDescriptors fsTableDescriptors = new FSTableDescriptors(conf);
647 meta = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, testDir, conf,
648 fsTableDescriptors.get(TableName.META_TABLE_NAME));
649 }
650
651 protected void closeRootAndMeta() throws IOException {
652 HRegion.closeHRegion(meta);
653 }
654
655 public static void assertByteEquals(byte[] expected,
656 byte[] actual) {
657 if (Bytes.compareTo(expected, actual) != 0) {
658 throw new AssertionFailedError("expected:<" +
659 Bytes.toString(expected) + "> but was:<" +
660 Bytes.toString(actual) + ">");
661 }
662 }
663
664 public static void assertEquals(byte[] expected,
665 byte[] actual) {
666 if (Bytes.compareTo(expected, actual) != 0) {
667 throw new AssertionFailedError("expected:<" +
668 Bytes.toStringBinary(expected) + "> but was:<" +
669 Bytes.toStringBinary(actual) + ">");
670 }
671 }
672
673 }