1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase;
20
21 import java.io.IOException;
22 import java.util.Iterator;
23 import java.util.List;
24 import java.util.NavigableMap;
25
26 import junit.framework.AssertionFailedError;
27 import junit.framework.TestCase;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.conf.Configuration;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.hbase.client.Delete;
35 import org.apache.hadoop.hbase.client.Durability;
36 import org.apache.hadoop.hbase.client.Get;
37 import org.apache.hadoop.hbase.client.HTable;
38 import org.apache.hadoop.hbase.client.Put;
39 import org.apache.hadoop.hbase.client.Result;
40 import org.apache.hadoop.hbase.client.ResultScanner;
41 import org.apache.hadoop.hbase.client.Scan;
42 import org.apache.hadoop.hbase.regionserver.HRegion;
43 import org.apache.hadoop.hbase.regionserver.InternalScanner;
44 import org.apache.hadoop.hbase.util.Bytes;
45 import org.apache.hadoop.hbase.util.FSUtils;
46 import org.apache.hadoop.hdfs.MiniDFSCluster;
47
48
49
50
51
52
53 public abstract class HBaseTestCase extends TestCase {
54 private static final Log LOG = LogFactory.getLog(HBaseTestCase.class);
55
56 protected final static byte [] fam1 = Bytes.toBytes("colfamily11");
57 protected final static byte [] fam2 = Bytes.toBytes("colfamily21");
58 protected final static byte [] fam3 = Bytes.toBytes("colfamily31");
59
60 protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
61
62 private boolean localfs = false;
63 protected static Path testDir = null;
64 protected FileSystem fs = null;
65 protected HRegion meta = null;
66 protected static final char FIRST_CHAR = 'a';
67 protected static final char LAST_CHAR = 'z';
68 protected static final String PUNCTUATION = "~`@#$%^&*()-_+=:;',.<>/?[]{}|";
69 protected static final byte [] START_KEY_BYTES = {FIRST_CHAR, FIRST_CHAR, FIRST_CHAR};
70 protected String START_KEY = new String(START_KEY_BYTES, HConstants.UTF8_CHARSET);
71 protected static final int MAXVERSIONS = 3;
72
73 protected final HBaseTestingUtility testUtil = new HBaseTestingUtility();
74
75 public volatile Configuration conf = HBaseConfiguration.create();
76
77
78 public HBaseTestCase() {
79 super();
80 }
81
82
83
84
85 public HBaseTestCase(String name) {
86 super(name);
87 }
88
89
90
91
92
93 @Override
94 protected void setUp() throws Exception {
95 super.setUp();
96 localfs =
97 (conf.get("fs.defaultFS", "file:///").compareTo("file:///") == 0);
98
99 if (fs == null) {
100 this.fs = FileSystem.get(conf);
101 }
102 try {
103 if (localfs) {
104 this.testDir = getUnitTestdir(getName());
105 if (fs.exists(testDir)) {
106 fs.delete(testDir, true);
107 }
108 } else {
109 this.testDir = FSUtils.getRootDir(conf);
110 }
111 } catch (Exception e) {
112 LOG.fatal("error during setup", e);
113 throw e;
114 }
115 }
116
117 @Override
118 protected void tearDown() throws Exception {
119 try {
120 if (localfs) {
121 if (this.fs.exists(testDir)) {
122 this.fs.delete(testDir, true);
123 }
124 }
125 } catch (Exception e) {
126 LOG.fatal("error during tear down", e);
127 }
128 super.tearDown();
129 }
130
131
132
133
134
135
136 protected Path getUnitTestdir(String testName) {
137 return testUtil.getDataTestDir(testName);
138 }
139
140
141
142
143
144
145
146
147
148
149
150 public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
151 byte [] endKey)
152 throws IOException {
153 return createNewHRegion(desc, startKey, endKey, this.conf);
154 }
155
156 public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
157 byte [] endKey, Configuration conf)
158 throws IOException {
159 HRegionInfo hri = new HRegionInfo(desc.getTableName(), startKey, endKey);
160 return HRegion.createHRegion(hri, testDir, conf, desc);
161 }
162
163 protected HRegion openClosedRegion(final HRegion closedRegion)
164 throws IOException {
165 return HRegion.openHRegion(closedRegion, null);
166 }
167
168
169
170
171
172
173
174 protected HTableDescriptor createTableDescriptor(final String name) {
175 return createTableDescriptor(name, MAXVERSIONS);
176 }
177
178
179
180
181
182
183
184
185 protected HTableDescriptor createTableDescriptor(final String name,
186 final int versions) {
187 return createTableDescriptor(name, HColumnDescriptor.DEFAULT_MIN_VERSIONS,
188 versions, HConstants.FOREVER, HColumnDescriptor.DEFAULT_KEEP_DELETED);
189 }
190
191
192
193
194
195
196
197
198 protected HTableDescriptor createTableDescriptor(final String name,
199 final int minVersions, final int versions, final int ttl, boolean keepDeleted) {
200 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
201 for (byte[] cfName : new byte[][]{ fam1, fam2, fam3 }) {
202 htd.addFamily(new HColumnDescriptor(cfName)
203 .setMinVersions(minVersions)
204 .setMaxVersions(versions)
205 .setKeepDeletedCells(keepDeleted)
206 .setBlockCacheEnabled(false)
207 .setTimeToLive(ttl)
208 );
209 }
210 return htd;
211 }
212
213
214
215
216
217
218
219
220
221
222
223 public static long addContent(final HRegion r, final byte [] columnFamily, final byte[] column)
224 throws IOException {
225 byte [] startKey = r.getRegionInfo().getStartKey();
226 byte [] endKey = r.getRegionInfo().getEndKey();
227 byte [] startKeyBytes = startKey;
228 if (startKeyBytes == null || startKeyBytes.length == 0) {
229 startKeyBytes = START_KEY_BYTES;
230 }
231 return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily), Bytes.toString(column),
232 startKeyBytes, endKey, -1);
233 }
234
235 public static long addContent(final HRegion r, final byte [] columnFamily)
236 throws IOException {
237 return addContent(r, columnFamily, null);
238 }
239
240
241
242
243
244
245
246
247
248
249
250 public static long addContent(final Incommon updater,
251 final String columnFamily) throws IOException {
252 return addContent(updater, columnFamily, START_KEY_BYTES, null);
253 }
254
255 public static long addContent(final Incommon updater, final String family,
256 final String column) throws IOException {
257 return addContent(updater, family, column, START_KEY_BYTES, null);
258 }
259
260
261
262
263
264
265
266
267
268
269
270
271
272 public static long addContent(final Incommon updater, final String columnFamily,
273 final byte [] startKeyBytes, final byte [] endKey)
274 throws IOException {
275 return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1);
276 }
277
278 public static long addContent(final Incommon updater, final String family, String column,
279 final byte [] startKeyBytes, final byte [] endKey) throws IOException {
280 return addContent(updater, family, column, startKeyBytes, endKey, -1);
281 }
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296 public static long addContent(final Incommon updater,
297 final String columnFamily,
298 final String column,
299 final byte [] startKeyBytes, final byte [] endKey, final long ts)
300 throws IOException {
301 long count = 0;
302
303
304
305
306 char secondCharStart = (char)startKeyBytes[1];
307 char thirdCharStart = (char)startKeyBytes[2];
308 EXIT: for (char c = (char)startKeyBytes[0]; c <= LAST_CHAR; c++) {
309 for (char d = secondCharStart; d <= LAST_CHAR; d++) {
310 for (char e = thirdCharStart; e <= LAST_CHAR; e++) {
311 byte [] t = new byte [] {(byte)c, (byte)d, (byte)e};
312 if (endKey != null && endKey.length > 0
313 && Bytes.compareTo(endKey, t) <= 0) {
314 break EXIT;
315 }
316 try {
317 Put put;
318 if(ts != -1) {
319 put = new Put(t, ts);
320 } else {
321 put = new Put(t);
322 }
323 try {
324 StringBuilder sb = new StringBuilder();
325 if (column != null && column.contains(":")) {
326 sb.append(column);
327 } else {
328 if (columnFamily != null) {
329 sb.append(columnFamily);
330 if (!columnFamily.endsWith(":")) {
331 sb.append(":");
332 }
333 if (column != null) {
334 sb.append(column);
335 }
336 }
337 }
338 byte[][] split =
339 KeyValue.parseColumn(Bytes.toBytes(sb.toString()));
340 if(split.length == 1) {
341 put.add(split[0], new byte[0], t);
342 } else {
343 put.add(split[0], split[1], t);
344 }
345 put.setDurability(Durability.SKIP_WAL);
346 updater.put(put);
347 count++;
348 } catch (RuntimeException ex) {
349 ex.printStackTrace();
350 throw ex;
351 } catch (IOException ex) {
352 ex.printStackTrace();
353 throw ex;
354 }
355 } catch (RuntimeException ex) {
356 ex.printStackTrace();
357 throw ex;
358 } catch (IOException ex) {
359 ex.printStackTrace();
360 throw ex;
361 }
362 }
363
364 thirdCharStart = FIRST_CHAR;
365 }
366 secondCharStart = FIRST_CHAR;
367 }
368 return count;
369 }
370
371
372
373
374 public interface FlushCache {
375
376
377
378 void flushcache() throws IOException;
379 }
380
381
382
383
384
385
386
387 public interface Incommon {
388
389
390
391
392
393
394 void delete(Delete delete, boolean writeToWAL)
395 throws IOException;
396
397
398
399
400
401 void put(Put put) throws IOException;
402
403 Result get(Get get) throws IOException;
404
405
406
407
408
409
410
411
412
413 ScannerIncommon getScanner(
414 byte[] family, byte[][] qualifiers, byte[] firstRow, long ts
415 )
416 throws IOException;
417 }
418
419
420
421
422 public static class HRegionIncommon implements Incommon, FlushCache {
423 final HRegion region;
424
425
426
427
428 public HRegionIncommon(final HRegion HRegion) {
429 this.region = HRegion;
430 }
431
432 public void put(Put put) throws IOException {
433 region.put(put);
434 }
435
436 public void delete(Delete delete, boolean writeToWAL)
437 throws IOException {
438 this.region.delete(delete);
439 }
440
441 public Result get(Get get) throws IOException {
442 return region.get(get);
443 }
444
445 public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
446 byte [] firstRow, long ts)
447 throws IOException {
448 Scan scan = new Scan(firstRow);
449 if(qualifiers == null || qualifiers.length == 0) {
450 scan.addFamily(family);
451 } else {
452 for(int i=0; i<qualifiers.length; i++){
453 scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
454 }
455 }
456 scan.setTimeRange(0, ts);
457 return new
458 InternalScannerIncommon(region.getScanner(scan));
459 }
460
461 public void flushcache() throws IOException {
462 this.region.flushcache();
463 }
464 }
465
466
467
468
469 public static class HTableIncommon implements Incommon {
470 final HTable table;
471
472
473
474
475 public HTableIncommon(final HTable table) {
476 super();
477 this.table = table;
478 }
479
480 public void put(Put put) throws IOException {
481 table.put(put);
482 }
483
484
485 public void delete(Delete delete, boolean writeToWAL)
486 throws IOException {
487 this.table.delete(delete);
488 }
489
490 public Result get(Get get) throws IOException {
491 return table.get(get);
492 }
493
494 public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
495 byte [] firstRow, long ts)
496 throws IOException {
497 Scan scan = new Scan(firstRow);
498 if(qualifiers == null || qualifiers.length == 0) {
499 scan.addFamily(family);
500 } else {
501 for(int i=0; i<qualifiers.length; i++){
502 scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
503 }
504 }
505 scan.setTimeRange(0, ts);
506 return new
507 ClientScannerIncommon(table.getScanner(scan));
508 }
509 }
510
511 public interface ScannerIncommon
512 extends Iterable<Result> {
513 boolean next(List<Cell> values)
514 throws IOException;
515
516 void close() throws IOException;
517 }
518
519 public static class ClientScannerIncommon implements ScannerIncommon {
520 ResultScanner scanner;
521 public ClientScannerIncommon(ResultScanner scanner) {
522 this.scanner = scanner;
523 }
524
525 @Override
526 public boolean next(List<Cell> values)
527 throws IOException {
528 Result results = scanner.next();
529 if (results == null) {
530 return false;
531 }
532 values.clear();
533 values.addAll(results.listCells());
534 return true;
535 }
536
537 public void close() throws IOException {
538 scanner.close();
539 }
540
541 public Iterator<Result> iterator() {
542 return scanner.iterator();
543 }
544 }
545
546 public static class InternalScannerIncommon implements ScannerIncommon {
547 InternalScanner scanner;
548
549 public InternalScannerIncommon(InternalScanner scanner) {
550 this.scanner = scanner;
551 }
552
553 @Override
554 public boolean next(List<Cell> results)
555 throws IOException {
556 return scanner.next(results);
557 }
558
559 @Override
560 public void close() throws IOException {
561 scanner.close();
562 }
563
564 @Override
565 public Iterator<Result> iterator() {
566 throw new UnsupportedOperationException();
567 }
568 }
569
570 protected void assertResultEquals(final HRegion region, final byte [] row,
571 final byte [] family, final byte [] qualifier, final long timestamp,
572 final byte [] value)
573 throws IOException {
574 Get get = new Get(row);
575 get.setTimeStamp(timestamp);
576 Result res = region.get(get);
577 NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map =
578 res.getMap();
579 byte [] res_value = map.get(family).get(qualifier).get(timestamp);
580
581 if (value == null) {
582 assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
583 " at timestamp " + timestamp, null, res_value);
584 } else {
585 if (res_value == null) {
586 fail(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
587 " at timestamp " + timestamp + "\" was expected to be \"" +
588 Bytes.toStringBinary(value) + " but was null");
589 }
590 if (res_value != null) {
591 assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
592 " at timestamp " +
593 timestamp, value, new String(res_value));
594 }
595 }
596 }
597
598
599
600
601
602
603 public static void shutdownDfs(MiniDFSCluster cluster) {
604 if (cluster != null) {
605 LOG.info("Shutting down Mini DFS ");
606 try {
607 cluster.shutdown();
608 } catch (Exception e) {
609
610
611
612 }
613 try {
614 FileSystem fs = cluster.getFileSystem();
615 if (fs != null) {
616 LOG.info("Shutting down FileSystem");
617 fs.close();
618 }
619 FileSystem.closeAll();
620 } catch (IOException e) {
621 LOG.error("error closing file system", e);
622 }
623 }
624 }
625
626
627
628
629
630
631 protected void createMetaRegion() throws IOException {
632 meta = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, testDir,
633 conf, HTableDescriptor.META_TABLEDESC);
634 }
635
636 protected void closeRootAndMeta() throws IOException {
637 HRegion.closeHRegion(meta);
638 }
639
640 public static void assertByteEquals(byte[] expected,
641 byte[] actual) {
642 if (Bytes.compareTo(expected, actual) != 0) {
643 throw new AssertionFailedError("expected:<" +
644 Bytes.toString(expected) + "> but was:<" +
645 Bytes.toString(actual) + ">");
646 }
647 }
648
649 public static void assertEquals(byte[] expected,
650 byte[] actual) {
651 if (Bytes.compareTo(expected, actual) != 0) {
652 throw new AssertionFailedError("expected:<" +
653 Bytes.toStringBinary(expected) + "> but was:<" +
654 Bytes.toStringBinary(actual) + ">");
655 }
656 }
657
658 }