1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.snapshot;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertFalse;
22 import static org.junit.Assert.assertTrue;
23
24 import java.io.IOException;
25 import java.util.Collections;
26 import java.util.ArrayList;
27 import java.util.HashSet;
28 import java.util.List;
29 import java.util.Map;
30 import java.util.Set;
31 import java.util.TreeSet;
32
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.conf.Configuration;
36 import org.apache.hadoop.fs.FSDataOutputStream;
37 import org.apache.hadoop.fs.FileSystem;
38 import org.apache.hadoop.fs.Path;
39 import org.apache.hadoop.hbase.HBaseTestingUtility;
40 import org.apache.hadoop.hbase.HColumnDescriptor;
41 import org.apache.hadoop.hbase.HConstants;
42 import org.apache.hadoop.hbase.HRegionInfo;
43 import org.apache.hadoop.hbase.HTableDescriptor;
44 import org.apache.hadoop.hbase.TableName;
45 import org.apache.hadoop.hbase.TableNotEnabledException;
46 import org.apache.hadoop.hbase.client.Admin;
47 import org.apache.hadoop.hbase.client.BufferedMutator;
48 import org.apache.hadoop.hbase.client.Durability;
49 import org.apache.hadoop.hbase.client.HTable;
50 import org.apache.hadoop.hbase.client.Put;
51 import org.apache.hadoop.hbase.client.Table;
52 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
53 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
54 import org.apache.hadoop.hbase.io.HFileLink;
55 import org.apache.hadoop.hbase.master.HMaster;
56 import org.apache.hadoop.hbase.master.MasterFileSystem;
57 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
58 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
59 import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
60 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneRequest;
61 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
62 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
63 import org.apache.hadoop.hbase.regionserver.HRegionServer;
64 import org.apache.hadoop.hbase.regionserver.Region;
65 import org.apache.hadoop.hbase.util.Bytes;
66 import org.apache.hadoop.hbase.util.FSTableDescriptors;
67 import org.apache.hadoop.hbase.util.FSVisitor;
68 import org.apache.hadoop.hbase.util.FSUtils;
69 import org.apache.hadoop.hbase.util.MD5Hash;
70 import org.junit.Assert;
71
72 import com.google.protobuf.ServiceException;
73
74
75
76
77 public class SnapshotTestingUtils {
78
79 private static final Log LOG = LogFactory.getLog(SnapshotTestingUtils.class);
80 private static byte[] KEYS = Bytes.toBytes("0123456789");
81
82
83
84
85
86
87
88 public static void assertNoSnapshots(Admin admin) throws IOException {
89 assertEquals("Have some previous snapshots", 0, admin.listSnapshots()
90 .size());
91 }
92
93
94
95
96
97 public static List<SnapshotDescription> assertExistsMatchingSnapshot(
98 Admin admin, String snapshotName, TableName tableName)
99 throws IOException {
100
101 List<SnapshotDescription> snapshots = admin.listSnapshots();
102
103 List<SnapshotDescription> returnedSnapshots = new ArrayList<SnapshotDescription>();
104 for (SnapshotDescription sd : snapshots) {
105 if (snapshotName.equals(sd.getName()) &&
106 tableName.equals(TableName.valueOf(sd.getTable()))) {
107 returnedSnapshots.add(sd);
108 }
109 }
110
111 Assert.assertTrue("No matching snapshots found.", returnedSnapshots.size()>0);
112 return returnedSnapshots;
113 }
114
115
116
117
118 public static void assertOneSnapshotThatMatches(Admin admin,
119 SnapshotDescription snapshot) throws IOException {
120 assertOneSnapshotThatMatches(admin, snapshot.getName(),
121 TableName.valueOf(snapshot.getTable()));
122 }
123
124
125
126
127
128 public static List<SnapshotDescription> assertOneSnapshotThatMatches(
129 Admin admin, String snapshotName, TableName tableName)
130 throws IOException {
131
132 List<SnapshotDescription> snapshots = admin.listSnapshots();
133
134 assertEquals("Should only have 1 snapshot", 1, snapshots.size());
135 assertEquals(snapshotName, snapshots.get(0).getName());
136 assertEquals(tableName, TableName.valueOf(snapshots.get(0).getTable()));
137
138 return snapshots;
139 }
140
141
142
143
144
145 public static List<SnapshotDescription> assertOneSnapshotThatMatches(
146 Admin admin, byte[] snapshot, TableName tableName) throws IOException {
147 return assertOneSnapshotThatMatches(admin, Bytes.toString(snapshot),
148 tableName);
149 }
150
151 public static void confirmSnapshotValid(HBaseTestingUtility testUtil,
152 SnapshotDescription snapshotDescriptor, TableName tableName, byte[] family)
153 throws IOException {
154 MasterFileSystem mfs = testUtil.getHBaseCluster().getMaster().getMasterFileSystem();
155 confirmSnapshotValid(snapshotDescriptor, tableName, family,
156 mfs.getRootDir(), testUtil.getHBaseAdmin(), mfs.getFileSystem());
157 }
158
159
160
161
162
163 public static void confirmSnapshotValid(
164 SnapshotDescription snapshotDescriptor, TableName tableName,
165 byte[] testFamily, Path rootDir, Admin admin, FileSystem fs)
166 throws IOException {
167 ArrayList nonEmptyTestFamilies = new ArrayList(1);
168 nonEmptyTestFamilies.add(testFamily);
169 confirmSnapshotValid(snapshotDescriptor, tableName,
170 nonEmptyTestFamilies, null, rootDir, admin, fs);
171 }
172
173
174
175
176 public static void confirmEmptySnapshotValid(
177 SnapshotDescription snapshotDescriptor, TableName tableName,
178 byte[] testFamily, Path rootDir, Admin admin, FileSystem fs)
179 throws IOException {
180 ArrayList emptyTestFamilies = new ArrayList(1);
181 emptyTestFamilies.add(testFamily);
182 confirmSnapshotValid(snapshotDescriptor, tableName,
183 null, emptyTestFamilies, rootDir, admin, fs);
184 }
185
186
187
188
189
190
191
192 public static void confirmSnapshotValid(
193 SnapshotDescription snapshotDescriptor, TableName tableName,
194 List<byte[]> nonEmptyTestFamilies, List<byte[]> emptyTestFamilies,
195 Path rootDir, Admin admin, FileSystem fs) throws IOException {
196 final Configuration conf = admin.getConfiguration();
197
198
199 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
200 snapshotDescriptor, rootDir);
201 assertTrue(fs.exists(snapshotDir));
202
203 SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
204
205
206 final Set<byte[]> snapshotFamilies = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
207
208 SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, desc);
209 Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap();
210 for (SnapshotRegionManifest regionManifest: regionManifests.values()) {
211 SnapshotReferenceUtil.visitRegionStoreFiles(regionManifest,
212 new SnapshotReferenceUtil.StoreFileVisitor() {
213 @Override
214 public void storeFile(final HRegionInfo regionInfo, final String family,
215 final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
216 snapshotFamilies.add(Bytes.toBytes(family));
217 }
218 });
219 }
220
221
222 if (nonEmptyTestFamilies != null) {
223 for (final byte[] familyName: nonEmptyTestFamilies) {
224 assertTrue(snapshotFamilies.contains(familyName));
225 }
226 }
227
228
229 if (emptyTestFamilies != null) {
230 for (final byte[] familyName: emptyTestFamilies) {
231 assertFalse(snapshotFamilies.contains(familyName));
232 }
233 }
234
235
236 List<HRegionInfo> regions = admin.getTableRegions(tableName);
237
238 RegionReplicaUtil.removeNonDefaultRegions(regions);
239 assertEquals(regions.size(), regionManifests.size());
240
241
242 for (HRegionInfo info : regions) {
243 String regionName = info.getEncodedName();
244 assertTrue(regionManifests.containsKey(regionName));
245 }
246 }
247
248
249
250
251
252
253
254
255
256
257 public static void waitForSnapshotToComplete(HMaster master,
258 SnapshotDescription snapshot, long sleep) throws ServiceException {
259 final IsSnapshotDoneRequest request = IsSnapshotDoneRequest.newBuilder()
260 .setSnapshot(snapshot).build();
261 IsSnapshotDoneResponse done = IsSnapshotDoneResponse.newBuilder()
262 .buildPartial();
263 while (!done.getDone()) {
264 done = master.getMasterRpcServices().isSnapshotDone(null, request);
265 try {
266 Thread.sleep(sleep);
267 } catch (InterruptedException e) {
268 throw new ServiceException(e);
269 }
270 }
271 }
272
273
274
275
276
277 public static void snapshot(Admin admin,
278 final String snapshotName, final String tableName,
279 SnapshotDescription.Type type, int numTries) throws IOException {
280 int tries = 0;
281 CorruptedSnapshotException lastEx = null;
282 while (tries++ < numTries) {
283 try {
284 admin.snapshot(snapshotName, TableName.valueOf(tableName), type);
285 return;
286 } catch (CorruptedSnapshotException cse) {
287 LOG.warn("Got CorruptedSnapshotException", cse);
288 lastEx = cse;
289 }
290 }
291 throw lastEx;
292 }
293
294 public static void cleanupSnapshot(Admin admin, byte[] tableName)
295 throws IOException {
296 SnapshotTestingUtils.cleanupSnapshot(admin, Bytes.toString(tableName));
297 }
298
299 public static void cleanupSnapshot(Admin admin, String snapshotName)
300 throws IOException {
301
302 admin.deleteSnapshot(snapshotName);
303 assertNoSnapshots(admin);
304 }
305
306
307
308
309
310
311
312
313
314 public static void expectSnapshotDoneException(HMaster master,
315 IsSnapshotDoneRequest snapshot,
316 Class<? extends HBaseSnapshotException> clazz) {
317 try {
318 master.getMasterRpcServices().isSnapshotDone(null, snapshot);
319 Assert.fail("didn't fail to lookup a snapshot");
320 } catch (ServiceException se) {
321 try {
322 throw ProtobufUtil.getRemoteException(se);
323 } catch (HBaseSnapshotException e) {
324 assertEquals("Threw wrong snapshot exception!", clazz, e.getClass());
325 } catch (Throwable t) {
326 Assert.fail("Threw an unexpected exception:" + t);
327 }
328 }
329 }
330
331
332
333
334
335
336
337
338
339 public static ArrayList<String> listHFileNames(final FileSystem fs, final Path tableDir)
340 throws IOException {
341 final ArrayList<String> hfiles = new ArrayList<String>();
342 FSVisitor.visitTableStoreFiles(fs, tableDir, new FSVisitor.StoreFileVisitor() {
343 @Override
344 public void storeFile(final String region, final String family, final String hfileName)
345 throws IOException {
346 hfiles.add(hfileName);
347 }
348 });
349 Collections.sort(hfiles);
350 return hfiles;
351 }
352
353
354
355
356
357
358 public static void createSnapshotAndValidate(Admin admin,
359 TableName tableName, String familyName, String snapshotNameString,
360 Path rootDir, FileSystem fs, boolean onlineSnapshot)
361 throws Exception {
362 ArrayList<byte[]> nonEmptyFamilyNames = new ArrayList<byte[]>(1);
363 nonEmptyFamilyNames.add(Bytes.toBytes(familyName));
364 createSnapshotAndValidate(admin, tableName, nonEmptyFamilyNames,
365 snapshotNameString, rootDir, fs, onlineSnapshot);
366 }
367
368
369
370
371
372 public static void createSnapshotAndValidate(Admin admin,
373 TableName tableName, List<byte[]> nonEmptyFamilyNames, List<byte[]> emptyFamilyNames,
374 String snapshotNameString, Path rootDir, FileSystem fs, boolean onlineSnapshot)
375 throws Exception {
376 if (!onlineSnapshot) {
377 try {
378 admin.disableTable(tableName);
379 } catch (TableNotEnabledException tne) {
380 LOG.info("In attempting to disable " + tableName + " it turns out that the this table is " +
381 "already disabled.");
382 }
383 }
384 admin.snapshot(snapshotNameString, tableName);
385
386 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertExistsMatchingSnapshot(admin,
387 snapshotNameString, tableName);
388 if (snapshots == null || snapshots.size() != 1) {
389 Assert.fail("Incorrect number of snapshots for table " + tableName);
390 }
391
392 SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), tableName, nonEmptyFamilyNames,
393 emptyFamilyNames, rootDir, admin, fs);
394 }
395
396
397
398
399
400
401
402
403
404 public static ArrayList corruptSnapshot(final HBaseTestingUtility util, final String snapshotName)
405 throws IOException {
406 final MasterFileSystem mfs = util.getHBaseCluster().getMaster().getMasterFileSystem();
407 final FileSystem fs = mfs.getFileSystem();
408
409 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName,
410 mfs.getRootDir());
411 SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
412 final TableName table = TableName.valueOf(snapshotDesc.getTable());
413
414 final ArrayList corruptedFiles = new ArrayList();
415 final Configuration conf = util.getConfiguration();
416 SnapshotReferenceUtil.visitTableStoreFiles(conf, fs, snapshotDir, snapshotDesc,
417 new SnapshotReferenceUtil.StoreFileVisitor() {
418 @Override
419 public void storeFile(final HRegionInfo regionInfo, final String family,
420 final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
421 String region = regionInfo.getEncodedName();
422 String hfile = storeFile.getName();
423 HFileLink link = HFileLink.build(conf, table, region, family, hfile);
424 if (corruptedFiles.size() % 2 == 0) {
425 fs.delete(link.getAvailablePath(fs), true);
426 corruptedFiles.add(hfile);
427 }
428 }
429 });
430
431 assertTrue(corruptedFiles.size() > 0);
432 return corruptedFiles;
433 }
434
435
436
437
438 public static class SnapshotMock {
439 private final static String TEST_FAMILY = "cf";
440 public final static int TEST_NUM_REGIONS = 4;
441
442 private final Configuration conf;
443 private final FileSystem fs;
444 private final Path rootDir;
445
446 static class RegionData {
447 public HRegionInfo hri;
448 public Path tableDir;
449 public Path[] files;
450
451 public RegionData(final Path tableDir, final HRegionInfo hri, final int nfiles) {
452 this.tableDir = tableDir;
453 this.hri = hri;
454 this.files = new Path[nfiles];
455 }
456 }
457
458 public static class SnapshotBuilder {
459 private final RegionData[] tableRegions;
460 private final SnapshotDescription desc;
461 private final HTableDescriptor htd;
462 private final Configuration conf;
463 private final FileSystem fs;
464 private final Path rootDir;
465 private Path snapshotDir;
466 private int snapshotted = 0;
467
468 public SnapshotBuilder(final Configuration conf, final FileSystem fs,
469 final Path rootDir, final HTableDescriptor htd,
470 final SnapshotDescription desc, final RegionData[] tableRegions)
471 throws IOException {
472 this.fs = fs;
473 this.conf = conf;
474 this.rootDir = rootDir;
475 this.htd = htd;
476 this.desc = desc;
477 this.tableRegions = tableRegions;
478 this.snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
479 new FSTableDescriptors(conf)
480 .createTableDescriptorForTableDirectory(snapshotDir, htd, false);
481 }
482
483 public HTableDescriptor getTableDescriptor() {
484 return this.htd;
485 }
486
487 public SnapshotDescription getSnapshotDescription() {
488 return this.desc;
489 }
490
491 public Path getSnapshotsDir() {
492 return this.snapshotDir;
493 }
494
495 public Path[] addRegion() throws IOException {
496 return addRegion(desc);
497 }
498
499 public Path[] addRegionV1() throws IOException {
500 return addRegion(desc.toBuilder()
501 .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION)
502 .build());
503 }
504
505 public Path[] addRegionV2() throws IOException {
506 return addRegion(desc.toBuilder()
507 .setVersion(SnapshotManifestV2.DESCRIPTOR_VERSION)
508 .build());
509 }
510
511 private Path[] addRegion(final SnapshotDescription desc) throws IOException {
512 if (this.snapshotted == tableRegions.length) {
513 throw new UnsupportedOperationException("No more regions in the table");
514 }
515
516 RegionData regionData = tableRegions[this.snapshotted++];
517 ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
518 SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
519 manifest.addRegion(regionData.tableDir, regionData.hri);
520 return regionData.files;
521 }
522
523 public Path commit() throws IOException {
524 ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
525 SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
526 manifest.addTableDescriptor(htd);
527 manifest.consolidate();
528 SnapshotDescriptionUtils.completeSnapshot(desc, rootDir, snapshotDir, fs);
529 snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(desc, rootDir);
530 return snapshotDir;
531 }
532 }
533
534 public SnapshotMock(final Configuration conf, final FileSystem fs, final Path rootDir) {
535 this.fs = fs;
536 this.conf = conf;
537 this.rootDir = rootDir;
538 }
539
540 public SnapshotBuilder createSnapshotV1(final String snapshotName, final String tableName)
541 throws IOException {
542 return createSnapshot(snapshotName, tableName, SnapshotManifestV1.DESCRIPTOR_VERSION);
543 }
544
545 public SnapshotBuilder createSnapshotV1(final String snapshotName, final String tableName,
546 final int numRegions) throws IOException {
547 return createSnapshot(snapshotName, tableName, numRegions, SnapshotManifestV1.DESCRIPTOR_VERSION);
548 }
549
550 public SnapshotBuilder createSnapshotV2(final String snapshotName, final String tableName)
551 throws IOException {
552 return createSnapshot(snapshotName, tableName, SnapshotManifestV2.DESCRIPTOR_VERSION);
553 }
554
555 public SnapshotBuilder createSnapshotV2(final String snapshotName, final String tableName,
556 final int numRegions) throws IOException {
557 return createSnapshot(snapshotName, tableName, numRegions, SnapshotManifestV2.DESCRIPTOR_VERSION);
558 }
559
560 private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName,
561 final int version) throws IOException {
562 return createSnapshot(snapshotName, tableName, TEST_NUM_REGIONS, version);
563 }
564
565 private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName,
566 final int numRegions, final int version) throws IOException {
567 HTableDescriptor htd = createHtd(tableName);
568 RegionData[] regions = createTable(htd, numRegions);
569
570 SnapshotDescription desc = SnapshotDescription.newBuilder()
571 .setTable(htd.getNameAsString())
572 .setName(snapshotName)
573 .setVersion(version)
574 .build();
575
576 Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
577 SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, fs);
578 return new SnapshotBuilder(conf, fs, rootDir, htd, desc, regions);
579 }
580
581 public HTableDescriptor createHtd(final String tableName) {
582 HTableDescriptor htd = new HTableDescriptor(tableName);
583 htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
584 return htd;
585 }
586
587 private RegionData[] createTable(final HTableDescriptor htd, final int nregions)
588 throws IOException {
589 Path tableDir = FSUtils.getTableDir(rootDir, htd.getTableName());
590 new FSTableDescriptors(conf).createTableDescriptorForTableDirectory(tableDir, htd, false);
591
592 assertTrue(nregions % 2 == 0);
593 RegionData[] regions = new RegionData[nregions];
594 for (int i = 0; i < regions.length; i += 2) {
595 byte[] startKey = Bytes.toBytes(0 + i * 2);
596 byte[] endKey = Bytes.toBytes(1 + i * 2);
597
598
599 HRegionInfo hri = new HRegionInfo(htd.getTableName(), startKey, endKey);
600 HRegionFileSystem rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
601 regions[i] = new RegionData(tableDir, hri, 3);
602 for (int j = 0; j < regions[i].files.length; ++j) {
603 Path storeFile = createStoreFile(rfs.createTempName());
604 regions[i].files[j] = rfs.commitStoreFile(TEST_FAMILY, storeFile);
605 }
606
607
608
609 startKey = Bytes.toBytes(2 + i * 2);
610 endKey = Bytes.toBytes(3 + i * 2);
611 hri = new HRegionInfo(htd.getTableName());
612 rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
613 regions[i+1] = new RegionData(tableDir, hri, regions[i].files.length);
614 for (int j = 0; j < regions[i].files.length; ++j) {
615 String refName = regions[i].files[j].getName() + '.' + regions[i].hri.getEncodedName();
616 Path refFile = createStoreFile(new Path(rootDir, refName));
617 regions[i+1].files[j] = rfs.commitStoreFile(TEST_FAMILY, refFile);
618 }
619 }
620 return regions;
621 }
622
623 private Path createStoreFile(final Path storeFile)
624 throws IOException {
625 FSDataOutputStream out = fs.create(storeFile);
626 try {
627 out.write(Bytes.toBytes(storeFile.toString()));
628 } finally {
629 out.close();
630 }
631 return storeFile;
632 }
633 }
634
635
636
637
638 public static void waitForTableToBeOnline(final HBaseTestingUtility util,
639 final TableName tableName)
640 throws IOException, InterruptedException {
641 HRegionServer rs = util.getRSForFirstRegionInTable(tableName);
642 List<Region> onlineRegions = rs.getOnlineRegions(tableName);
643 for (Region region : onlineRegions) {
644 region.waitForFlushesAndCompactions();
645 }
646
647 util.waitFor(60000, util.predicateTableAvailable(tableName));
648 }
649
650 public static void createTable(final HBaseTestingUtility util, final TableName tableName,
651 int regionReplication, final byte[]... families) throws IOException, InterruptedException {
652 HTableDescriptor htd = new HTableDescriptor(tableName);
653 htd.setRegionReplication(regionReplication);
654 for (byte[] family : families) {
655 HColumnDescriptor hcd = new HColumnDescriptor(family);
656 htd.addFamily(hcd);
657 }
658 byte[][] splitKeys = getSplitKeys();
659 util.createTable(htd, splitKeys);
660 assertEquals((splitKeys.length + 1) * regionReplication,
661 util.getHBaseAdmin().getTableRegions(tableName).size());
662 }
663
664 public static byte[][] getSplitKeys() {
665 byte[][] splitKeys = new byte[KEYS.length-2][];
666 for (int i = 0; i < splitKeys.length; ++i) {
667 splitKeys[i] = new byte[] { KEYS[i+1] };
668 }
669 return splitKeys;
670 }
671
672 public static void createTable(final HBaseTestingUtility util, final TableName tableName,
673 final byte[]... families) throws IOException, InterruptedException {
674 createTable(util, tableName, 1, families);
675 }
676
677 public static void loadData(final HBaseTestingUtility util, final TableName tableName, int rows,
678 byte[]... families) throws IOException, InterruptedException {
679 try (BufferedMutator mutator = util.getConnection().getBufferedMutator(tableName)) {
680 loadData(util, mutator, rows, families);
681 }
682 }
683
684 public static void loadData(final HBaseTestingUtility util, final BufferedMutator mutator, int rows,
685 byte[]... families) throws IOException, InterruptedException {
686
687 assertTrue(rows >= KEYS.length);
688 for (byte k0: KEYS) {
689 byte[] k = new byte[] { k0 };
690 byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), k);
691 byte[] key = Bytes.add(k, Bytes.toBytes(MD5Hash.getMD5AsHex(value)));
692 final byte[][] families1 = families;
693 final byte[] key1 = key;
694 final byte[] value1 = value;
695 mutator.mutate(createPut(families1, key1, value1));
696 rows--;
697 }
698
699
700 while (rows-- > 0) {
701 byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows));
702 byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
703 final byte[][] families1 = families;
704 final byte[] key1 = key;
705 final byte[] value1 = value;
706 mutator.mutate(createPut(families1, key1, value1));
707 }
708 mutator.flush();
709
710 waitForTableToBeOnline(util, mutator.getName());
711 }
712
713 private static Put createPut(final byte[][] families, final byte[] key, final byte[] value) {
714 byte[] q = Bytes.toBytes("q");
715 Put put = new Put(key);
716 put.setDurability(Durability.SKIP_WAL);
717 for (byte[] family: families) {
718 put.add(family, q, value);
719 }
720 return put;
721 }
722
723 public static void deleteAllSnapshots(final Admin admin)
724 throws IOException {
725
726 for (SnapshotDescription snapshot: admin.listSnapshots()) {
727 admin.deleteSnapshot(snapshot.getName());
728 }
729 SnapshotTestingUtils.assertNoSnapshots(admin);
730 }
731
732 public static void deleteArchiveDirectory(final HBaseTestingUtility util)
733 throws IOException {
734
735 MasterFileSystem mfs = util.getMiniHBaseCluster().getMaster().getMasterFileSystem();
736 Path archiveDir = new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);
737 mfs.getFileSystem().delete(archiveDir, true);
738 }
739
740 public static void verifyRowCount(final HBaseTestingUtility util, final TableName tableName,
741 long expectedRows) throws IOException {
742 Table table = new HTable(util.getConfiguration(), tableName);
743 try {
744 assertEquals(expectedRows, util.countRows(table));
745 } finally {
746 table.close();
747 }
748 }
749
750 public static void verifyReplicasCameOnline(TableName tableName, Admin admin,
751 int regionReplication) throws IOException {
752 List<HRegionInfo> regions = admin.getTableRegions(tableName);
753 HashSet<HRegionInfo> set = new HashSet<HRegionInfo>();
754 for (HRegionInfo hri : regions) {
755 set.add(RegionReplicaUtil.getRegionInfoForDefaultReplica(hri));
756 for (int i = 0; i < regionReplication; i++) {
757 HRegionInfo replica = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
758 if (!regions.contains(replica)) {
759 Assert.fail(replica + " is not contained in the list of online regions");
760 }
761 }
762 }
763 assert(set.size() == getSplitKeys().length + 1);
764 }
765 }