View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.snapshot;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertTrue;
23  
24  import java.io.IOException;
25  import java.util.Collections;
26  import java.util.ArrayList;
27  import java.util.HashSet;
28  import java.util.List;
29  import java.util.Map;
30  import java.util.Set;
31  import java.util.TreeSet;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.conf.Configuration;
36  import org.apache.hadoop.fs.FSDataOutputStream;
37  import org.apache.hadoop.fs.FileSystem;
38  import org.apache.hadoop.fs.Path;
39  import org.apache.hadoop.hbase.HBaseTestingUtility;
40  import org.apache.hadoop.hbase.HColumnDescriptor;
41  import org.apache.hadoop.hbase.HConstants;
42  import org.apache.hadoop.hbase.HRegionInfo;
43  import org.apache.hadoop.hbase.HTableDescriptor;
44  import org.apache.hadoop.hbase.TableName;
45  import org.apache.hadoop.hbase.TableNotEnabledException;
46  import org.apache.hadoop.hbase.client.Admin;
47  import org.apache.hadoop.hbase.client.BufferedMutator;
48  import org.apache.hadoop.hbase.client.Durability;
49  import org.apache.hadoop.hbase.client.HTable;
50  import org.apache.hadoop.hbase.client.Put;
51  import org.apache.hadoop.hbase.client.Table;
52  import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
53  import org.apache.hadoop.hbase.client.RegionReplicaUtil;
54  import org.apache.hadoop.hbase.io.HFileLink;
55  import org.apache.hadoop.hbase.master.HMaster;
56  import org.apache.hadoop.hbase.master.MasterFileSystem;
57  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
58  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
59  import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
60  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneRequest;
61  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
62  import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
63  import org.apache.hadoop.hbase.regionserver.HRegionServer;
64  import org.apache.hadoop.hbase.regionserver.Region;
65  import org.apache.hadoop.hbase.util.Bytes;
66  import org.apache.hadoop.hbase.util.FSTableDescriptors;
67  import org.apache.hadoop.hbase.util.FSVisitor;
68  import org.apache.hadoop.hbase.util.FSUtils;
69  import org.apache.hadoop.hbase.util.MD5Hash;
70  import org.junit.Assert;
71  
72  import com.google.protobuf.ServiceException;
73  
74  /**
75   * Utilities class for snapshots
76   */
77  public class SnapshotTestingUtils {
78  
79    private static final Log LOG = LogFactory.getLog(SnapshotTestingUtils.class);
80    private static byte[] KEYS = Bytes.toBytes("0123456789");
81  
82    /**
83     * Assert that we don't have any snapshots lists
84     *
85     * @throws IOException
86     *           if the admin operation fails
87     */
88    public static void assertNoSnapshots(Admin admin) throws IOException {
89      assertEquals("Have some previous snapshots", 0, admin.listSnapshots()
90          .size());
91    }
92  
93    /**
94     * Make sure that there is only one snapshot returned from the master and its
95     * name and table match the passed in parameters.
96     */
97    public static List<SnapshotDescription> assertExistsMatchingSnapshot(
98        Admin admin, String snapshotName, TableName tableName)
99        throws IOException {
100     // list the snapshot
101     List<SnapshotDescription> snapshots = admin.listSnapshots();
102 
103     List<SnapshotDescription> returnedSnapshots = new ArrayList<SnapshotDescription>();
104     for (SnapshotDescription sd : snapshots) {
105       if (snapshotName.equals(sd.getName()) &&
106           tableName.equals(TableName.valueOf(sd.getTable()))) {
107         returnedSnapshots.add(sd);
108       }
109     }
110 
111     Assert.assertTrue("No matching snapshots found.", returnedSnapshots.size()>0);
112     return returnedSnapshots;
113   }
114 
115   /**
116    * Make sure that there is only one snapshot returned from the master
117    */
118   public static void assertOneSnapshotThatMatches(Admin admin,
119       SnapshotDescription snapshot) throws IOException {
120     assertOneSnapshotThatMatches(admin, snapshot.getName(),
121         TableName.valueOf(snapshot.getTable()));
122   }
123 
124   /**
125    * Make sure that there is only one snapshot returned from the master and its
126    * name and table match the passed in parameters.
127    */
128   public static List<SnapshotDescription> assertOneSnapshotThatMatches(
129       Admin admin, String snapshotName, TableName tableName)
130       throws IOException {
131     // list the snapshot
132     List<SnapshotDescription> snapshots = admin.listSnapshots();
133 
134     assertEquals("Should only have 1 snapshot", 1, snapshots.size());
135     assertEquals(snapshotName, snapshots.get(0).getName());
136     assertEquals(tableName, TableName.valueOf(snapshots.get(0).getTable()));
137 
138     return snapshots;
139   }
140 
141   /**
142    * Make sure that there is only one snapshot returned from the master and its
143    * name and table match the passed in parameters.
144    */
145   public static List<SnapshotDescription> assertOneSnapshotThatMatches(
146       Admin admin, byte[] snapshot, TableName tableName) throws IOException {
147     return assertOneSnapshotThatMatches(admin, Bytes.toString(snapshot),
148         tableName);
149   }
150 
151   public static void confirmSnapshotValid(HBaseTestingUtility testUtil,
152       SnapshotDescription snapshotDescriptor, TableName tableName, byte[] family)
153       throws IOException {
154     MasterFileSystem mfs = testUtil.getHBaseCluster().getMaster().getMasterFileSystem();
155     confirmSnapshotValid(snapshotDescriptor, tableName, family,
156         mfs.getRootDir(), testUtil.getHBaseAdmin(), mfs.getFileSystem());
157   }
158 
159   /**
160    * Confirm that the snapshot contains references to all the files that should
161    * be in the snapshot.
162    */
163   public static void confirmSnapshotValid(
164       SnapshotDescription snapshotDescriptor, TableName tableName,
165       byte[] testFamily, Path rootDir, Admin admin, FileSystem fs)
166       throws IOException {
167     ArrayList nonEmptyTestFamilies = new ArrayList(1);
168     nonEmptyTestFamilies.add(testFamily);
169     confirmSnapshotValid(snapshotDescriptor, tableName,
170       nonEmptyTestFamilies, null, rootDir, admin, fs);
171   }
172 
173   /**
174    * Confirm that the snapshot has no references files but only metadata.
175    */
176   public static void confirmEmptySnapshotValid(
177       SnapshotDescription snapshotDescriptor, TableName tableName,
178       byte[] testFamily, Path rootDir, Admin admin, FileSystem fs)
179       throws IOException {
180     ArrayList emptyTestFamilies = new ArrayList(1);
181     emptyTestFamilies.add(testFamily);
182     confirmSnapshotValid(snapshotDescriptor, tableName,
183       null, emptyTestFamilies, rootDir, admin, fs);
184   }
185 
186   /**
187    * Confirm that the snapshot contains references to all the files that should
188    * be in the snapshot. This method also perform some redundant check like
189    * the existence of the snapshotinfo or the regioninfo which are done always
190    * by the MasterSnapshotVerifier, at the end of the snapshot operation.
191    */
192   public static void confirmSnapshotValid(
193       SnapshotDescription snapshotDescriptor, TableName tableName,
194       List<byte[]> nonEmptyTestFamilies, List<byte[]> emptyTestFamilies,
195       Path rootDir, Admin admin, FileSystem fs) throws IOException {
196     final Configuration conf = admin.getConfiguration();
197 
198     // check snapshot dir
199     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
200         snapshotDescriptor, rootDir);
201     assertTrue(fs.exists(snapshotDir));
202 
203     SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
204 
205     // Extract regions and families with store files
206     final Set<byte[]> snapshotFamilies = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
207 
208     SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, desc);
209     Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap();
210     for (SnapshotRegionManifest regionManifest: regionManifests.values()) {
211       SnapshotReferenceUtil.visitRegionStoreFiles(regionManifest,
212           new SnapshotReferenceUtil.StoreFileVisitor() {
213         @Override
214         public void storeFile(final HRegionInfo regionInfo, final String family,
215               final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
216           snapshotFamilies.add(Bytes.toBytes(family));
217         }
218       });
219     }
220 
221     // Verify that there are store files in the specified families
222     if (nonEmptyTestFamilies != null) {
223       for (final byte[] familyName: nonEmptyTestFamilies) {
224         assertTrue(snapshotFamilies.contains(familyName));
225       }
226     }
227 
228     // Verify that there are no store files in the specified families
229     if (emptyTestFamilies != null) {
230       for (final byte[] familyName: emptyTestFamilies) {
231         assertFalse(snapshotFamilies.contains(familyName));
232       }
233     }
234 
235     // check the region snapshot for all the regions
236     List<HRegionInfo> regions = admin.getTableRegions(tableName);
237     // remove the non-default regions
238     RegionReplicaUtil.removeNonDefaultRegions(regions);
239     assertEquals(regions.size(), regionManifests.size());
240 
241     // Verify Regions (redundant check, see MasterSnapshotVerifier)
242     for (HRegionInfo info : regions) {
243       String regionName = info.getEncodedName();
244       assertTrue(regionManifests.containsKey(regionName));
245     }
246   }
247 
248   /**
249    * Helper method for testing async snapshot operations. Just waits for the
250    * given snapshot to complete on the server by repeatedly checking the master.
251    *
252    * @param master: the master running the snapshot
253    * @param snapshot: the snapshot to check
254    * @param sleep: amount to sleep between checks to see if the snapshot is done
255    * @throws ServiceException if the snapshot fails
256    */
257   public static void waitForSnapshotToComplete(HMaster master,
258       SnapshotDescription snapshot, long sleep) throws ServiceException {
259     final IsSnapshotDoneRequest request = IsSnapshotDoneRequest.newBuilder()
260         .setSnapshot(snapshot).build();
261     IsSnapshotDoneResponse done = IsSnapshotDoneResponse.newBuilder()
262         .buildPartial();
263     while (!done.getDone()) {
264       done = master.getMasterRpcServices().isSnapshotDone(null, request);
265       try {
266         Thread.sleep(sleep);
267       } catch (InterruptedException e) {
268         throw new ServiceException(e);
269       }
270     }
271   }
272 
273   /*
274    * Take snapshot with maximum of numTries attempts, ignoring CorruptedSnapshotException
275    * except for the last CorruptedSnapshotException
276    */
277   public static void snapshot(Admin admin,
278       final String snapshotName, final String tableName,
279       SnapshotDescription.Type type, int numTries) throws IOException {
280     int tries = 0;
281     CorruptedSnapshotException lastEx = null;
282     while (tries++ < numTries) {
283       try {
284         admin.snapshot(snapshotName, TableName.valueOf(tableName), type);
285         return;
286       } catch (CorruptedSnapshotException cse) {
287         LOG.warn("Got CorruptedSnapshotException", cse);
288         lastEx = cse;
289       }
290     }
291     throw lastEx;
292   }
293 
294   public static void cleanupSnapshot(Admin admin, byte[] tableName)
295       throws IOException {
296     SnapshotTestingUtils.cleanupSnapshot(admin, Bytes.toString(tableName));
297   }
298 
299   public static void cleanupSnapshot(Admin admin, String snapshotName)
300       throws IOException {
301     // delete the taken snapshot
302     admin.deleteSnapshot(snapshotName);
303     assertNoSnapshots(admin);
304   }
305 
306   /**
307    * Expect the snapshot to throw an error when checking if the snapshot is
308    * complete
309    *
310    * @param master master to check
311    * @param snapshot the {@link SnapshotDescription} request to pass to the master
312    * @param clazz expected exception from the master
313    */
314   public static void expectSnapshotDoneException(HMaster master,
315       IsSnapshotDoneRequest snapshot,
316       Class<? extends HBaseSnapshotException> clazz) {
317     try {
318       master.getMasterRpcServices().isSnapshotDone(null, snapshot);
319       Assert.fail("didn't fail to lookup a snapshot");
320     } catch (ServiceException se) {
321       try {
322         throw ProtobufUtil.getRemoteException(se);
323       } catch (HBaseSnapshotException e) {
324         assertEquals("Threw wrong snapshot exception!", clazz, e.getClass());
325       } catch (Throwable t) {
326         Assert.fail("Threw an unexpected exception:" + t);
327       }
328     }
329   }
330 
331   /**
332    * List all the HFiles in the given table
333    *
334    * @param fs: FileSystem where the table lives
335    * @param tableDir directory of the table
336    * @return array of the current HFiles in the table (could be a zero-length array)
337    * @throws IOException on unexecpted error reading the FS
338    */
339   public static ArrayList<String> listHFileNames(final FileSystem fs, final Path tableDir)
340       throws IOException {
341     final ArrayList<String> hfiles = new ArrayList<String>();
342     FSVisitor.visitTableStoreFiles(fs, tableDir, new FSVisitor.StoreFileVisitor() {
343       @Override
344       public void storeFile(final String region, final String family, final String hfileName)
345           throws IOException {
346         hfiles.add(hfileName);
347       }
348     });
349     Collections.sort(hfiles);
350     return hfiles;
351   }
352 
353   /**
354    * Take a snapshot of the specified table and verify that the given family is
355    * not empty. Note that this will leave the table disabled
356    * in the case of an offline snapshot.
357    */
358   public static void createSnapshotAndValidate(Admin admin,
359       TableName tableName, String familyName, String snapshotNameString,
360       Path rootDir, FileSystem fs, boolean onlineSnapshot)
361       throws Exception {
362     ArrayList<byte[]> nonEmptyFamilyNames = new ArrayList<byte[]>(1);
363     nonEmptyFamilyNames.add(Bytes.toBytes(familyName));
364     createSnapshotAndValidate(admin, tableName, nonEmptyFamilyNames, /* emptyFamilyNames= */ null,
365                               snapshotNameString, rootDir, fs, onlineSnapshot);
366   }
367 
368   /**
369    * Take a snapshot of the specified table and verify the given families.
370    * Note that this will leave the table disabled in the case of an offline snapshot.
371    */
372   public static void createSnapshotAndValidate(Admin admin,
373       TableName tableName, List<byte[]> nonEmptyFamilyNames, List<byte[]> emptyFamilyNames,
374       String snapshotNameString, Path rootDir, FileSystem fs, boolean onlineSnapshot)
375         throws Exception {
376     if (!onlineSnapshot) {
377       try {
378         admin.disableTable(tableName);
379       } catch (TableNotEnabledException tne) {
380         LOG.info("In attempting to disable " + tableName + " it turns out that the this table is " +
381             "already disabled.");
382       }
383     }
384     admin.snapshot(snapshotNameString, tableName);
385 
386     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertExistsMatchingSnapshot(admin,
387       snapshotNameString, tableName);
388     if (snapshots == null || snapshots.size() != 1) {
389       Assert.fail("Incorrect number of snapshots for table " + tableName);
390     }
391 
392     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), tableName, nonEmptyFamilyNames,
393       emptyFamilyNames, rootDir, admin, fs);
394   }
395 
396   /**
397    * Corrupt the specified snapshot by deleting some files.
398    *
399    * @param util {@link HBaseTestingUtility}
400    * @param snapshotName name of the snapshot to corrupt
401    * @return array of the corrupted HFiles
402    * @throws IOException on unexecpted error reading the FS
403    */
404   public static ArrayList corruptSnapshot(final HBaseTestingUtility util, final String snapshotName)
405       throws IOException {
406     final MasterFileSystem mfs = util.getHBaseCluster().getMaster().getMasterFileSystem();
407     final FileSystem fs = mfs.getFileSystem();
408 
409     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName,
410                                                                         mfs.getRootDir());
411     SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
412     final TableName table = TableName.valueOf(snapshotDesc.getTable());
413 
414     final ArrayList corruptedFiles = new ArrayList();
415     final Configuration conf = util.getConfiguration();
416     SnapshotReferenceUtil.visitTableStoreFiles(conf, fs, snapshotDir, snapshotDesc,
417         new SnapshotReferenceUtil.StoreFileVisitor() {
418       @Override
419       public void storeFile(final HRegionInfo regionInfo, final String family,
420             final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
421         String region = regionInfo.getEncodedName();
422         String hfile = storeFile.getName();
423         HFileLink link = HFileLink.build(conf, table, region, family, hfile);
424         if (corruptedFiles.size() % 2 == 0) {
425           fs.delete(link.getAvailablePath(fs), true);
426           corruptedFiles.add(hfile);
427         }
428       }
429     });
430 
431     assertTrue(corruptedFiles.size() > 0);
432     return corruptedFiles;
433   }
434 
435   // ==========================================================================
436   //  Snapshot Mock
437   // ==========================================================================
438   public static class SnapshotMock {
439     private final static String TEST_FAMILY = "cf";
440     public final static int TEST_NUM_REGIONS = 4;
441 
442     private final Configuration conf;
443     private final FileSystem fs;
444     private final Path rootDir;
445 
446     static class RegionData {
447       public HRegionInfo hri;
448       public Path tableDir;
449       public Path[] files;
450 
451       public RegionData(final Path tableDir, final HRegionInfo hri, final int nfiles) {
452         this.tableDir = tableDir;
453         this.hri = hri;
454         this.files = new Path[nfiles];
455       }
456     }
457 
458     public static class SnapshotBuilder {
459       private final RegionData[] tableRegions;
460       private final SnapshotDescription desc;
461       private final HTableDescriptor htd;
462       private final Configuration conf;
463       private final FileSystem fs;
464       private final Path rootDir;
465       private Path snapshotDir;
466       private int snapshotted = 0;
467 
468       public SnapshotBuilder(final Configuration conf, final FileSystem fs,
469           final Path rootDir, final HTableDescriptor htd,
470           final SnapshotDescription desc, final RegionData[] tableRegions)
471           throws IOException {
472         this.fs = fs;
473         this.conf = conf;
474         this.rootDir = rootDir;
475         this.htd = htd;
476         this.desc = desc;
477         this.tableRegions = tableRegions;
478         this.snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
479         new FSTableDescriptors(conf)
480           .createTableDescriptorForTableDirectory(snapshotDir, htd, false);
481       }
482 
483       public HTableDescriptor getTableDescriptor() {
484         return this.htd;
485       }
486 
487       public SnapshotDescription getSnapshotDescription() {
488         return this.desc;
489       }
490 
491       public Path getSnapshotsDir() {
492         return this.snapshotDir;
493       }
494 
495       public Path[] addRegion() throws IOException {
496         return addRegion(desc);
497       }
498 
499       public Path[] addRegionV1() throws IOException {
500         return addRegion(desc.toBuilder()
501                           .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION)
502                           .build());
503       }
504 
505       public Path[] addRegionV2() throws IOException {
506         return addRegion(desc.toBuilder()
507                           .setVersion(SnapshotManifestV2.DESCRIPTOR_VERSION)
508                           .build());
509       }
510 
511       private Path[] addRegion(final SnapshotDescription desc) throws IOException {
512         if (this.snapshotted == tableRegions.length) {
513           throw new UnsupportedOperationException("No more regions in the table");
514         }
515 
516         RegionData regionData = tableRegions[this.snapshotted++];
517         ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
518         SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
519         manifest.addRegion(regionData.tableDir, regionData.hri);
520         return regionData.files;
521       }
522 
523       public Path commit() throws IOException {
524         ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
525         SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
526         manifest.addTableDescriptor(htd);
527         manifest.consolidate();
528         SnapshotDescriptionUtils.completeSnapshot(desc, rootDir, snapshotDir, fs);
529         snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(desc, rootDir);
530         return snapshotDir;
531       }
532     }
533 
534     public SnapshotMock(final Configuration conf, final FileSystem fs, final Path rootDir) {
535       this.fs = fs;
536       this.conf = conf;
537       this.rootDir = rootDir;
538     }
539 
540     public SnapshotBuilder createSnapshotV1(final String snapshotName, final String tableName)
541         throws IOException {
542       return createSnapshot(snapshotName, tableName, SnapshotManifestV1.DESCRIPTOR_VERSION);
543     }
544 
545     public SnapshotBuilder createSnapshotV1(final String snapshotName, final String tableName,
546         final int numRegions) throws IOException {
547       return createSnapshot(snapshotName, tableName, numRegions, SnapshotManifestV1.DESCRIPTOR_VERSION);
548     }
549 
550     public SnapshotBuilder createSnapshotV2(final String snapshotName, final String tableName)
551         throws IOException {
552       return createSnapshot(snapshotName, tableName, SnapshotManifestV2.DESCRIPTOR_VERSION);
553     }
554 
555     public SnapshotBuilder createSnapshotV2(final String snapshotName, final String tableName,
556         final int numRegions) throws IOException {
557       return createSnapshot(snapshotName, tableName, numRegions, SnapshotManifestV2.DESCRIPTOR_VERSION);
558     }
559 
560     private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName,
561         final int version) throws IOException {
562       return createSnapshot(snapshotName, tableName, TEST_NUM_REGIONS, version);
563     }
564 
565     private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName,
566         final int numRegions, final int version) throws IOException {
567       HTableDescriptor htd = createHtd(tableName);
568       RegionData[] regions = createTable(htd, numRegions);
569 
570       SnapshotDescription desc = SnapshotDescription.newBuilder()
571         .setTable(htd.getNameAsString())
572         .setName(snapshotName)
573         .setVersion(version)
574         .build();
575 
576       Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
577       SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, fs);
578       return new SnapshotBuilder(conf, fs, rootDir, htd, desc, regions);
579     }
580 
581     public HTableDescriptor createHtd(final String tableName) {
582       HTableDescriptor htd = new HTableDescriptor(tableName);
583       htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
584       return htd;
585     }
586 
587     private RegionData[] createTable(final HTableDescriptor htd, final int nregions)
588         throws IOException {
589       Path tableDir = FSUtils.getTableDir(rootDir, htd.getTableName());
590       new FSTableDescriptors(conf).createTableDescriptorForTableDirectory(tableDir, htd, false);
591 
592       assertTrue(nregions % 2 == 0);
593       RegionData[] regions = new RegionData[nregions];
594       for (int i = 0; i < regions.length; i += 2) {
595         byte[] startKey = Bytes.toBytes(0 + i * 2);
596         byte[] endKey = Bytes.toBytes(1 + i * 2);
597 
598         // First region, simple with one plain hfile.
599         HRegionInfo hri = new HRegionInfo(htd.getTableName(), startKey, endKey);
600         HRegionFileSystem rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
601         regions[i] = new RegionData(tableDir, hri, 3);
602         for (int j = 0; j < regions[i].files.length; ++j) {
603           Path storeFile = createStoreFile(rfs.createTempName());
604           regions[i].files[j] = rfs.commitStoreFile(TEST_FAMILY, storeFile);
605         }
606 
607         // Second region, used to test the split case.
608         // This region contains a reference to the hfile in the first region.
609         startKey = Bytes.toBytes(2 + i * 2);
610         endKey = Bytes.toBytes(3 + i * 2);
611         hri = new HRegionInfo(htd.getTableName());
612         rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
613         regions[i+1] = new RegionData(tableDir, hri, regions[i].files.length);
614         for (int j = 0; j < regions[i].files.length; ++j) {
615           String refName = regions[i].files[j].getName() + '.' + regions[i].hri.getEncodedName();
616           Path refFile = createStoreFile(new Path(rootDir, refName));
617           regions[i+1].files[j] = rfs.commitStoreFile(TEST_FAMILY, refFile);
618         }
619       }
620       return regions;
621     }
622 
623     private Path createStoreFile(final Path storeFile)
624         throws IOException {
625       FSDataOutputStream out = fs.create(storeFile);
626       try {
627         out.write(Bytes.toBytes(storeFile.toString()));
628       } finally {
629         out.close();
630       }
631       return storeFile;
632     }
633   }
634 
635   // ==========================================================================
636   //  Table Helpers
637   // ==========================================================================
638   public static void waitForTableToBeOnline(final HBaseTestingUtility util,
639                                             final TableName tableName)
640       throws IOException, InterruptedException {
641     HRegionServer rs = util.getRSForFirstRegionInTable(tableName);
642     List<Region> onlineRegions = rs.getOnlineRegions(tableName);
643     for (Region region : onlineRegions) {
644       region.waitForFlushesAndCompactions();
645     }
646     // Wait up to 60 seconds for a table to be available.
647     util.waitFor(60000, util.predicateTableAvailable(tableName));
648   }
649 
650   public static void createTable(final HBaseTestingUtility util, final TableName tableName,
651       int regionReplication, final byte[]... families) throws IOException, InterruptedException {
652     HTableDescriptor htd = new HTableDescriptor(tableName);
653     htd.setRegionReplication(regionReplication);
654     for (byte[] family : families) {
655       HColumnDescriptor hcd = new HColumnDescriptor(family);
656       htd.addFamily(hcd);
657     }
658     byte[][] splitKeys = getSplitKeys();
659     util.createTable(htd, splitKeys);
660     assertEquals((splitKeys.length + 1) * regionReplication,
661         util.getHBaseAdmin().getTableRegions(tableName).size());
662   }
663 
664   public static byte[][] getSplitKeys() {
665     byte[][] splitKeys = new byte[KEYS.length-2][];
666     for (int i = 0; i < splitKeys.length; ++i) {
667       splitKeys[i] = new byte[] { KEYS[i+1] };
668     }
669     return splitKeys;
670   }
671 
672   public static void createTable(final HBaseTestingUtility util, final TableName tableName,
673       final byte[]... families) throws IOException, InterruptedException {
674     createTable(util, tableName, 1, families);
675   }
676 
677   public static void loadData(final HBaseTestingUtility util, final TableName tableName, int rows,
678       byte[]... families) throws IOException, InterruptedException {
679     try (BufferedMutator mutator = util.getConnection().getBufferedMutator(tableName)) {
680       loadData(util, mutator, rows, families);
681     }
682   }
683 
684   public static void loadData(final HBaseTestingUtility util, final BufferedMutator mutator, int rows,
685       byte[]... families) throws IOException, InterruptedException {
686     // Ensure one row per region
687     assertTrue(rows >= KEYS.length);
688     for (byte k0: KEYS) {
689       byte[] k = new byte[] { k0 };
690       byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), k);
691       byte[] key = Bytes.add(k, Bytes.toBytes(MD5Hash.getMD5AsHex(value)));
692       final byte[][] families1 = families;
693       final byte[] key1 = key;
694       final byte[] value1 = value;
695       mutator.mutate(createPut(families1, key1, value1));
696       rows--;
697     }
698 
699     // Add other extra rows. more rows, more files
700     while (rows-- > 0) {
701       byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows));
702       byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
703       final byte[][] families1 = families;
704       final byte[] key1 = key;
705       final byte[] value1 = value;
706       mutator.mutate(createPut(families1, key1, value1));
707     }
708     mutator.flush();
709 
710     waitForTableToBeOnline(util, mutator.getName());
711   }
712 
713   private static Put createPut(final byte[][] families, final byte[] key, final byte[] value) {
714     byte[] q = Bytes.toBytes("q");
715     Put put = new Put(key);
716     put.setDurability(Durability.SKIP_WAL);
717     for (byte[] family: families) {
718       put.add(family, q, value);
719     }
720     return put;
721   }
722 
723   public static void deleteAllSnapshots(final Admin admin)
724       throws IOException {
725     // Delete all the snapshots
726     for (SnapshotDescription snapshot: admin.listSnapshots()) {
727       admin.deleteSnapshot(snapshot.getName());
728     }
729     SnapshotTestingUtils.assertNoSnapshots(admin);
730   }
731 
732   public static void deleteArchiveDirectory(final HBaseTestingUtility util)
733       throws IOException {
734     // Ensure the archiver to be empty
735     MasterFileSystem mfs = util.getMiniHBaseCluster().getMaster().getMasterFileSystem();
736     Path archiveDir = new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);
737     mfs.getFileSystem().delete(archiveDir, true);
738   }
739 
740   public static void verifyRowCount(final HBaseTestingUtility util, final TableName tableName,
741       long expectedRows) throws IOException {
742     Table table = new HTable(util.getConfiguration(), tableName);
743     try {
744       assertEquals(expectedRows, util.countRows(table));
745     } finally {
746       table.close();
747     }
748   }
749 
750   public static void verifyReplicasCameOnline(TableName tableName, Admin admin,
751       int regionReplication) throws IOException {
752     List<HRegionInfo> regions = admin.getTableRegions(tableName);
753     HashSet<HRegionInfo> set = new HashSet<HRegionInfo>();
754     for (HRegionInfo hri : regions) {
755       set.add(RegionReplicaUtil.getRegionInfoForDefaultReplica(hri));
756       for (int i = 0; i < regionReplication; i++) {
757         HRegionInfo replica = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
758         if (!regions.contains(replica)) {
759           Assert.fail(replica + " is not contained in the list of online regions");
760         }
761       }
762     }
763     assert(set.size() == getSplitKeys().length + 1);
764   }
765 }