View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.snapshot;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertTrue;
22  import static org.junit.Assert.fail;
23  
24  import java.io.IOException;
25  import java.util.Collections;
26  import java.util.Comparator;
27  import java.util.HashMap;
28  import java.util.HashSet;
29  import java.util.List;
30  import java.util.Map;
31  import java.util.Set;
32  import java.util.concurrent.CountDownLatch;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.commons.logging.impl.Log4JLogger;
37  import org.apache.hadoop.conf.Configuration;
38  import org.apache.hadoop.fs.FileSystem;
39  import org.apache.hadoop.fs.Path;
40  import org.apache.hadoop.hbase.HBaseTestingUtility;
41  import org.apache.hadoop.hbase.HConstants;
42  import org.apache.hadoop.hbase.HRegionInfo;
43  import org.apache.hadoop.hbase.HTableDescriptor;
44  import org.apache.hadoop.hbase.LargeTests;
45  import org.apache.hadoop.hbase.TableName;
46  import org.apache.hadoop.hbase.TableNotFoundException;
47  import org.apache.hadoop.hbase.client.HBaseAdmin;
48  import org.apache.hadoop.hbase.client.HTable;
49  import org.apache.hadoop.hbase.client.ScannerCallable;
50  import org.apache.hadoop.hbase.ipc.RpcClient;
51  import org.apache.hadoop.hbase.ipc.RpcServer;
52  import org.apache.hadoop.hbase.master.HMaster;
53  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
54  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
55  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
56  import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
57  import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
58  import org.apache.hadoop.hbase.util.Bytes;
59  import org.apache.hadoop.hbase.util.FSTableDescriptors;
60  import org.apache.hadoop.hbase.util.FSUtils;
61  import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
62  import org.apache.log4j.Level;
63  import org.junit.After;
64  import org.junit.AfterClass;
65  import org.junit.Before;
66  import org.junit.BeforeClass;
67  import org.junit.Test;
68  import org.junit.experimental.categories.Category;
69  
70  /**
71   * Test creating/using/deleting snapshots from the client
72   * <p>
73   * This is an end-to-end test for the snapshot utility
74   *
75   * TODO This is essentially a clone of TestSnapshotFromClient.  This is worth refactoring this
76   * because there will be a few more flavors of snapshots that need to run these tests.
77   */
78  @Category(LargeTests.class)
79  public class TestFlushSnapshotFromClient {
80    private static final Log LOG = LogFactory.getLog(TestFlushSnapshotFromClient.class);
81    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
82    private static final int NUM_RS = 2;
83    private static final String STRING_TABLE_NAME = "test";
84    private static final byte[] TEST_FAM = Bytes.toBytes("fam");
85    private static final byte[] TEST_QUAL = Bytes.toBytes("q");
86    private static final TableName TABLE_NAME =
87        TableName.valueOf(STRING_TABLE_NAME);
88    private final int DEFAULT_NUM_ROWS = 100;
89  
90    /**
91     * Setup the config for the cluster
92     * @throws Exception on failure
93     */
94    @BeforeClass
95    public static void setupCluster() throws Exception {
96      ((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL);
97      ((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL);
98      ((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL);
99      setupConf(UTIL.getConfiguration());
100     UTIL.startMiniCluster(NUM_RS);
101   }
102 
103   private static void setupConf(Configuration conf) {
104     // disable the ui
105     conf.setInt("hbase.regionsever.info.port", -1);
106     // change the flush size to a small amount, regulating number of store files
107     conf.setInt("hbase.hregion.memstore.flush.size", 25000);
108     // so make sure we get a compaction when doing a load, but keep around some
109     // files in the store
110     conf.setInt("hbase.hstore.compaction.min", 10);
111     conf.setInt("hbase.hstore.compactionThreshold", 10);
112     // block writes if we get to 12 store files
113     conf.setInt("hbase.hstore.blockingStoreFiles", 12);
114     // Enable snapshot
115     conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
116     conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
117       ConstantSizeRegionSplitPolicy.class.getName());
118   }
119 
120   @Before
121   public void setup() throws Exception {
122     SnapshotTestingUtils.createTable(UTIL, TABLE_NAME, TEST_FAM);
123   }
124 
125   @After
126   public void tearDown() throws Exception {
127     UTIL.deleteTable(TABLE_NAME);
128 
129     SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
130     SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
131   }
132 
133   @AfterClass
134   public static void cleanupTest() throws Exception {
135     try {
136       UTIL.shutdownMiniCluster();
137     } catch (Exception e) {
138       LOG.warn("failure shutting down cluster", e);
139     }
140   }
141 
142   /**
143    * Test simple flush snapshotting a table that is online
144    * @throws Exception
145    */
146   @Test (timeout=300000)
147   public void testFlushTableSnapshot() throws Exception {
148     HBaseAdmin admin = UTIL.getHBaseAdmin();
149     // make sure we don't fail on listing snapshots
150     SnapshotTestingUtils.assertNoSnapshots(admin);
151 
152     // put some stuff in the table
153     HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
154     SnapshotTestingUtils.loadData(UTIL, table, DEFAULT_NUM_ROWS, TEST_FAM);
155 
156     // get the name of all the regionservers hosting the snapshotted table
157     Set<String> snapshotServers = new HashSet<String>();
158     List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
159     for (RegionServerThread server : servers) {
160       if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
161         snapshotServers.add(server.getRegionServer().getServerName().toString());
162       }
163     }
164 
165     LOG.debug("FS state before snapshot:");
166     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
167         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
168 
169     // take a snapshot of the enabled table
170     String snapshotString = "offlineTableSnapshot";
171     byte[] snapshot = Bytes.toBytes(snapshotString);
172     admin.snapshot(snapshotString, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
173     LOG.debug("Snapshot completed.");
174 
175     // make sure we have the snapshot
176     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
177       snapshot, TABLE_NAME);
178 
179     // make sure its a valid snapshot
180     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
181     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
182     LOG.debug("FS state after snapshot:");
183     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
184         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
185 
186     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
187         admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
188   }
189 
190    /**
191    * Test snapshotting a table that is online without flushing
192    * @throws Exception
193    */
194   @Test
195   public void testSkipFlushTableSnapshot() throws Exception {
196     HBaseAdmin admin = UTIL.getHBaseAdmin();
197     // make sure we don't fail on listing snapshots
198     SnapshotTestingUtils.assertNoSnapshots(admin);
199 
200     // put some stuff in the table
201     HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
202     UTIL.loadTable(table, TEST_FAM);
203 
204     // get the name of all the regionservers hosting the snapshotted table
205     Set<String> snapshotServers = new HashSet<String>();
206     List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
207     for (RegionServerThread server : servers) {
208       if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
209         snapshotServers.add(server.getRegionServer().getServerName().toString());
210       }
211     }
212 
213     LOG.debug("FS state before snapshot:");
214     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
215         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
216 
217     // take a snapshot of the enabled table
218     String snapshotString = "skipFlushTableSnapshot";
219     byte[] snapshot = Bytes.toBytes(snapshotString);
220     admin.snapshot(snapshotString, STRING_TABLE_NAME, SnapshotDescription.Type.SKIPFLUSH);
221     LOG.debug("Snapshot completed.");
222 
223     // make sure we have the snapshot
224     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
225         snapshot, TABLE_NAME);
226 
227     // make sure its a valid snapshot
228     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
229     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
230     LOG.debug("FS state after snapshot:");
231     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
232         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
233 
234     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
235         admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
236 
237     admin.deleteSnapshot(snapshot);
238     snapshots = admin.listSnapshots();
239     SnapshotTestingUtils.assertNoSnapshots(admin);
240   }
241 
242 
243   /**
244    * Test simple flush snapshotting a table that is online
245    * @throws Exception
246    */
247   @Test (timeout=300000)
248   public void testFlushTableSnapshotWithProcedure() throws Exception {
249     HBaseAdmin admin = UTIL.getHBaseAdmin();
250     // make sure we don't fail on listing snapshots
251     SnapshotTestingUtils.assertNoSnapshots(admin);
252 
253     // put some stuff in the table
254     HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
255     SnapshotTestingUtils.loadData(UTIL, table, DEFAULT_NUM_ROWS, TEST_FAM);
256 
257     // get the name of all the regionservers hosting the snapshotted table
258     Set<String> snapshotServers = new HashSet<String>();
259     List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
260     for (RegionServerThread server : servers) {
261       if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
262         snapshotServers.add(server.getRegionServer().getServerName().toString());
263       }
264     }
265 
266     LOG.debug("FS state before snapshot:");
267     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
268         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
269 
270     // take a snapshot of the enabled table
271     String snapshotString = "offlineTableSnapshot";
272     byte[] snapshot = Bytes.toBytes(snapshotString);
273     Map<String, String> props = new HashMap<String, String>();
274     props.put("table", STRING_TABLE_NAME);
275     admin.execProcedure(SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION,
276         snapshotString, props);
277 
278 
279     LOG.debug("Snapshot completed.");
280 
281     // make sure we have the snapshot
282     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
283       snapshot, TABLE_NAME);
284 
285     // make sure its a valid snapshot
286     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
287     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
288     LOG.debug("FS state after snapshot:");
289     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
290         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
291 
292     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
293         admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
294   }
295 
296   @Test (timeout=300000)
297   public void testSnapshotFailsOnNonExistantTable() throws Exception {
298     HBaseAdmin admin = UTIL.getHBaseAdmin();
299     // make sure we don't fail on listing snapshots
300     SnapshotTestingUtils.assertNoSnapshots(admin);
301     String tableName = "_not_a_table";
302 
303     // make sure the table doesn't exist
304     boolean fail = false;
305     do {
306     try {
307       admin.getTableDescriptor(Bytes.toBytes(tableName));
308       fail = true;
309       LOG.error("Table:" + tableName + " already exists, checking a new name");
310       tableName = tableName+"!";
311     } catch (TableNotFoundException e) {
312       fail = false;
313       }
314     } while (fail);
315 
316     // snapshot the non-existant table
317     try {
318       admin.snapshot("fail", tableName, SnapshotDescription.Type.FLUSH);
319       fail("Snapshot succeeded even though there is not table.");
320     } catch (SnapshotCreationException e) {
321       LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
322     }
323   }
324 
325   @Test(timeout = 300000)
326   public void testAsyncFlushSnapshot() throws Exception {
327     HBaseAdmin admin = UTIL.getHBaseAdmin();
328     SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("asyncSnapshot")
329         .setTable(TABLE_NAME.getNameAsString())
330         .setType(SnapshotDescription.Type.FLUSH)
331         .build();
332 
333     // take the snapshot async
334     admin.takeSnapshotAsync(snapshot);
335 
336     // constantly loop, looking for the snapshot to complete
337     HMaster master = UTIL.getMiniHBaseCluster().getMaster();
338     SnapshotTestingUtils.waitForSnapshotToComplete(master, snapshot, 200);
339     LOG.info(" === Async Snapshot Completed ===");
340     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
341       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
342     // make sure we get the snapshot
343     SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot);
344   }
345 
346   @Test (timeout=300000)
347   public void testSnapshotStateAfterMerge() throws Exception {
348     int numRows = DEFAULT_NUM_ROWS;
349     HBaseAdmin admin = UTIL.getHBaseAdmin();
350     // make sure we don't fail on listing snapshots
351     SnapshotTestingUtils.assertNoSnapshots(admin);
352     // load the table so we have some data
353     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
354 
355     // Take a snapshot
356     String snapshotBeforeMergeName = "snapshotBeforeMerge";
357     admin.snapshot(snapshotBeforeMergeName, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
358 
359     // Clone the table
360     String cloneBeforeMergeName = "cloneBeforeMerge";
361     admin.cloneSnapshot(snapshotBeforeMergeName, cloneBeforeMergeName);
362     SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TableName.valueOf(cloneBeforeMergeName));
363 
364     // Merge two regions
365     List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
366     Collections.sort(regions, new Comparator<HRegionInfo>() {
367       public int compare(HRegionInfo r1, HRegionInfo r2) {
368         return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
369       }
370     });
371 
372     int numRegions = admin.getTableRegions(TABLE_NAME).size();
373     int numRegionsAfterMerge = numRegions - 2;
374     admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(),
375         regions.get(2).getEncodedNameAsBytes(), true);
376     admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(),
377         regions.get(6).getEncodedNameAsBytes(), true);
378 
379     // Verify that there's one region less
380     waitRegionsAfterMerge(numRegionsAfterMerge);
381     assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
382 
383     // Clone the table
384     String cloneAfterMergeName = "cloneAfterMerge";
385     admin.cloneSnapshot(snapshotBeforeMergeName, cloneAfterMergeName);
386     SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TableName.valueOf(cloneAfterMergeName));
387 
388     SnapshotTestingUtils.verifyRowCount(UTIL, TABLE_NAME, numRows);
389     SnapshotTestingUtils.verifyRowCount(UTIL, TableName.valueOf(cloneBeforeMergeName), numRows);
390     SnapshotTestingUtils.verifyRowCount(UTIL, TableName.valueOf(cloneAfterMergeName), numRows);
391 
392     // test that we can delete the snapshot
393     UTIL.deleteTable(cloneAfterMergeName);
394     UTIL.deleteTable(cloneBeforeMergeName);
395   }
396 
397   @Test (timeout=300000)
398   public void testTakeSnapshotAfterMerge() throws Exception {
399     int numRows = DEFAULT_NUM_ROWS;
400     HBaseAdmin admin = UTIL.getHBaseAdmin();
401     // make sure we don't fail on listing snapshots
402     SnapshotTestingUtils.assertNoSnapshots(admin);
403     // load the table so we have some data
404     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
405 
406     // Merge two regions
407     List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
408     Collections.sort(regions, new Comparator<HRegionInfo>() {
409       public int compare(HRegionInfo r1, HRegionInfo r2) {
410         return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
411       }
412     });
413 
414     int numRegions = admin.getTableRegions(TABLE_NAME).size();
415     int numRegionsAfterMerge = numRegions - 2;
416     admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(),
417         regions.get(2).getEncodedNameAsBytes(), true);
418     admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(),
419         regions.get(6).getEncodedNameAsBytes(), true);
420 
421     waitRegionsAfterMerge(numRegionsAfterMerge);
422     assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
423 
424     // Take a snapshot
425     String snapshotName = "snapshotAfterMerge";
426     SnapshotTestingUtils.snapshot(admin, snapshotName, STRING_TABLE_NAME,
427       SnapshotDescription.Type.FLUSH, 3);
428 
429     // Clone the table
430     String cloneName = "cloneMerge";
431     admin.cloneSnapshot(snapshotName, cloneName);
432     SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TableName.valueOf(cloneName));
433 
434     SnapshotTestingUtils.verifyRowCount(UTIL, TABLE_NAME, numRows);
435     SnapshotTestingUtils.verifyRowCount(UTIL, TableName.valueOf(cloneName), numRows);
436 
437     // test that we can delete the snapshot
438     UTIL.deleteTable(cloneName);
439   }
440 
441   /**
442    * Basic end-to-end test of simple-flush-based snapshots
443    */
444   @Test (timeout=300000)
445   public void testFlushCreateListDestroy() throws Exception {
446     LOG.debug("------- Starting Snapshot test -------------");
447     HBaseAdmin admin = UTIL.getHBaseAdmin();
448     // make sure we don't fail on listing snapshots
449     SnapshotTestingUtils.assertNoSnapshots(admin);
450     // load the table so we have some data
451     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
452 
453     String snapshotName = "flushSnapshotCreateListDestroy";
454     // test creating the snapshot
455     admin.snapshot(snapshotName, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
456     logFSTree(FSUtils.getRootDir(UTIL.getConfiguration()));
457 
458     // make sure we only have 1 matching snapshot
459     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
460       snapshotName, TABLE_NAME);
461 
462     // check the directory structure
463     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
464     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
465     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshots.get(0), rootDir);
466     assertTrue(fs.exists(snapshotDir));
467     FSUtils.logFileSystemState(UTIL.getTestFileSystem(), snapshotDir, LOG);
468     Path snapshotinfo = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
469     assertTrue(fs.exists(snapshotinfo));
470 
471     // check the table info
472     HTableDescriptor desc = FSTableDescriptors.getTableDescriptorFromFs(fs,
473         rootDir, TABLE_NAME);
474     HTableDescriptor snapshotDesc = FSTableDescriptors.getTableDescriptorFromFs(fs,
475         new Path(SnapshotDescriptionUtils.getSnapshotsDir(rootDir), snapshotName));
476     assertEquals(desc, snapshotDesc);
477 
478     // check the region snapshot for all the regions
479     List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
480     assertTrue(regions.size() > 1);
481     for (HRegionInfo info : regions) {
482       String regionName = info.getEncodedName();
483       Path regionDir = new Path(snapshotDir, regionName);
484       HRegionInfo snapshotRegionInfo = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
485       assertEquals(info, snapshotRegionInfo);
486       // check to make sure we have the family
487       Path familyDir = new Path(regionDir, Bytes.toString(TEST_FAM));
488       assertTrue("Missing region " + Bytes.toString(snapshotRegionInfo.getStartKey()),
489                  fs.exists(familyDir));
490 
491       // make sure we have some file references
492       assertTrue(fs.listStatus(familyDir).length > 0);
493     }
494   }
495 
496   /**
497    * Demonstrate that we reject snapshot requests if there is a snapshot already running on the
498    * same table currently running and that concurrent snapshots on different tables can both
499    * succeed concurretly.
500    */
501   @Test(timeout=300000)
502   public void testConcurrentSnapshottingAttempts() throws IOException, InterruptedException {
503     final String STRING_TABLE2_NAME = STRING_TABLE_NAME + "2";
504     final TableName TABLE2_NAME =
505         TableName.valueOf(STRING_TABLE2_NAME);
506 
507     int ssNum = 20;
508     HBaseAdmin admin = UTIL.getHBaseAdmin();
509     // make sure we don't fail on listing snapshots
510     SnapshotTestingUtils.assertNoSnapshots(admin);
511     // create second testing table
512     SnapshotTestingUtils.createTable(UTIL, TABLE2_NAME, TEST_FAM);
513     // load the table so we have some data
514     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
515     SnapshotTestingUtils.loadData(UTIL, TABLE2_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
516 
517     final CountDownLatch toBeSubmitted = new CountDownLatch(ssNum);
518     // We'll have one of these per thread
519     class SSRunnable implements Runnable {
520       SnapshotDescription ss;
521       SSRunnable(SnapshotDescription ss) {
522         this.ss = ss;
523       }
524 
525       @Override
526       public void run() {
527         try {
528           HBaseAdmin admin = UTIL.getHBaseAdmin();
529           LOG.info("Submitting snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
530           admin.takeSnapshotAsync(ss);
531         } catch (Exception e) {
532           LOG.info("Exception during snapshot request: " + ClientSnapshotDescriptionUtils.toString(
533               ss)
534               + ".  This is ok, we expect some", e);
535         }
536         LOG.info("Submitted snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
537         toBeSubmitted.countDown();
538       }
539     };
540 
541     // build descriptions
542     SnapshotDescription[] descs = new SnapshotDescription[ssNum];
543     for (int i = 0; i < ssNum; i++) {
544       SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
545       builder.setTable(((i % 2) == 0 ? TABLE_NAME : TABLE2_NAME).getNameAsString());
546       builder.setName("ss"+i);
547       builder.setType(SnapshotDescription.Type.FLUSH);
548       descs[i] = builder.build();
549     }
550 
551     // kick each off its own thread
552     for (int i=0 ; i < ssNum; i++) {
553       new Thread(new SSRunnable(descs[i])).start();
554     }
555 
556     // wait until all have been submitted
557     toBeSubmitted.await();
558 
559     // loop until all are done.
560     while (true) {
561       int doneCount = 0;
562       for (SnapshotDescription ss : descs) {
563         try {
564           if (admin.isSnapshotFinished(ss)) {
565             doneCount++;
566           }
567         } catch (Exception e) {
568           LOG.warn("Got an exception when checking for snapshot " + ss.getName(), e);
569           doneCount++;
570         }
571       }
572       if (doneCount == descs.length) {
573         break;
574       }
575       Thread.sleep(100);
576     }
577 
578     // dump for debugging
579     logFSTree(FSUtils.getRootDir(UTIL.getConfiguration()));
580 
581     List<SnapshotDescription> taken = admin.listSnapshots();
582     int takenSize = taken.size();
583     LOG.info("Taken " + takenSize + " snapshots:  " + taken);
584     assertTrue("We expect at least 1 request to be rejected because of we concurrently" +
585         " issued many requests", takenSize < ssNum && takenSize > 0);
586 
587     // Verify that there's at least one snapshot per table
588     int t1SnapshotsCount = 0;
589     int t2SnapshotsCount = 0;
590     for (SnapshotDescription ss : taken) {
591       if (TableName.valueOf(ss.getTable()).equals(TABLE_NAME)) {
592         t1SnapshotsCount++;
593       } else if (TableName.valueOf(ss.getTable()).equals(TABLE2_NAME)) {
594         t2SnapshotsCount++;
595       }
596     }
597     assertTrue("We expect at least 1 snapshot of table1 ", t1SnapshotsCount > 0);
598     assertTrue("We expect at least 1 snapshot of table2 ", t2SnapshotsCount > 0);
599 
600     UTIL.deleteTable(TABLE2_NAME);
601   }
602 
603   private void logFSTree(Path root) throws IOException {
604     FSUtils.logFileSystemState(UTIL.getDFSCluster().getFileSystem(), root, LOG);
605   }
606 
607   private void waitRegionsAfterMerge(final long numRegionsAfterMerge)
608       throws IOException, InterruptedException {
609     HBaseAdmin admin = UTIL.getHBaseAdmin();
610     // Verify that there's one region less
611     long startTime = System.currentTimeMillis();
612     while (admin.getTableRegions(TABLE_NAME).size() != numRegionsAfterMerge) {
613       // This may be flaky... if after 15sec the merge is not complete give up
614       // it will fail in the assertEquals(numRegionsAfterMerge).
615       if ((System.currentTimeMillis() - startTime) > 15000)
616         break;
617       Thread.sleep(100);
618     }
619     SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TABLE_NAME);
620   }
621 }