View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.client;
20  
21  import java.io.IOException;
22  import java.util.List;
23  
24  import org.apache.commons.logging.Log;
25  import org.apache.commons.logging.LogFactory;
26  import org.apache.hadoop.conf.Configuration;
27  import org.apache.hadoop.fs.FileSystem;
28  import org.apache.hadoop.fs.Path;
29  import org.apache.hadoop.hbase.TableName;
30  import org.apache.hadoop.hbase.HBaseTestingUtility;
31  import org.apache.hadoop.hbase.HColumnDescriptor;
32  import org.apache.hadoop.hbase.HConstants;
33  import org.apache.hadoop.hbase.HRegionInfo;
34  import org.apache.hadoop.hbase.HTableDescriptor;
35  import org.apache.hadoop.hbase.LargeTests;
36  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
37  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
38  import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
39  import org.apache.hadoop.hbase.util.Bytes;
40  import org.junit.After;
41  import org.junit.AfterClass;
42  import org.junit.Assert;
43  import org.junit.Before;
44  import org.junit.BeforeClass;
45  import org.junit.Test;
46  import org.junit.experimental.categories.Category;
47  
48  /**
49   * Test to verify that the cloned table is independent of the table from which it was cloned
50   */
51  @Category(LargeTests.class)
52  public class TestSnapshotCloneIndependence {
53    private static final Log LOG = LogFactory.getLog(TestSnapshotCloneIndependence.class);
54  
55    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
56  
57    private static final int NUM_RS = 2;
58    private static final String STRING_TABLE_NAME = "test";
59    private static final String TEST_FAM_STR = "fam";
60    private static final byte[] TEST_FAM = Bytes.toBytes(TEST_FAM_STR);
61    private static final byte[] TABLE_NAME = Bytes.toBytes(STRING_TABLE_NAME);
62  
63    /**
64     * Setup the config for the cluster and start it
65     * @throws Exception on failure
66     */
67    @BeforeClass
68    public static void setupCluster() throws Exception {
69      setupConf(UTIL.getConfiguration());
70      UTIL.startMiniCluster(NUM_RS);
71    }
72  
73    private static void setupConf(Configuration conf) {
74      // enable snapshot support
75      conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
76      // disable the ui
77      conf.setInt("hbase.regionsever.info.port", -1);
78      // change the flush size to a small amount, regulating number of store files
79      conf.setInt("hbase.hregion.memstore.flush.size", 25000);
80      // so make sure we get a compaction when doing a load, but keep around
81      // some files in the store
82      conf.setInt("hbase.hstore.compaction.min", 10);
83      conf.setInt("hbase.hstore.compactionThreshold", 10);
84      // block writes if we get to 12 store files
85      conf.setInt("hbase.hstore.blockingStoreFiles", 12);
86      conf.setInt("hbase.regionserver.msginterval", 100);
87      conf.setBoolean("hbase.master.enabletable.roundrobin", true);
88      // Avoid potentially aggressive splitting which would cause snapshot to fail
89      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
90        ConstantSizeRegionSplitPolicy.class.getName());
91    }
92  
93    @Before
94    public void setup() throws Exception {
95      UTIL.createTable(TABLE_NAME, TEST_FAM);
96    }
97  
98    @After
99    public void tearDown() throws Exception {
100     UTIL.deleteTable(TABLE_NAME);
101     SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
102     SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
103   }
104 
105   @AfterClass
106   public static void cleanupTest() throws Exception {
107     try {
108       UTIL.shutdownMiniCluster();
109     } catch (Exception e) {
110       LOG.warn("failure shutting down cluster", e);
111     }
112   }
113 
114   /**
115    * Verify that adding data to the cloned table will not affect the original, and vice-versa when
116    * it is taken as an online snapshot.
117    */
118   @Test (timeout=300000)
119   public void testOnlineSnapshotAppendIndependent() throws Exception {
120     runTestSnapshotAppendIndependent(true);
121   }
122 
123   /**
124    * Verify that adding data to the cloned table will not affect the original, and vice-versa when
125    * it is taken as an offline snapshot.
126    */
127   @Test (timeout=300000)
128   public void testOfflineSnapshotAppendIndependent() throws Exception {
129     runTestSnapshotAppendIndependent(false);
130   }
131 
132   /**
133    * Verify that adding metadata to the cloned table will not affect the original, and vice-versa
134    * when it is taken as an online snapshot.
135    */
136   @Test (timeout=300000)
137   public void testOnlineSnapshotMetadataChangesIndependent() throws Exception {
138     runTestSnapshotMetadataChangesIndependent(true);
139   }
140 
141   /**
142    * Verify that adding netadata to the cloned table will not affect the original, and vice-versa
143    * when is taken as an online snapshot.
144    */
145   @Test (timeout=300000)
146   public void testOfflineSnapshotMetadataChangesIndependent() throws Exception {
147     runTestSnapshotMetadataChangesIndependent(false);
148   }
149 
150   /**
151    * Verify that region operations, in this case splitting a region, are independent between the
152    * cloned table and the original.
153    */
154   @Test (timeout=300000)
155   public void testOfflineSnapshotRegionOperationsIndependent() throws Exception {
156     runTestRegionOperationsIndependent(false);
157   }
158 
159   /**
160    * Verify that region operations, in this case splitting a region, are independent between the
161    * cloned table and the original.
162    */
163   @Test (timeout=300000)
164   public void testOnlineSnapshotRegionOperationsIndependent() throws Exception {
165     runTestRegionOperationsIndependent(true);
166   }
167 
168   private static void waitOnSplit(final HTable t, int originalCount) throws Exception {
169     for (int i = 0; i < 200; i++) {
170       try {
171         Thread.sleep(50);
172       } catch (InterruptedException e) {
173         // Restore the interrupted status
174         Thread.currentThread().interrupt();
175       }
176       if (t.getRegionLocations().size() > originalCount) {
177         return;
178       }
179     }
180     throw new Exception("Split did not increase the number of regions");
181   }
182 
183   /*
184    * Take a snapshot of a table, add data, and verify that this only
185    * affects one table
186    * @param online - Whether the table is online or not during the snapshot
187    */
188   private void runTestSnapshotAppendIndependent(boolean online) throws Exception {
189     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
190     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
191 
192     HBaseAdmin admin = UTIL.getHBaseAdmin();
193     final long startTime = System.currentTimeMillis();
194     final TableName localTableName =
195         TableName.valueOf(STRING_TABLE_NAME + startTime);
196 
197     HTable original = UTIL.createTable(localTableName, TEST_FAM);
198     try {
199 
200       UTIL.loadTable(original, TEST_FAM);
201       final int origTableRowCount = UTIL.countRows(original);
202 
203       // Take a snapshot
204       final String snapshotNameAsString = "snapshot_" + localTableName;
205       byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
206 
207       SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
208         snapshotNameAsString, rootDir, fs, online);
209 
210       if (!online) {
211         admin.enableTable(localTableName);
212       }
213       byte[] cloneTableName = Bytes.toBytes("test-clone-" + localTableName);
214       admin.cloneSnapshot(snapshotName, cloneTableName);
215 
216       HTable clonedTable = new HTable(UTIL.getConfiguration(), cloneTableName);
217 
218       try {
219         final int clonedTableRowCount = UTIL.countRows(clonedTable);
220 
221         Assert.assertEquals(
222           "The line counts of original and cloned tables do not match after clone. ",
223           origTableRowCount, clonedTableRowCount);
224 
225         // Attempt to add data to the test
226         final String rowKey = "new-row-" + System.currentTimeMillis();
227 
228         Put p = new Put(Bytes.toBytes(rowKey));
229         p.add(TEST_FAM, Bytes.toBytes("someQualifier"), Bytes.toBytes("someString"));
230         original.put(p);
231         original.flushCommits();
232 
233         // Verify that it is not present in the original table
234         Assert.assertEquals("The row count of the original table was not modified by the put",
235           origTableRowCount + 1, UTIL.countRows(original));
236         Assert.assertEquals(
237           "The row count of the cloned table changed as a result of addition to the original",
238           clonedTableRowCount, UTIL.countRows(clonedTable));
239 
240         p = new Put(Bytes.toBytes(rowKey));
241         p.add(TEST_FAM, Bytes.toBytes("someQualifier"), Bytes.toBytes("someString"));
242         clonedTable.put(p);
243         clonedTable.flushCommits();
244 
245         // Verify that the new family is not in the restored table's description
246         Assert.assertEquals(
247           "The row count of the original table was modified by the put to the clone",
248           origTableRowCount + 1, UTIL.countRows(original));
249         Assert.assertEquals("The row count of the cloned table was not modified by the put",
250           clonedTableRowCount + 1, UTIL.countRows(clonedTable));
251       } finally {
252 
253         clonedTable.close();
254       }
255     } finally {
256 
257       original.close();
258     }
259   }
260 
261   /*
262    * Take a snapshot of a table, do a split, and verify that this only affects one table
263    * @param online - Whether the table is online or not during the snapshot
264    */
265   private void runTestRegionOperationsIndependent(boolean online) throws Exception {
266     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
267     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
268 
269     // Create a table
270     HBaseAdmin admin = UTIL.getHBaseAdmin();
271     final long startTime = System.currentTimeMillis();
272     final TableName localTableName =
273         TableName.valueOf(STRING_TABLE_NAME + startTime);
274     HTable original = UTIL.createTable(localTableName, TEST_FAM);
275     UTIL.loadTable(original, TEST_FAM);
276     final int loadedTableCount = UTIL.countRows(original);
277     System.out.println("Original table has: " + loadedTableCount + " rows");
278 
279     final String snapshotNameAsString = "snapshot_" + localTableName;
280 
281     // Create a snapshot
282     SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
283       snapshotNameAsString, rootDir, fs, online);
284 
285     if (!online) {
286       admin.enableTable(localTableName);
287     }
288 
289     byte[] cloneTableName = Bytes.toBytes("test-clone-" + localTableName);
290 
291     // Clone the snapshot
292     byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
293     admin.cloneSnapshot(snapshotName, cloneTableName);
294 
295     // Verify that region information is the same pre-split
296     original.clearRegionCache();
297     List<HRegionInfo> originalTableHRegions = admin.getTableRegions(localTableName);
298 
299     final int originalRegionCount = originalTableHRegions.size();
300     final int cloneTableRegionCount = admin.getTableRegions(cloneTableName).size();
301     Assert.assertEquals(
302       "The number of regions in the cloned table is different than in the original table.",
303       originalRegionCount, cloneTableRegionCount);
304 
305     // Split a region on the parent table
306     admin.split(originalTableHRegions.get(0).getRegionName());
307     waitOnSplit(original, originalRegionCount);
308 
309     // Verify that the cloned table region is not split
310     final int cloneTableRegionCount2 = admin.getTableRegions(cloneTableName).size();
311     Assert.assertEquals(
312       "The number of regions in the cloned table changed though none of its regions were split.",
313       cloneTableRegionCount, cloneTableRegionCount2);
314   }
315 
316   /*
317    * Take a snapshot of a table, add metadata, and verify that this only
318    * affects one table
319    * @param online - Whether the table is online or not during the snapshot
320    */
321   private void runTestSnapshotMetadataChangesIndependent(boolean online) throws Exception {
322     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
323     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
324 
325     // Create a table
326     HBaseAdmin admin = UTIL.getHBaseAdmin();
327     final long startTime = System.currentTimeMillis();
328     final TableName localTableName =
329         TableName.valueOf(STRING_TABLE_NAME + startTime);
330     HTable original = UTIL.createTable(localTableName, TEST_FAM);
331     UTIL.loadTable(original, TEST_FAM);
332 
333     final String snapshotNameAsString = "snapshot_" + localTableName;
334 
335     // Create a snapshot
336     SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
337       snapshotNameAsString, rootDir, fs, online);
338 
339     if (!online) {
340       admin.enableTable(localTableName);
341     }
342     byte[] cloneTableName = Bytes.toBytes("test-clone-" + localTableName);
343 
344     // Clone the snapshot
345     byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
346     admin.cloneSnapshot(snapshotName, cloneTableName);
347 
348     // Add a new column family to the original table
349     byte[] TEST_FAM_2 = Bytes.toBytes("fam2");
350     HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAM_2);
351 
352     admin.disableTable(localTableName);
353     admin.addColumn(localTableName, hcd);
354 
355     // Verify that it is not in the snapshot
356     admin.enableTable(localTableName);
357 
358     // get a description of the cloned table
359     // get a list of its families
360     // assert that the family is there
361     HTableDescriptor originalTableDescriptor = original.getTableDescriptor();
362     HTableDescriptor clonedTableDescriptor = admin.getTableDescriptor(cloneTableName);
363 
364     Assert.assertTrue("The original family was not found. There is something wrong. ",
365       originalTableDescriptor.hasFamily(TEST_FAM));
366     Assert.assertTrue("The original family was not found in the clone. There is something wrong. ",
367       clonedTableDescriptor.hasFamily(TEST_FAM));
368 
369     Assert.assertTrue("The new family was not found. ",
370       originalTableDescriptor.hasFamily(TEST_FAM_2));
371     Assert.assertTrue("The new family was not found. ",
372       !clonedTableDescriptor.hasFamily(TEST_FAM_2));
373   }
374 }