View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.client;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertTrue;
23  
24  import java.util.ArrayList;
25  import java.util.List;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.fs.FileSystem;
31  import org.apache.hadoop.fs.Path;
32  import org.apache.hadoop.hbase.HBaseTestingUtility;
33  import org.apache.hadoop.hbase.HColumnDescriptor;
34  import org.apache.hadoop.hbase.HConstants;
35  import org.apache.hadoop.hbase.HTableDescriptor;
36  import org.apache.hadoop.hbase.MediumTests;
37  import org.apache.hadoop.hbase.TableName;
38  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
39  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
40  import org.apache.hadoop.hbase.regionserver.BloomType;
41  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
42  import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
43  import org.apache.hadoop.hbase.util.Bytes;
44  import org.junit.After;
45  import org.junit.AfterClass;
46  import org.junit.Before;
47  import org.junit.BeforeClass;
48  import org.junit.Test;
49  import org.junit.experimental.categories.Category;
50  
51  /**
52   * Test class to verify that metadata is consistent before and after a snapshot attempt.
53   */
54  @Category(MediumTests.class)
55  public class TestSnapshotMetadata {
56    private static final Log LOG = LogFactory.getLog(TestSnapshotMetadata.class);
57  
58    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
59    private static final int NUM_RS = 2;
60    private static final String STRING_TABLE_NAME = "TestSnapshotMetadata";
61  
62    private static final String MAX_VERSIONS_FAM_STR = "fam_max_columns";
63    private static final byte[] MAX_VERSIONS_FAM = Bytes.toBytes(MAX_VERSIONS_FAM_STR);
64  
65    private static final String COMPRESSED_FAM_STR = "fam_compressed";
66    private static final byte[] COMPRESSED_FAM = Bytes.toBytes(COMPRESSED_FAM_STR);
67  
68    private static final String BLOCKSIZE_FAM_STR = "fam_blocksize";
69    private static final byte[] BLOCKSIZE_FAM = Bytes.toBytes(BLOCKSIZE_FAM_STR);
70  
71    private static final String BLOOMFILTER_FAM_STR = "fam_bloomfilter";
72    private static final byte[] BLOOMFILTER_FAM = Bytes.toBytes(BLOOMFILTER_FAM_STR);
73  
74    private static final String TEST_CONF_CUSTOM_VALUE = "TestCustomConf";
75    private static final String TEST_CUSTOM_VALUE = "TestCustomValue";
76  
77    private static final byte[][] families = {
78      MAX_VERSIONS_FAM, BLOOMFILTER_FAM, COMPRESSED_FAM, BLOCKSIZE_FAM
79    };
80  
81    private static final DataBlockEncoding DATA_BLOCK_ENCODING_TYPE = DataBlockEncoding.FAST_DIFF;
82    private static final BloomType BLOOM_TYPE = BloomType.ROW;
83    private static final int BLOCK_SIZE = 98;
84    private static final int MAX_VERSIONS = 8;
85  
86    private HBaseAdmin admin;
87    private String originalTableDescription;
88    private HTableDescriptor originalTableDescriptor;
89    TableName originalTableName;
90  
91    private static FileSystem fs;
92    private static Path rootDir;
93  
94    @BeforeClass
95    public static void setupCluster() throws Exception {
96      setupConf(UTIL.getConfiguration());
97      UTIL.startMiniCluster(NUM_RS);
98  
99      fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
100     rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
101   }
102 
103   @AfterClass
104   public static void cleanupTest() throws Exception {
105     try {
106       UTIL.shutdownMiniCluster();
107     } catch (Exception e) {
108       LOG.warn("failure shutting down cluster", e);
109     }
110   }
111 
112   private static void setupConf(Configuration conf) {
113     // enable snapshot support
114     conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
115     // disable the ui
116     conf.setInt("hbase.regionsever.info.port", -1);
117     // change the flush size to a small amount, regulating number of store files
118     conf.setInt("hbase.hregion.memstore.flush.size", 25000);
119     // so make sure we get a compaction when doing a load, but keep around
120     // some files in the store
121     conf.setInt("hbase.hstore.compaction.min", 10);
122     conf.setInt("hbase.hstore.compactionThreshold", 10);
123     // block writes if we get to 12 store files
124     conf.setInt("hbase.hstore.blockingStoreFiles", 12);
125     conf.setInt("hbase.regionserver.msginterval", 100);
126     conf.setBoolean("hbase.master.enabletable.roundrobin", true);
127     // Avoid potentially aggressive splitting which would cause snapshot to fail
128     conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
129       ConstantSizeRegionSplitPolicy.class.getName());
130   }
131 
132   @Before
133   public void setup() throws Exception {
134     admin = UTIL.getHBaseAdmin();
135     createTableWithNonDefaultProperties();
136   }
137 
138   @After
139   public void tearDown() throws Exception {
140     SnapshotTestingUtils.deleteAllSnapshots(admin);
141   }
142 
143   /*
144    *  Create a table that has non-default properties so we can see if they hold
145    */
146   private void createTableWithNonDefaultProperties() throws Exception {
147     final long startTime = System.currentTimeMillis();
148     final String sourceTableNameAsString = STRING_TABLE_NAME + startTime;
149     originalTableName = TableName.valueOf(sourceTableNameAsString);
150 
151     // enable replication on a column family
152     HColumnDescriptor maxVersionsColumn = new HColumnDescriptor(MAX_VERSIONS_FAM);
153     HColumnDescriptor bloomFilterColumn = new HColumnDescriptor(BLOOMFILTER_FAM);
154     HColumnDescriptor dataBlockColumn = new HColumnDescriptor(COMPRESSED_FAM);
155     HColumnDescriptor blockSizeColumn = new HColumnDescriptor(BLOCKSIZE_FAM);
156 
157     maxVersionsColumn.setMaxVersions(MAX_VERSIONS);
158     bloomFilterColumn.setBloomFilterType(BLOOM_TYPE);
159     dataBlockColumn.setDataBlockEncoding(DATA_BLOCK_ENCODING_TYPE);
160     blockSizeColumn.setBlocksize(BLOCK_SIZE);
161 
162     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(sourceTableNameAsString));
163     htd.addFamily(maxVersionsColumn);
164     htd.addFamily(bloomFilterColumn);
165     htd.addFamily(dataBlockColumn);
166     htd.addFamily(blockSizeColumn);
167     htd.setValue(TEST_CUSTOM_VALUE, TEST_CUSTOM_VALUE);
168     htd.setConfiguration(TEST_CONF_CUSTOM_VALUE, TEST_CONF_CUSTOM_VALUE);
169     assertTrue(htd.getConfiguration().size() > 0);
170 
171     admin.createTable(htd);
172     HTable original = new HTable(UTIL.getConfiguration(), originalTableName);
173     originalTableName = TableName.valueOf(sourceTableNameAsString);
174     originalTableDescriptor = admin.getTableDescriptor(originalTableName);
175     originalTableDescription = originalTableDescriptor.toStringCustomizedValues();
176 
177     original.close();
178   }
179 
180 
181   /**
182    * Verify that the describe for a cloned table matches the describe from the original.
183    */
184   @Test (timeout=300000)
185   public void testDescribeMatchesAfterClone() throws Exception {
186     // Clone the original table
187     final String clonedTableNameAsString = "clone" + originalTableName;
188     final byte[] clonedTableName = Bytes.toBytes(clonedTableNameAsString);
189     final String snapshotNameAsString = "snapshot" + originalTableName
190         + System.currentTimeMillis();
191     final byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
192 
193     // restore the snapshot into a cloned table and examine the output
194     List<byte[]> familiesList = new ArrayList<byte[]>();
195     for (byte[] family : families) {
196       familiesList.add(family);
197     }
198 
199     // Create a snapshot in which all families are empty
200     SnapshotTestingUtils.createSnapshotAndValidate(admin, originalTableName, null,
201       familiesList, snapshotNameAsString, rootDir, fs, /* onlineSnapshot= */ false);
202 
203     admin.cloneSnapshot(snapshotName, clonedTableName);
204     HTable clonedTable = new HTable(UTIL.getConfiguration(), clonedTableName);
205     HTableDescriptor cloneHtd = admin.getTableDescriptor(clonedTableName);
206     assertEquals(
207       originalTableDescription.replace(originalTableName.getNameAsString(),clonedTableNameAsString),
208       cloneHtd.toStringCustomizedValues());
209 
210     // Verify the custom fields
211     assertEquals(originalTableDescriptor.getValues().size(),
212                         cloneHtd.getValues().size());
213     assertEquals(originalTableDescriptor.getConfiguration().size(),
214                         cloneHtd.getConfiguration().size());
215     assertEquals(cloneHtd.getValue(TEST_CUSTOM_VALUE), TEST_CUSTOM_VALUE);
216     assertEquals(cloneHtd.getConfigurationValue(TEST_CONF_CUSTOM_VALUE), TEST_CONF_CUSTOM_VALUE);
217     assertEquals(originalTableDescriptor.getValues(), cloneHtd.getValues());
218     assertEquals(originalTableDescriptor.getConfiguration(), cloneHtd.getConfiguration());
219 
220     admin.enableTable(originalTableName);
221     clonedTable.close();
222   }
223 
224   /**
225    * Verify that the describe for a restored table matches the describe for one the original.
226    */
227   @Test (timeout=300000)
228   public void testDescribeMatchesAfterRestore() throws Exception {
229     runRestoreWithAdditionalMetadata(false);
230   }
231 
232   /**
233    * Verify that if metadata changed after a snapshot was taken, that the old metadata replaces the
234    * new metadata during a restore
235    */
236   @Test (timeout=300000)
237   public void testDescribeMatchesAfterMetadataChangeAndRestore() throws Exception {
238     runRestoreWithAdditionalMetadata(true);
239   }
240 
241   /**
242    * Verify that when the table is empty, making metadata changes after the restore does not affect
243    * the restored table's original metadata
244    * @throws Exception
245    */
246   @Test (timeout=300000)
247   public void testDescribeOnEmptyTableMatchesAfterMetadataChangeAndRestore() throws Exception {
248     runRestoreWithAdditionalMetadata(true, false);
249   }
250 
251   private void runRestoreWithAdditionalMetadata(boolean changeMetadata) throws Exception {
252     runRestoreWithAdditionalMetadata(changeMetadata, true);
253   }
254 
255   private void runRestoreWithAdditionalMetadata(boolean changeMetadata, boolean addData)
256       throws Exception {
257 
258     if (admin.isTableDisabled(originalTableName)) {
259       admin.enableTable(originalTableName);
260     }
261 
262     // populate it with data
263     final byte[] familyForUpdate = BLOCKSIZE_FAM;
264 
265     List<byte[]> familiesWithDataList = new ArrayList<byte[]>();
266     List<byte[]> emptyFamiliesList = new ArrayList<byte[]>();
267     if (addData) {
268       HTable original = new HTable(UTIL.getConfiguration(), originalTableName);
269       UTIL.loadTable(original, familyForUpdate); // family arbitrarily chosen
270       original.close();
271 
272       for (byte[] family : families) {
273         if (family != familyForUpdate) {
274           emptyFamiliesList.add(family);
275         }
276       }
277       familiesWithDataList.add(familyForUpdate);
278     } else {
279       for (byte[] family : families) {
280         emptyFamiliesList.add(family);
281       }
282     }
283 
284     // take a "disabled" snapshot
285     final String snapshotNameAsString = "snapshot" + originalTableName
286         + System.currentTimeMillis();
287     final byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
288 
289     SnapshotTestingUtils.createSnapshotAndValidate(admin, originalTableName,
290       familiesWithDataList, emptyFamiliesList, snapshotNameAsString, rootDir, fs,
291       /* onlineSnapshot= */ false);
292 
293     admin.enableTable(originalTableName);
294 
295     if (changeMetadata) {
296       final String newFamilyNameAsString = "newFamily" + System.currentTimeMillis();
297       final byte[] newFamilyName = Bytes.toBytes(newFamilyNameAsString);
298 
299       admin.disableTable(originalTableName);
300       HColumnDescriptor hcd = new HColumnDescriptor(newFamilyName);
301       admin.addColumn(originalTableName, hcd);
302       assertTrue("New column family was not added.",
303         admin.getTableDescriptor(originalTableName).toString().contains(newFamilyNameAsString));
304     }
305 
306     // restore it
307     if (!admin.isTableDisabled(originalTableName)) {
308       admin.disableTable(originalTableName);
309     }
310 
311     admin.restoreSnapshot(snapshotName);
312     admin.enableTable(originalTableName);
313 
314     // verify that the descrption is reverted
315     HTable original = new HTable(UTIL.getConfiguration(), originalTableName);
316     try {
317       assertTrue(originalTableDescriptor.equals(admin.getTableDescriptor(originalTableName)));
318       assertTrue(originalTableDescriptor.equals(original.getTableDescriptor()));
319     } finally {
320       original.close();
321     }
322   }
323 }