View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.client;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertTrue;
23  
24  import java.util.ArrayList;
25  import java.util.Collections;
26  import java.util.List;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.fs.FileSystem;
32  import org.apache.hadoop.fs.Path;
33  import org.apache.hadoop.hbase.HBaseTestingUtility;
34  import org.apache.hadoop.hbase.HColumnDescriptor;
35  import org.apache.hadoop.hbase.HConstants;
36  import org.apache.hadoop.hbase.HTableDescriptor;
37  import org.apache.hadoop.hbase.testclassification.MediumTests;
38  import org.apache.hadoop.hbase.TableName;
39  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
40  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
41  import org.apache.hadoop.hbase.regionserver.BloomType;
42  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
43  import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
44  import org.apache.hadoop.hbase.util.Bytes;
45  import org.junit.After;
46  import org.junit.AfterClass;
47  import org.junit.Before;
48  import org.junit.BeforeClass;
49  import org.junit.Test;
50  import org.junit.experimental.categories.Category;
51  
52  /**
53   * Test class to verify that metadata is consistent before and after a snapshot attempt.
54   */
55  @Category(MediumTests.class)
56  public class TestSnapshotMetadata {
57    private static final Log LOG = LogFactory.getLog(TestSnapshotMetadata.class);
58  
59    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
60    private static final int NUM_RS = 2;
61    private static final String STRING_TABLE_NAME = "TestSnapshotMetadata";
62  
63    private static final String MAX_VERSIONS_FAM_STR = "fam_max_columns";
64    private static final byte[] MAX_VERSIONS_FAM = Bytes.toBytes(MAX_VERSIONS_FAM_STR);
65  
66    private static final String COMPRESSED_FAM_STR = "fam_compressed";
67    private static final byte[] COMPRESSED_FAM = Bytes.toBytes(COMPRESSED_FAM_STR);
68  
69    private static final String BLOCKSIZE_FAM_STR = "fam_blocksize";
70    private static final byte[] BLOCKSIZE_FAM = Bytes.toBytes(BLOCKSIZE_FAM_STR);
71  
72    private static final String BLOOMFILTER_FAM_STR = "fam_bloomfilter";
73    private static final byte[] BLOOMFILTER_FAM = Bytes.toBytes(BLOOMFILTER_FAM_STR);
74  
75    private static final String TEST_CONF_CUSTOM_VALUE = "TestCustomConf";
76    private static final String TEST_CUSTOM_VALUE = "TestCustomValue";
77  
78    private static final byte[][] families = {
79      MAX_VERSIONS_FAM, BLOOMFILTER_FAM, COMPRESSED_FAM, BLOCKSIZE_FAM
80    };
81  
82    private static final DataBlockEncoding DATA_BLOCK_ENCODING_TYPE = DataBlockEncoding.FAST_DIFF;
83    private static final BloomType BLOOM_TYPE = BloomType.ROW;
84    private static final int BLOCK_SIZE = 98;
85    private static final int MAX_VERSIONS = 8;
86  
87    private Admin admin;
88    private String originalTableDescription;
89    private HTableDescriptor originalTableDescriptor;
90    TableName originalTableName;
91  
92    private static FileSystem fs;
93    private static Path rootDir;
94  
95    @BeforeClass
96    public static void setupCluster() throws Exception {
97      setupConf(UTIL.getConfiguration());
98      UTIL.startMiniCluster(NUM_RS);
99  
100     fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
101     rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
102   }
103 
104   @AfterClass
105   public static void cleanupTest() throws Exception {
106     try {
107       UTIL.shutdownMiniCluster();
108     } catch (Exception e) {
109       LOG.warn("failure shutting down cluster", e);
110     }
111   }
112 
113   private static void setupConf(Configuration conf) {
114     // enable snapshot support
115     conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
116     // disable the ui
117     conf.setInt("hbase.regionsever.info.port", -1);
118     // change the flush size to a small amount, regulating number of store files
119     conf.setInt("hbase.hregion.memstore.flush.size", 25000);
120     // so make sure we get a compaction when doing a load, but keep around
121     // some files in the store
122     conf.setInt("hbase.hstore.compaction.min", 10);
123     conf.setInt("hbase.hstore.compactionThreshold", 10);
124     // block writes if we get to 12 store files
125     conf.setInt("hbase.hstore.blockingStoreFiles", 12);
126     conf.setInt("hbase.regionserver.msginterval", 100);
127     conf.setBoolean("hbase.master.enabletable.roundrobin", true);
128     // Avoid potentially aggressive splitting which would cause snapshot to fail
129     conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
130       ConstantSizeRegionSplitPolicy.class.getName());
131   }
132 
133   @Before
134   public void setup() throws Exception {
135     admin = UTIL.getHBaseAdmin();
136     createTableWithNonDefaultProperties();
137   }
138 
139   @After
140   public void tearDown() throws Exception {
141     SnapshotTestingUtils.deleteAllSnapshots(admin);
142   }
143 
144   /*
145    *  Create a table that has non-default properties so we can see if they hold
146    */
147   private void createTableWithNonDefaultProperties() throws Exception {
148     final long startTime = System.currentTimeMillis();
149     final String sourceTableNameAsString = STRING_TABLE_NAME + startTime;
150     originalTableName = TableName.valueOf(sourceTableNameAsString);
151 
152     // enable replication on a column family
153     HColumnDescriptor maxVersionsColumn = new HColumnDescriptor(MAX_VERSIONS_FAM);
154     HColumnDescriptor bloomFilterColumn = new HColumnDescriptor(BLOOMFILTER_FAM);
155     HColumnDescriptor dataBlockColumn = new HColumnDescriptor(COMPRESSED_FAM);
156     HColumnDescriptor blockSizeColumn = new HColumnDescriptor(BLOCKSIZE_FAM);
157 
158     maxVersionsColumn.setMaxVersions(MAX_VERSIONS);
159     bloomFilterColumn.setBloomFilterType(BLOOM_TYPE);
160     dataBlockColumn.setDataBlockEncoding(DATA_BLOCK_ENCODING_TYPE);
161     blockSizeColumn.setBlocksize(BLOCK_SIZE);
162 
163     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(sourceTableNameAsString));
164     htd.addFamily(maxVersionsColumn);
165     htd.addFamily(bloomFilterColumn);
166     htd.addFamily(dataBlockColumn);
167     htd.addFamily(blockSizeColumn);
168     htd.setValue(TEST_CUSTOM_VALUE, TEST_CUSTOM_VALUE);
169     htd.setConfiguration(TEST_CONF_CUSTOM_VALUE, TEST_CONF_CUSTOM_VALUE);
170     assertTrue(htd.getConfiguration().size() > 0);
171 
172     admin.createTable(htd);
173     Table original = new HTable(UTIL.getConfiguration(), originalTableName);
174     originalTableName = TableName.valueOf(sourceTableNameAsString);
175     originalTableDescriptor = admin.getTableDescriptor(originalTableName);
176     originalTableDescription = originalTableDescriptor.toStringCustomizedValues();
177 
178     original.close();
179   }
180 
181 
182   /**
183    * Verify that the describe for a cloned table matches the describe from the original.
184    */
185   @Test (timeout=300000)
186   public void testDescribeMatchesAfterClone() throws Exception {
187     // Clone the original table
188     final String clonedTableNameAsString = "clone" + originalTableName;
189     final TableName clonedTableName = TableName.valueOf(clonedTableNameAsString);
190     final String snapshotNameAsString = "snapshot" + originalTableName
191         + System.currentTimeMillis();
192     final byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
193 
194     // restore the snapshot into a cloned table and examine the output
195     List<byte[]> familiesList = new ArrayList<byte[]>();
196     Collections.addAll(familiesList, families);
197 
198     // Create a snapshot in which all families are empty
199     SnapshotTestingUtils.createSnapshotAndValidate(admin, originalTableName, null,
200       familiesList, snapshotNameAsString, rootDir, fs, /* onlineSnapshot= */ false);
201 
202     admin.cloneSnapshot(snapshotName, clonedTableName);
203     Table clonedTable = new HTable(UTIL.getConfiguration(), clonedTableName);
204     HTableDescriptor cloneHtd = admin.getTableDescriptor(clonedTableName);
205     assertEquals(
206       originalTableDescription.replace(originalTableName.getNameAsString(),clonedTableNameAsString),
207       cloneHtd.toStringCustomizedValues());
208 
209     // Verify the custom fields
210     assertEquals(originalTableDescriptor.getValues().size(),
211                         cloneHtd.getValues().size());
212     assertEquals(originalTableDescriptor.getConfiguration().size(),
213                         cloneHtd.getConfiguration().size());
214     assertEquals(cloneHtd.getValue(TEST_CUSTOM_VALUE), TEST_CUSTOM_VALUE);
215     assertEquals(cloneHtd.getConfigurationValue(TEST_CONF_CUSTOM_VALUE), TEST_CONF_CUSTOM_VALUE);
216     assertEquals(originalTableDescriptor.getValues(), cloneHtd.getValues());
217     assertEquals(originalTableDescriptor.getConfiguration(), cloneHtd.getConfiguration());
218 
219     admin.enableTable(originalTableName);
220     clonedTable.close();
221   }
222 
223   /**
224    * Verify that the describe for a restored table matches the describe for one the original.
225    */
226   @Test (timeout=300000)
227   public void testDescribeMatchesAfterRestore() throws Exception {
228     runRestoreWithAdditionalMetadata(false);
229   }
230 
231   /**
232    * Verify that if metadata changed after a snapshot was taken, that the old metadata replaces the
233    * new metadata during a restore
234    */
235   @Test (timeout=300000)
236   public void testDescribeMatchesAfterMetadataChangeAndRestore() throws Exception {
237     runRestoreWithAdditionalMetadata(true);
238   }
239 
240   /**
241    * Verify that when the table is empty, making metadata changes after the restore does not affect
242    * the restored table's original metadata
243    * @throws Exception
244    */
245   @Test (timeout=300000)
246   public void testDescribeOnEmptyTableMatchesAfterMetadataChangeAndRestore() throws Exception {
247     runRestoreWithAdditionalMetadata(true, false);
248   }
249 
250   private void runRestoreWithAdditionalMetadata(boolean changeMetadata) throws Exception {
251     runRestoreWithAdditionalMetadata(changeMetadata, true);
252   }
253 
254   private void runRestoreWithAdditionalMetadata(boolean changeMetadata, boolean addData)
255       throws Exception {
256 
257     if (admin.isTableDisabled(originalTableName)) {
258       admin.enableTable(originalTableName);
259     }
260 
261     // populate it with data
262     final byte[] familyForUpdate = BLOCKSIZE_FAM;
263 
264     List<byte[]> familiesWithDataList = new ArrayList<byte[]>();
265     List<byte[]> emptyFamiliesList = new ArrayList<byte[]>();
266     if (addData) {
267       HTable original = new HTable(UTIL.getConfiguration(), originalTableName);
268       UTIL.loadTable(original, familyForUpdate); // family arbitrarily chosen
269       original.close();
270 
271       for (byte[] family : families) {
272         if (family != familyForUpdate) {
273           emptyFamiliesList.add(family);
274         }
275       }
276       familiesWithDataList.add(familyForUpdate);
277     } else {
278       Collections.addAll(emptyFamiliesList, families);
279     }
280 
281     // take a "disabled" snapshot
282     final String snapshotNameAsString = "snapshot" + originalTableName
283         + System.currentTimeMillis();
284     final byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
285 
286     SnapshotTestingUtils.createSnapshotAndValidate(admin, originalTableName,
287       familiesWithDataList, emptyFamiliesList, snapshotNameAsString, rootDir, fs,
288       /* onlineSnapshot= */ false);
289 
290     admin.enableTable(originalTableName);
291 
292     if (changeMetadata) {
293       final String newFamilyNameAsString = "newFamily" + System.currentTimeMillis();
294       final byte[] newFamilyName = Bytes.toBytes(newFamilyNameAsString);
295 
296       admin.disableTable(originalTableName);
297       HColumnDescriptor hcd = new HColumnDescriptor(newFamilyName);
298       admin.addColumn(originalTableName, hcd);
299       assertTrue("New column family was not added.",
300         admin.getTableDescriptor(originalTableName).toString().contains(newFamilyNameAsString));
301     }
302 
303     // restore it
304     if (!admin.isTableDisabled(originalTableName)) {
305       admin.disableTable(originalTableName);
306     }
307 
308     admin.restoreSnapshot(snapshotName);
309     admin.enableTable(originalTableName);
310 
311     // verify that the descrption is reverted
312     Table original = new HTable(UTIL.getConfiguration(), originalTableName);
313     try {
314       assertTrue(originalTableDescriptor.equals(admin.getTableDescriptor(originalTableName)));
315       assertTrue(originalTableDescriptor.equals(original.getTableDescriptor()));
316     } finally {
317       original.close();
318     }
319   }
320 }