1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.client;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertTrue;
23
24 import java.util.ArrayList;
25 import java.util.Collections;
26 import java.util.List;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.fs.FileSystem;
32 import org.apache.hadoop.fs.Path;
33 import org.apache.hadoop.hbase.HBaseTestingUtility;
34 import org.apache.hadoop.hbase.HColumnDescriptor;
35 import org.apache.hadoop.hbase.HConstants;
36 import org.apache.hadoop.hbase.HTableDescriptor;
37 import org.apache.hadoop.hbase.testclassification.MediumTests;
38 import org.apache.hadoop.hbase.TableName;
39 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
40 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
41 import org.apache.hadoop.hbase.regionserver.BloomType;
42 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
43 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
44 import org.apache.hadoop.hbase.util.Bytes;
45 import org.junit.After;
46 import org.junit.AfterClass;
47 import org.junit.Before;
48 import org.junit.BeforeClass;
49 import org.junit.Test;
50 import org.junit.experimental.categories.Category;
51
52
53
54
55 @Category(MediumTests.class)
56 public class TestSnapshotMetadata {
57 private static final Log LOG = LogFactory.getLog(TestSnapshotMetadata.class);
58
59 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
60 private static final int NUM_RS = 2;
61 private static final String STRING_TABLE_NAME = "TestSnapshotMetadata";
62
63 private static final String MAX_VERSIONS_FAM_STR = "fam_max_columns";
64 private static final byte[] MAX_VERSIONS_FAM = Bytes.toBytes(MAX_VERSIONS_FAM_STR);
65
66 private static final String COMPRESSED_FAM_STR = "fam_compressed";
67 private static final byte[] COMPRESSED_FAM = Bytes.toBytes(COMPRESSED_FAM_STR);
68
69 private static final String BLOCKSIZE_FAM_STR = "fam_blocksize";
70 private static final byte[] BLOCKSIZE_FAM = Bytes.toBytes(BLOCKSIZE_FAM_STR);
71
72 private static final String BLOOMFILTER_FAM_STR = "fam_bloomfilter";
73 private static final byte[] BLOOMFILTER_FAM = Bytes.toBytes(BLOOMFILTER_FAM_STR);
74
75 private static final String TEST_CONF_CUSTOM_VALUE = "TestCustomConf";
76 private static final String TEST_CUSTOM_VALUE = "TestCustomValue";
77
78 private static final byte[][] families = {
79 MAX_VERSIONS_FAM, BLOOMFILTER_FAM, COMPRESSED_FAM, BLOCKSIZE_FAM
80 };
81
82 private static final DataBlockEncoding DATA_BLOCK_ENCODING_TYPE = DataBlockEncoding.FAST_DIFF;
83 private static final BloomType BLOOM_TYPE = BloomType.ROW;
84 private static final int BLOCK_SIZE = 98;
85 private static final int MAX_VERSIONS = 8;
86
87 private Admin admin;
88 private String originalTableDescription;
89 private HTableDescriptor originalTableDescriptor;
90 TableName originalTableName;
91
92 private static FileSystem fs;
93 private static Path rootDir;
94
95 @BeforeClass
96 public static void setupCluster() throws Exception {
97 setupConf(UTIL.getConfiguration());
98 UTIL.startMiniCluster(NUM_RS);
99
100 fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
101 rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
102 }
103
104 @AfterClass
105 public static void cleanupTest() throws Exception {
106 try {
107 UTIL.shutdownMiniCluster();
108 } catch (Exception e) {
109 LOG.warn("failure shutting down cluster", e);
110 }
111 }
112
113 private static void setupConf(Configuration conf) {
114
115 conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
116
117 conf.setInt("hbase.regionsever.info.port", -1);
118
119 conf.setInt("hbase.hregion.memstore.flush.size", 25000);
120
121
122 conf.setInt("hbase.hstore.compaction.min", 10);
123 conf.setInt("hbase.hstore.compactionThreshold", 10);
124
125 conf.setInt("hbase.hstore.blockingStoreFiles", 12);
126 conf.setInt("hbase.regionserver.msginterval", 100);
127 conf.setBoolean("hbase.master.enabletable.roundrobin", true);
128
129 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
130 ConstantSizeRegionSplitPolicy.class.getName());
131 }
132
133 @Before
134 public void setup() throws Exception {
135 admin = UTIL.getHBaseAdmin();
136 createTableWithNonDefaultProperties();
137 }
138
139 @After
140 public void tearDown() throws Exception {
141 SnapshotTestingUtils.deleteAllSnapshots(admin);
142 }
143
144
145
146
147 private void createTableWithNonDefaultProperties() throws Exception {
148 final long startTime = System.currentTimeMillis();
149 final String sourceTableNameAsString = STRING_TABLE_NAME + startTime;
150 originalTableName = TableName.valueOf(sourceTableNameAsString);
151
152
153 HColumnDescriptor maxVersionsColumn = new HColumnDescriptor(MAX_VERSIONS_FAM);
154 HColumnDescriptor bloomFilterColumn = new HColumnDescriptor(BLOOMFILTER_FAM);
155 HColumnDescriptor dataBlockColumn = new HColumnDescriptor(COMPRESSED_FAM);
156 HColumnDescriptor blockSizeColumn = new HColumnDescriptor(BLOCKSIZE_FAM);
157
158 maxVersionsColumn.setMaxVersions(MAX_VERSIONS);
159 bloomFilterColumn.setBloomFilterType(BLOOM_TYPE);
160 dataBlockColumn.setDataBlockEncoding(DATA_BLOCK_ENCODING_TYPE);
161 blockSizeColumn.setBlocksize(BLOCK_SIZE);
162
163 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(sourceTableNameAsString));
164 htd.addFamily(maxVersionsColumn);
165 htd.addFamily(bloomFilterColumn);
166 htd.addFamily(dataBlockColumn);
167 htd.addFamily(blockSizeColumn);
168 htd.setValue(TEST_CUSTOM_VALUE, TEST_CUSTOM_VALUE);
169 htd.setConfiguration(TEST_CONF_CUSTOM_VALUE, TEST_CONF_CUSTOM_VALUE);
170 assertTrue(htd.getConfiguration().size() > 0);
171
172 admin.createTable(htd);
173 Table original = new HTable(UTIL.getConfiguration(), originalTableName);
174 originalTableName = TableName.valueOf(sourceTableNameAsString);
175 originalTableDescriptor = admin.getTableDescriptor(originalTableName);
176 originalTableDescription = originalTableDescriptor.toStringCustomizedValues();
177
178 original.close();
179 }
180
181
182
183
184
185 @Test (timeout=300000)
186 public void testDescribeMatchesAfterClone() throws Exception {
187
188 final String clonedTableNameAsString = "clone" + originalTableName;
189 final TableName clonedTableName = TableName.valueOf(clonedTableNameAsString);
190 final String snapshotNameAsString = "snapshot" + originalTableName
191 + System.currentTimeMillis();
192 final byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
193
194
195 List<byte[]> familiesList = new ArrayList<byte[]>();
196 Collections.addAll(familiesList, families);
197
198
199 SnapshotTestingUtils.createSnapshotAndValidate(admin, originalTableName, null,
200 familiesList, snapshotNameAsString, rootDir, fs,
201
202 admin.cloneSnapshot(snapshotName, clonedTableName);
203 Table clonedTable = new HTable(UTIL.getConfiguration(), clonedTableName);
204 HTableDescriptor cloneHtd = admin.getTableDescriptor(clonedTableName);
205 assertEquals(
206 originalTableDescription.replace(originalTableName.getNameAsString(),clonedTableNameAsString),
207 cloneHtd.toStringCustomizedValues());
208
209
210 assertEquals(originalTableDescriptor.getValues().size(),
211 cloneHtd.getValues().size());
212 assertEquals(originalTableDescriptor.getConfiguration().size(),
213 cloneHtd.getConfiguration().size());
214 assertEquals(cloneHtd.getValue(TEST_CUSTOM_VALUE), TEST_CUSTOM_VALUE);
215 assertEquals(cloneHtd.getConfigurationValue(TEST_CONF_CUSTOM_VALUE), TEST_CONF_CUSTOM_VALUE);
216 assertEquals(originalTableDescriptor.getValues(), cloneHtd.getValues());
217 assertEquals(originalTableDescriptor.getConfiguration(), cloneHtd.getConfiguration());
218
219 admin.enableTable(originalTableName);
220 clonedTable.close();
221 }
222
223
224
225
226 @Test (timeout=300000)
227 public void testDescribeMatchesAfterRestore() throws Exception {
228 runRestoreWithAdditionalMetadata(false);
229 }
230
231
232
233
234
235 @Test (timeout=300000)
236 public void testDescribeMatchesAfterMetadataChangeAndRestore() throws Exception {
237 runRestoreWithAdditionalMetadata(true);
238 }
239
240
241
242
243
244
245 @Test (timeout=300000)
246 public void testDescribeOnEmptyTableMatchesAfterMetadataChangeAndRestore() throws Exception {
247 runRestoreWithAdditionalMetadata(true, false);
248 }
249
250 private void runRestoreWithAdditionalMetadata(boolean changeMetadata) throws Exception {
251 runRestoreWithAdditionalMetadata(changeMetadata, true);
252 }
253
254 private void runRestoreWithAdditionalMetadata(boolean changeMetadata, boolean addData)
255 throws Exception {
256
257 if (admin.isTableDisabled(originalTableName)) {
258 admin.enableTable(originalTableName);
259 }
260
261
262 final byte[] familyForUpdate = BLOCKSIZE_FAM;
263
264 List<byte[]> familiesWithDataList = new ArrayList<byte[]>();
265 List<byte[]> emptyFamiliesList = new ArrayList<byte[]>();
266 if (addData) {
267 HTable original = new HTable(UTIL.getConfiguration(), originalTableName);
268 UTIL.loadTable(original, familyForUpdate);
269 original.close();
270
271 for (byte[] family : families) {
272 if (family != familyForUpdate) {
273 emptyFamiliesList.add(family);
274 }
275 }
276 familiesWithDataList.add(familyForUpdate);
277 } else {
278 Collections.addAll(emptyFamiliesList, families);
279 }
280
281
282 final String snapshotNameAsString = "snapshot" + originalTableName
283 + System.currentTimeMillis();
284 final byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
285
286 SnapshotTestingUtils.createSnapshotAndValidate(admin, originalTableName,
287 familiesWithDataList, emptyFamiliesList, snapshotNameAsString, rootDir, fs,
288
289
290 admin.enableTable(originalTableName);
291
292 if (changeMetadata) {
293 final String newFamilyNameAsString = "newFamily" + System.currentTimeMillis();
294 final byte[] newFamilyName = Bytes.toBytes(newFamilyNameAsString);
295
296 admin.disableTable(originalTableName);
297 HColumnDescriptor hcd = new HColumnDescriptor(newFamilyName);
298 admin.addColumn(originalTableName, hcd);
299 assertTrue("New column family was not added.",
300 admin.getTableDescriptor(originalTableName).toString().contains(newFamilyNameAsString));
301 }
302
303
304 if (!admin.isTableDisabled(originalTableName)) {
305 admin.disableTable(originalTableName);
306 }
307
308 admin.restoreSnapshot(snapshotName);
309 admin.enableTable(originalTableName);
310
311
312 Table original = new HTable(UTIL.getConfiguration(), originalTableName);
313 try {
314 assertTrue(originalTableDescriptor.equals(admin.getTableDescriptor(originalTableName)));
315 assertTrue(originalTableDescriptor.equals(original.getTableDescriptor()));
316 } finally {
317 original.close();
318 }
319 }
320 }