1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.client;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertTrue;
23
24 import java.util.ArrayList;
25 import java.util.List;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.fs.FileSystem;
31 import org.apache.hadoop.fs.Path;
32 import org.apache.hadoop.hbase.HBaseTestingUtility;
33 import org.apache.hadoop.hbase.HColumnDescriptor;
34 import org.apache.hadoop.hbase.HConstants;
35 import org.apache.hadoop.hbase.HTableDescriptor;
36 import org.apache.hadoop.hbase.MediumTests;
37 import org.apache.hadoop.hbase.TableName;
38 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
39 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
40 import org.apache.hadoop.hbase.regionserver.BloomType;
41 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
42 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
43 import org.apache.hadoop.hbase.util.Bytes;
44 import org.junit.After;
45 import org.junit.AfterClass;
46 import org.junit.Before;
47 import org.junit.BeforeClass;
48 import org.junit.Test;
49 import org.junit.experimental.categories.Category;
50
51
52
53
54 @Category(MediumTests.class)
55 public class TestSnapshotMetadata {
56 private static final Log LOG = LogFactory.getLog(TestSnapshotMetadata.class);
57
58 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
59 private static final int NUM_RS = 2;
60 private static final String STRING_TABLE_NAME = "TestSnapshotMetadata";
61
62 private static final String MAX_VERSIONS_FAM_STR = "fam_max_columns";
63 private static final byte[] MAX_VERSIONS_FAM = Bytes.toBytes(MAX_VERSIONS_FAM_STR);
64
65 private static final String COMPRESSED_FAM_STR = "fam_compressed";
66 private static final byte[] COMPRESSED_FAM = Bytes.toBytes(COMPRESSED_FAM_STR);
67
68 private static final String BLOCKSIZE_FAM_STR = "fam_blocksize";
69 private static final byte[] BLOCKSIZE_FAM = Bytes.toBytes(BLOCKSIZE_FAM_STR);
70
71 private static final String BLOOMFILTER_FAM_STR = "fam_bloomfilter";
72 private static final byte[] BLOOMFILTER_FAM = Bytes.toBytes(BLOOMFILTER_FAM_STR);
73
74 private static final String TEST_CONF_CUSTOM_VALUE = "TestCustomConf";
75 private static final String TEST_CUSTOM_VALUE = "TestCustomValue";
76
77 private static final byte[][] families = {
78 MAX_VERSIONS_FAM, BLOOMFILTER_FAM, COMPRESSED_FAM, BLOCKSIZE_FAM
79 };
80
81 private static final DataBlockEncoding DATA_BLOCK_ENCODING_TYPE = DataBlockEncoding.FAST_DIFF;
82 private static final BloomType BLOOM_TYPE = BloomType.ROW;
83 private static final int BLOCK_SIZE = 98;
84 private static final int MAX_VERSIONS = 8;
85
86 private HBaseAdmin admin;
87 private String originalTableDescription;
88 private HTableDescriptor originalTableDescriptor;
89 TableName originalTableName;
90
91 private static FileSystem fs;
92 private static Path rootDir;
93
94 @BeforeClass
95 public static void setupCluster() throws Exception {
96 setupConf(UTIL.getConfiguration());
97 UTIL.startMiniCluster(NUM_RS);
98
99 fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
100 rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
101 }
102
103 @AfterClass
104 public static void cleanupTest() throws Exception {
105 try {
106 UTIL.shutdownMiniCluster();
107 } catch (Exception e) {
108 LOG.warn("failure shutting down cluster", e);
109 }
110 }
111
112 private static void setupConf(Configuration conf) {
113
114 conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
115
116 conf.setInt("hbase.regionsever.info.port", -1);
117
118 conf.setInt("hbase.hregion.memstore.flush.size", 25000);
119
120
121 conf.setInt("hbase.hstore.compaction.min", 10);
122 conf.setInt("hbase.hstore.compactionThreshold", 10);
123
124 conf.setInt("hbase.hstore.blockingStoreFiles", 12);
125 conf.setInt("hbase.regionserver.msginterval", 100);
126 conf.setBoolean("hbase.master.enabletable.roundrobin", true);
127
128 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
129 ConstantSizeRegionSplitPolicy.class.getName());
130 }
131
132 @Before
133 public void setup() throws Exception {
134 admin = UTIL.getHBaseAdmin();
135 createTableWithNonDefaultProperties();
136 }
137
138 @After
139 public void tearDown() throws Exception {
140 SnapshotTestingUtils.deleteAllSnapshots(admin);
141 }
142
143
144
145
146 private void createTableWithNonDefaultProperties() throws Exception {
147 final long startTime = System.currentTimeMillis();
148 final String sourceTableNameAsString = STRING_TABLE_NAME + startTime;
149 originalTableName = TableName.valueOf(sourceTableNameAsString);
150
151
152 HColumnDescriptor maxVersionsColumn = new HColumnDescriptor(MAX_VERSIONS_FAM);
153 HColumnDescriptor bloomFilterColumn = new HColumnDescriptor(BLOOMFILTER_FAM);
154 HColumnDescriptor dataBlockColumn = new HColumnDescriptor(COMPRESSED_FAM);
155 HColumnDescriptor blockSizeColumn = new HColumnDescriptor(BLOCKSIZE_FAM);
156
157 maxVersionsColumn.setMaxVersions(MAX_VERSIONS);
158 bloomFilterColumn.setBloomFilterType(BLOOM_TYPE);
159 dataBlockColumn.setDataBlockEncoding(DATA_BLOCK_ENCODING_TYPE);
160 blockSizeColumn.setBlocksize(BLOCK_SIZE);
161
162 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(sourceTableNameAsString));
163 htd.addFamily(maxVersionsColumn);
164 htd.addFamily(bloomFilterColumn);
165 htd.addFamily(dataBlockColumn);
166 htd.addFamily(blockSizeColumn);
167 htd.setValue(TEST_CUSTOM_VALUE, TEST_CUSTOM_VALUE);
168 htd.setConfiguration(TEST_CONF_CUSTOM_VALUE, TEST_CONF_CUSTOM_VALUE);
169 assertTrue(htd.getConfiguration().size() > 0);
170
171 admin.createTable(htd);
172 HTable original = new HTable(UTIL.getConfiguration(), originalTableName);
173 originalTableName = TableName.valueOf(sourceTableNameAsString);
174 originalTableDescriptor = admin.getTableDescriptor(originalTableName);
175 originalTableDescription = originalTableDescriptor.toStringCustomizedValues();
176
177 original.close();
178 }
179
180
181
182
183
184 @Test (timeout=300000)
185 public void testDescribeMatchesAfterClone() throws Exception {
186
187 final String clonedTableNameAsString = "clone" + originalTableName;
188 final byte[] clonedTableName = Bytes.toBytes(clonedTableNameAsString);
189 final String snapshotNameAsString = "snapshot" + originalTableName
190 + System.currentTimeMillis();
191 final byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
192
193
194 List<byte[]> familiesList = new ArrayList<byte[]>();
195 for (byte[] family : families) {
196 familiesList.add(family);
197 }
198
199
200 SnapshotTestingUtils.createSnapshotAndValidate(admin, originalTableName, null,
201 familiesList, snapshotNameAsString, rootDir, fs,
202
203 admin.cloneSnapshot(snapshotName, clonedTableName);
204 HTable clonedTable = new HTable(UTIL.getConfiguration(), clonedTableName);
205 HTableDescriptor cloneHtd = admin.getTableDescriptor(clonedTableName);
206 assertEquals(
207 originalTableDescription.replace(originalTableName.getNameAsString(),clonedTableNameAsString),
208 cloneHtd.toStringCustomizedValues());
209
210
211 assertEquals(originalTableDescriptor.getValues().size(),
212 cloneHtd.getValues().size());
213 assertEquals(originalTableDescriptor.getConfiguration().size(),
214 cloneHtd.getConfiguration().size());
215 assertEquals(cloneHtd.getValue(TEST_CUSTOM_VALUE), TEST_CUSTOM_VALUE);
216 assertEquals(cloneHtd.getConfigurationValue(TEST_CONF_CUSTOM_VALUE), TEST_CONF_CUSTOM_VALUE);
217 assertEquals(originalTableDescriptor.getValues(), cloneHtd.getValues());
218 assertEquals(originalTableDescriptor.getConfiguration(), cloneHtd.getConfiguration());
219
220 admin.enableTable(originalTableName);
221 clonedTable.close();
222 }
223
224
225
226
227 @Test (timeout=300000)
228 public void testDescribeMatchesAfterRestore() throws Exception {
229 runRestoreWithAdditionalMetadata(false);
230 }
231
232
233
234
235
236 @Test (timeout=300000)
237 public void testDescribeMatchesAfterMetadataChangeAndRestore() throws Exception {
238 runRestoreWithAdditionalMetadata(true);
239 }
240
241
242
243
244
245
246 @Test (timeout=300000)
247 public void testDescribeOnEmptyTableMatchesAfterMetadataChangeAndRestore() throws Exception {
248 runRestoreWithAdditionalMetadata(true, false);
249 }
250
251 private void runRestoreWithAdditionalMetadata(boolean changeMetadata) throws Exception {
252 runRestoreWithAdditionalMetadata(changeMetadata, true);
253 }
254
255 private void runRestoreWithAdditionalMetadata(boolean changeMetadata, boolean addData)
256 throws Exception {
257
258 if (admin.isTableDisabled(originalTableName)) {
259 admin.enableTable(originalTableName);
260 }
261
262
263 final byte[] familyForUpdate = BLOCKSIZE_FAM;
264
265 List<byte[]> familiesWithDataList = new ArrayList<byte[]>();
266 List<byte[]> emptyFamiliesList = new ArrayList<byte[]>();
267 if (addData) {
268 HTable original = new HTable(UTIL.getConfiguration(), originalTableName);
269 UTIL.loadTable(original, familyForUpdate);
270 original.close();
271
272 for (byte[] family : families) {
273 if (family != familyForUpdate) {
274 emptyFamiliesList.add(family);
275 }
276 }
277 familiesWithDataList.add(familyForUpdate);
278 } else {
279 for (byte[] family : families) {
280 emptyFamiliesList.add(family);
281 }
282 }
283
284
285 final String snapshotNameAsString = "snapshot" + originalTableName
286 + System.currentTimeMillis();
287 final byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
288
289 SnapshotTestingUtils.createSnapshotAndValidate(admin, originalTableName,
290 familiesWithDataList, emptyFamiliesList, snapshotNameAsString, rootDir, fs,
291
292
293 admin.enableTable(originalTableName);
294
295 if (changeMetadata) {
296 final String newFamilyNameAsString = "newFamily" + System.currentTimeMillis();
297 final byte[] newFamilyName = Bytes.toBytes(newFamilyNameAsString);
298
299 admin.disableTable(originalTableName);
300 HColumnDescriptor hcd = new HColumnDescriptor(newFamilyName);
301 admin.addColumn(originalTableName, hcd);
302 assertTrue("New column family was not added.",
303 admin.getTableDescriptor(originalTableName).toString().contains(newFamilyNameAsString));
304 }
305
306
307 if (!admin.isTableDisabled(originalTableName)) {
308 admin.disableTable(originalTableName);
309 }
310
311 admin.restoreSnapshot(snapshotName);
312 admin.enableTable(originalTableName);
313
314
315 HTable original = new HTable(UTIL.getConfiguration(), originalTableName);
316 try {
317 assertTrue(originalTableDescriptor.equals(admin.getTableDescriptor(originalTableName)));
318 assertTrue(originalTableDescriptor.equals(original.getTableDescriptor()));
319 } finally {
320 original.close();
321 }
322 }
323 }