1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.client;
20
21 import java.io.IOException;
22 import java.util.List;
23
24 import org.apache.commons.logging.Log;
25 import org.apache.commons.logging.LogFactory;
26 import org.apache.hadoop.conf.Configuration;
27 import org.apache.hadoop.fs.FileSystem;
28 import org.apache.hadoop.fs.Path;
29 import org.apache.hadoop.hbase.TableName;
30 import org.apache.hadoop.hbase.HBaseTestingUtility;
31 import org.apache.hadoop.hbase.HColumnDescriptor;
32 import org.apache.hadoop.hbase.HConstants;
33 import org.apache.hadoop.hbase.HRegionInfo;
34 import org.apache.hadoop.hbase.HTableDescriptor;
35 import org.apache.hadoop.hbase.LargeTests;
36 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
37 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
38 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
39 import org.apache.hadoop.hbase.util.Bytes;
40 import org.junit.After;
41 import org.junit.AfterClass;
42 import org.junit.Assert;
43 import org.junit.Before;
44 import org.junit.BeforeClass;
45 import org.junit.Test;
46 import org.junit.experimental.categories.Category;
47
48
49
50
51 @Category(LargeTests.class)
52 public class TestSnapshotCloneIndependence {
53 private static final Log LOG = LogFactory.getLog(TestSnapshotCloneIndependence.class);
54
55 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
56
57 private static final int NUM_RS = 2;
58 private static final String STRING_TABLE_NAME = "test";
59 private static final String TEST_FAM_STR = "fam";
60 private static final byte[] TEST_FAM = Bytes.toBytes(TEST_FAM_STR);
61 private static final byte[] TABLE_NAME = Bytes.toBytes(STRING_TABLE_NAME);
62
63
64
65
66
67 @BeforeClass
68 public static void setupCluster() throws Exception {
69 setupConf(UTIL.getConfiguration());
70 UTIL.startMiniCluster(NUM_RS);
71 }
72
73 private static void setupConf(Configuration conf) {
74
75 conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
76
77 conf.setInt("hbase.regionsever.info.port", -1);
78
79 conf.setInt("hbase.hregion.memstore.flush.size", 25000);
80
81
82 conf.setInt("hbase.hstore.compaction.min", 10);
83 conf.setInt("hbase.hstore.compactionThreshold", 10);
84
85 conf.setInt("hbase.hstore.blockingStoreFiles", 12);
86 conf.setInt("hbase.regionserver.msginterval", 100);
87 conf.setBoolean("hbase.master.enabletable.roundrobin", true);
88
89 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
90 ConstantSizeRegionSplitPolicy.class.getName());
91 }
92
93 @Before
94 public void setup() throws Exception {
95 UTIL.createTable(TABLE_NAME, TEST_FAM);
96 }
97
98 @After
99 public void tearDown() throws Exception {
100 UTIL.deleteTable(TABLE_NAME);
101 SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
102 SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
103 }
104
105 @AfterClass
106 public static void cleanupTest() throws Exception {
107 try {
108 UTIL.shutdownMiniCluster();
109 } catch (Exception e) {
110 LOG.warn("failure shutting down cluster", e);
111 }
112 }
113
114
115
116
117
118 @Test (timeout=300000)
119 public void testOnlineSnapshotAppendIndependent() throws Exception {
120 runTestSnapshotAppendIndependent(true);
121 }
122
123
124
125
126
127 @Test (timeout=300000)
128 public void testOfflineSnapshotAppendIndependent() throws Exception {
129 runTestSnapshotAppendIndependent(false);
130 }
131
132
133
134
135
136 @Test (timeout=300000)
137 public void testOnlineSnapshotMetadataChangesIndependent() throws Exception {
138 runTestSnapshotMetadataChangesIndependent(true);
139 }
140
141
142
143
144
145 @Test (timeout=300000)
146 public void testOfflineSnapshotMetadataChangesIndependent() throws Exception {
147 runTestSnapshotMetadataChangesIndependent(false);
148 }
149
150
151
152
153
154 @Test (timeout=300000)
155 public void testOfflineSnapshotRegionOperationsIndependent() throws Exception {
156 runTestRegionOperationsIndependent(false);
157 }
158
159
160
161
162
163 @Test (timeout=300000)
164 public void testOnlineSnapshotRegionOperationsIndependent() throws Exception {
165 runTestRegionOperationsIndependent(true);
166 }
167
168 private static void waitOnSplit(final HTable t, int originalCount) throws Exception {
169 for (int i = 0; i < 200; i++) {
170 try {
171 Thread.sleep(50);
172 } catch (InterruptedException e) {
173
174 Thread.currentThread().interrupt();
175 }
176 if (t.getRegionLocations().size() > originalCount) {
177 return;
178 }
179 }
180 throw new Exception("Split did not increase the number of regions");
181 }
182
183
184
185
186
187
188 private void runTestSnapshotAppendIndependent(boolean online) throws Exception {
189 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
190 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
191
192 HBaseAdmin admin = UTIL.getHBaseAdmin();
193 final long startTime = System.currentTimeMillis();
194 final TableName localTableName =
195 TableName.valueOf(STRING_TABLE_NAME + startTime);
196
197 HTable original = UTIL.createTable(localTableName, TEST_FAM);
198 try {
199
200 UTIL.loadTable(original, TEST_FAM);
201 final int origTableRowCount = UTIL.countRows(original);
202
203
204 final String snapshotNameAsString = "snapshot_" + localTableName;
205 byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
206
207 SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
208 snapshotNameAsString, rootDir, fs, online);
209
210 if (!online) {
211 admin.enableTable(localTableName);
212 }
213 byte[] cloneTableName = Bytes.toBytes("test-clone-" + localTableName);
214 admin.cloneSnapshot(snapshotName, cloneTableName);
215
216 HTable clonedTable = new HTable(UTIL.getConfiguration(), cloneTableName);
217
218 try {
219 final int clonedTableRowCount = UTIL.countRows(clonedTable);
220
221 Assert.assertEquals(
222 "The line counts of original and cloned tables do not match after clone. ",
223 origTableRowCount, clonedTableRowCount);
224
225
226 final String rowKey = "new-row-" + System.currentTimeMillis();
227
228 Put p = new Put(Bytes.toBytes(rowKey));
229 p.add(TEST_FAM, Bytes.toBytes("someQualifier"), Bytes.toBytes("someString"));
230 original.put(p);
231 original.flushCommits();
232
233
234 Assert.assertEquals("The row count of the original table was not modified by the put",
235 origTableRowCount + 1, UTIL.countRows(original));
236 Assert.assertEquals(
237 "The row count of the cloned table changed as a result of addition to the original",
238 clonedTableRowCount, UTIL.countRows(clonedTable));
239
240 p = new Put(Bytes.toBytes(rowKey));
241 p.add(TEST_FAM, Bytes.toBytes("someQualifier"), Bytes.toBytes("someString"));
242 clonedTable.put(p);
243 clonedTable.flushCommits();
244
245
246 Assert.assertEquals(
247 "The row count of the original table was modified by the put to the clone",
248 origTableRowCount + 1, UTIL.countRows(original));
249 Assert.assertEquals("The row count of the cloned table was not modified by the put",
250 clonedTableRowCount + 1, UTIL.countRows(clonedTable));
251 } finally {
252
253 clonedTable.close();
254 }
255 } finally {
256
257 original.close();
258 }
259 }
260
261
262
263
264
265 private void runTestRegionOperationsIndependent(boolean online) throws Exception {
266 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
267 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
268
269
270 HBaseAdmin admin = UTIL.getHBaseAdmin();
271 final long startTime = System.currentTimeMillis();
272 final TableName localTableName =
273 TableName.valueOf(STRING_TABLE_NAME + startTime);
274 HTable original = UTIL.createTable(localTableName, TEST_FAM);
275 UTIL.loadTable(original, TEST_FAM);
276 final int loadedTableCount = UTIL.countRows(original);
277 System.out.println("Original table has: " + loadedTableCount + " rows");
278
279 final String snapshotNameAsString = "snapshot_" + localTableName;
280
281
282 SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
283 snapshotNameAsString, rootDir, fs, online);
284
285 if (!online) {
286 admin.enableTable(localTableName);
287 }
288
289 byte[] cloneTableName = Bytes.toBytes("test-clone-" + localTableName);
290
291
292 byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
293 admin.cloneSnapshot(snapshotName, cloneTableName);
294
295
296 original.clearRegionCache();
297 List<HRegionInfo> originalTableHRegions = admin.getTableRegions(localTableName);
298
299 final int originalRegionCount = originalTableHRegions.size();
300 final int cloneTableRegionCount = admin.getTableRegions(cloneTableName).size();
301 Assert.assertEquals(
302 "The number of regions in the cloned table is different than in the original table.",
303 originalRegionCount, cloneTableRegionCount);
304
305
306 admin.split(originalTableHRegions.get(0).getRegionName());
307 waitOnSplit(original, originalRegionCount);
308
309
310 final int cloneTableRegionCount2 = admin.getTableRegions(cloneTableName).size();
311 Assert.assertEquals(
312 "The number of regions in the cloned table changed though none of its regions were split.",
313 cloneTableRegionCount, cloneTableRegionCount2);
314 }
315
316
317
318
319
320
321 private void runTestSnapshotMetadataChangesIndependent(boolean online) throws Exception {
322 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
323 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
324
325
326 HBaseAdmin admin = UTIL.getHBaseAdmin();
327 final long startTime = System.currentTimeMillis();
328 final TableName localTableName =
329 TableName.valueOf(STRING_TABLE_NAME + startTime);
330 HTable original = UTIL.createTable(localTableName, TEST_FAM);
331 UTIL.loadTable(original, TEST_FAM);
332
333 final String snapshotNameAsString = "snapshot_" + localTableName;
334
335
336 SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
337 snapshotNameAsString, rootDir, fs, online);
338
339 if (!online) {
340 admin.enableTable(localTableName);
341 }
342 byte[] cloneTableName = Bytes.toBytes("test-clone-" + localTableName);
343
344
345 byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
346 admin.cloneSnapshot(snapshotName, cloneTableName);
347
348
349 byte[] TEST_FAM_2 = Bytes.toBytes("fam2");
350 HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAM_2);
351
352 admin.disableTable(localTableName);
353 admin.addColumn(localTableName, hcd);
354
355
356 admin.enableTable(localTableName);
357
358
359
360
361 HTableDescriptor originalTableDescriptor = original.getTableDescriptor();
362 HTableDescriptor clonedTableDescriptor = admin.getTableDescriptor(cloneTableName);
363
364 Assert.assertTrue("The original family was not found. There is something wrong. ",
365 originalTableDescriptor.hasFamily(TEST_FAM));
366 Assert.assertTrue("The original family was not found in the clone. There is something wrong. ",
367 clonedTableDescriptor.hasFamily(TEST_FAM));
368
369 Assert.assertTrue("The new family was not found. ",
370 originalTableDescriptor.hasFamily(TEST_FAM_2));
371 Assert.assertTrue("The new family was not found. ",
372 !clonedTableDescriptor.hasFamily(TEST_FAM_2));
373 }
374 }