1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.client;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertFalse;
22 import static org.junit.Assert.fail;
23
24 import java.io.IOException;
25 import java.util.HashSet;
26 import java.util.Set;
27
28 import org.apache.hadoop.fs.Path;
29 import org.apache.hadoop.hbase.CategoryBasedTimeout;
30 import org.apache.hadoop.hbase.HBaseTestingUtility;
31 import org.apache.hadoop.hbase.HColumnDescriptor;
32 import org.apache.hadoop.hbase.HConstants;
33 import org.apache.hadoop.hbase.HTableDescriptor;
34 import org.apache.hadoop.hbase.testclassification.LargeTests;
35 import org.apache.hadoop.hbase.TableName;
36 import org.apache.hadoop.hbase.master.MasterFileSystem;
37 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
38 import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
39 import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
40 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
41 import org.apache.hadoop.hbase.util.Bytes;
42 import org.apache.hadoop.hbase.util.FSUtils;
43 import org.junit.After;
44 import org.junit.AfterClass;
45 import org.junit.Before;
46 import org.junit.BeforeClass;
47 import org.junit.Rule;
48 import org.junit.Test;
49 import org.junit.experimental.categories.Category;
50 import org.junit.rules.TestRule;
51
52
53
54
55 @Category(LargeTests.class)
56 public class TestRestoreSnapshotFromClient {
57 @Rule public final TestRule timeout = CategoryBasedTimeout.builder()
58 .withTimeout(this.getClass())
59 .withLookingForStuckThread(true)
60 .build();
61
62 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
63
64 private final byte[] FAMILY = Bytes.toBytes("cf");
65
66 private byte[] emptySnapshot;
67 private byte[] snapshotName0;
68 private byte[] snapshotName1;
69 private byte[] snapshotName2;
70 private int snapshot0Rows;
71 private int snapshot1Rows;
72 private TableName tableName;
73 private Admin admin;
74
75 @BeforeClass
76 public static void setUpBeforeClass() throws Exception {
77 TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
78 TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true);
79 TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
80 TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
81 TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
82 TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
83 TEST_UTIL.getConfiguration().setBoolean(
84 "hbase.master.enabletable.roundrobin", true);
85 TEST_UTIL.startMiniCluster(3);
86 }
87
88 @AfterClass
89 public static void tearDownAfterClass() throws Exception {
90 TEST_UTIL.shutdownMiniCluster();
91 }
92
93
94
95
96
97
98 @Before
99 public void setup() throws Exception {
100 this.admin = TEST_UTIL.getHBaseAdmin();
101
102 long tid = System.currentTimeMillis();
103 tableName =
104 TableName.valueOf("testtb-" + tid);
105 emptySnapshot = Bytes.toBytes("emptySnaptb-" + tid);
106 snapshotName0 = Bytes.toBytes("snaptb0-" + tid);
107 snapshotName1 = Bytes.toBytes("snaptb1-" + tid);
108 snapshotName2 = Bytes.toBytes("snaptb2-" + tid);
109
110
111 SnapshotTestingUtils.createTable(TEST_UTIL, tableName, getNumReplicas(), FAMILY);
112 admin.disableTable(tableName);
113
114
115 admin.snapshot(emptySnapshot, tableName);
116
117
118 admin.enableTable(tableName);
119 SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, FAMILY);
120 try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
121 snapshot0Rows = TEST_UTIL.countRows(table);
122 }
123 admin.disableTable(tableName);
124
125
126 admin.snapshot(snapshotName0, tableName);
127
128
129 admin.enableTable(tableName);
130 SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, FAMILY);
131 try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
132 snapshot1Rows = TEST_UTIL.countRows(table);
133 }
134 }
135
136 @After
137 public void tearDown() throws Exception {
138 TEST_UTIL.deleteTable(tableName);
139 SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getHBaseAdmin());
140 SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
141 }
142
143 @Test
144 public void testRestoreSnapshot() throws IOException {
145 SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
146 admin.disableTable(tableName);
147 admin.snapshot(snapshotName1, tableName);
148
149 admin.restoreSnapshot(snapshotName0);
150 admin.enableTable(tableName);
151 SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
152 SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
153
154
155 admin.disableTable(tableName);
156 admin.restoreSnapshot(emptySnapshot);
157 admin.enableTable(tableName);
158 SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, 0);
159 SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
160
161
162 admin.disableTable(tableName);
163 admin.restoreSnapshot(snapshotName1);
164 admin.enableTable(tableName);
165 SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
166 SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
167
168
169 TEST_UTIL.deleteTable(tableName);
170 admin.restoreSnapshot(snapshotName1);
171 SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
172 SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
173 }
174
175 protected int getNumReplicas() {
176 return 1;
177 }
178
179 @Test
180 public void testRestoreSchemaChange() throws Exception {
181 byte[] TEST_FAMILY2 = Bytes.toBytes("cf2");
182
183 HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
184
185
186 admin.disableTable(tableName);
187 admin.addColumn(tableName, new HColumnDescriptor(TEST_FAMILY2));
188 admin.enableTable(tableName);
189 assertEquals(2, table.getTableDescriptor().getFamilies().size());
190 HTableDescriptor htd = admin.getTableDescriptor(tableName);
191 assertEquals(2, htd.getFamilies().size());
192 SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, TEST_FAMILY2);
193 long snapshot2Rows = snapshot1Rows + 500;
194 assertEquals(snapshot2Rows, TEST_UTIL.countRows(table));
195 assertEquals(500, TEST_UTIL.countRows(table, TEST_FAMILY2));
196 Set<String> fsFamilies = getFamiliesFromFS(tableName);
197 assertEquals(2, fsFamilies.size());
198
199
200 admin.disableTable(tableName);
201 admin.snapshot(snapshotName2, tableName);
202
203
204 admin.restoreSnapshot(snapshotName0);
205 admin.enableTable(tableName);
206 assertEquals(1, table.getTableDescriptor().getFamilies().size());
207 try {
208 TEST_UTIL.countRows(table, TEST_FAMILY2);
209 fail("family '" + Bytes.toString(TEST_FAMILY2) + "' should not exists");
210 } catch (NoSuchColumnFamilyException e) {
211
212 }
213 assertEquals(snapshot0Rows, TEST_UTIL.countRows(table));
214 htd = admin.getTableDescriptor(tableName);
215 assertEquals(1, htd.getFamilies().size());
216 fsFamilies = getFamiliesFromFS(tableName);
217 assertEquals(1, fsFamilies.size());
218
219
220 admin.disableTable(tableName);
221 admin.restoreSnapshot(snapshotName2);
222 admin.enableTable(tableName);
223 htd = admin.getTableDescriptor(tableName);
224 assertEquals(2, htd.getFamilies().size());
225 assertEquals(2, table.getTableDescriptor().getFamilies().size());
226 assertEquals(500, TEST_UTIL.countRows(table, TEST_FAMILY2));
227 assertEquals(snapshot2Rows, TEST_UTIL.countRows(table));
228 fsFamilies = getFamiliesFromFS(tableName);
229 assertEquals(2, fsFamilies.size());
230 table.close();
231 }
232
233 @Test
234 public void testCloneSnapshotOfCloned() throws IOException, InterruptedException {
235 TableName clonedTableName =
236 TableName.valueOf("clonedtb-" + System.currentTimeMillis());
237 admin.cloneSnapshot(snapshotName0, clonedTableName);
238 SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
239 SnapshotTestingUtils.verifyReplicasCameOnline(clonedTableName, admin, getNumReplicas());
240 admin.disableTable(clonedTableName);
241 admin.snapshot(snapshotName2, clonedTableName);
242 TEST_UTIL.deleteTable(clonedTableName);
243 waitCleanerRun();
244
245 admin.cloneSnapshot(snapshotName2, clonedTableName);
246 SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
247 SnapshotTestingUtils.verifyReplicasCameOnline(clonedTableName, admin, getNumReplicas());
248 TEST_UTIL.deleteTable(clonedTableName);
249 }
250
251 @Test
252 public void testCloneAndRestoreSnapshot() throws IOException, InterruptedException {
253 TEST_UTIL.deleteTable(tableName);
254 waitCleanerRun();
255
256 admin.cloneSnapshot(snapshotName0, tableName);
257 SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
258 SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
259 waitCleanerRun();
260
261 admin.disableTable(tableName);
262 admin.restoreSnapshot(snapshotName0);
263 admin.enableTable(tableName);
264 SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
265 SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
266 }
267
268 @Test
269 public void testCorruptedSnapshot() throws IOException, InterruptedException {
270 SnapshotTestingUtils.corruptSnapshot(TEST_UTIL, Bytes.toString(snapshotName0));
271 TableName cloneName = TableName.valueOf("corruptedClone-" + System.currentTimeMillis());
272 try {
273 admin.cloneSnapshot(snapshotName0, cloneName);
274 fail("Expected CorruptedSnapshotException, got succeeded cloneSnapshot()");
275 } catch (CorruptedSnapshotException e) {
276
277
278 assertFalse(admin.tableExists(cloneName));
279 } catch (Exception e) {
280 fail("Expected CorruptedSnapshotException got: " + e);
281 }
282 }
283
284
285
286
287 private void waitCleanerRun() throws InterruptedException {
288 TEST_UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().choreForTesting();
289 }
290
291 private Set<String> getFamiliesFromFS(final TableName tableName) throws IOException {
292 MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
293 Set<String> families = new HashSet<String>();
294 Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName);
295 for (Path regionDir: FSUtils.getRegionDirs(mfs.getFileSystem(), tableDir)) {
296 for (Path familyDir: FSUtils.getFamilyDirs(mfs.getFileSystem(), regionDir)) {
297 families.add(familyDir.getName());
298 }
299 }
300 return families;
301 }
302 }