1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.client;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.fail;
22
23 import java.util.HashSet;
24 import java.util.List;
25 import java.util.Set;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.fs.FileSystem;
31 import org.apache.hadoop.fs.Path;
32 import org.apache.hadoop.hbase.TableName;
33 import org.apache.hadoop.hbase.HBaseTestingUtility;
34 import org.apache.hadoop.hbase.HConstants;
35 import org.apache.hadoop.hbase.LargeTests;
36 import org.apache.hadoop.hbase.TableNotFoundException;
37 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
38 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
39 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
40 import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
41 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
42 import org.apache.hadoop.hbase.util.Bytes;
43 import org.apache.hadoop.hbase.util.FSUtils;
44 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
45 import org.junit.After;
46 import org.junit.AfterClass;
47 import org.junit.Before;
48 import org.junit.BeforeClass;
49 import org.junit.Test;
50 import org.junit.experimental.categories.Category;
51
52 import com.google.common.collect.Lists;
53
54
55
56
57
58
59 @Category(LargeTests.class)
60 public class TestSnapshotFromClient {
61 private static final Log LOG = LogFactory.getLog(TestSnapshotFromClient.class);
62 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
63 private static final int NUM_RS = 2;
64 private static final String STRING_TABLE_NAME = "test";
65 private static final byte[] TEST_FAM = Bytes.toBytes("fam");
66 private static final TableName TABLE_NAME =
67 TableName.valueOf(STRING_TABLE_NAME);
68
69
70
71
72
73 @BeforeClass
74 public static void setupCluster() throws Exception {
75 setupConf(UTIL.getConfiguration());
76 UTIL.startMiniCluster(NUM_RS);
77 }
78
79 private static void setupConf(Configuration conf) {
80
81 conf.setInt("hbase.regionsever.info.port", -1);
82
83 conf.setInt("hbase.hregion.memstore.flush.size", 25000);
84
85
86 conf.setInt("hbase.hstore.compaction.min", 10);
87 conf.setInt("hbase.hstore.compactionThreshold", 10);
88
89 conf.setInt("hbase.hstore.blockingStoreFiles", 12);
90
91 conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
92 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
93 ConstantSizeRegionSplitPolicy.class.getName());
94 }
95
96 @Before
97 public void setup() throws Exception {
98 UTIL.createTable(TABLE_NAME, TEST_FAM);
99 }
100
101 @After
102 public void tearDown() throws Exception {
103 UTIL.deleteTable(TABLE_NAME);
104 SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
105 SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
106 }
107
108 @AfterClass
109 public static void cleanupTest() throws Exception {
110 try {
111 UTIL.shutdownMiniCluster();
112 } catch (Exception e) {
113 LOG.warn("failure shutting down cluster", e);
114 }
115 }
116
117
118
119
120
121 @Test (timeout=300000)
122 public void testMetaTablesSnapshot() throws Exception {
123 HBaseAdmin admin = UTIL.getHBaseAdmin();
124 byte[] snapshotName = Bytes.toBytes("metaSnapshot");
125
126 try {
127 admin.snapshot(snapshotName, TableName.META_TABLE_NAME);
128 fail("taking a snapshot of hbase:meta should not be allowed");
129 } catch (IllegalArgumentException e) {
130
131 }
132 }
133
134
135
136
137
138
139 @Test (timeout=300000)
140 public void testSnapshotDeletionWithRegex() throws Exception {
141 HBaseAdmin admin = UTIL.getHBaseAdmin();
142
143 SnapshotTestingUtils.assertNoSnapshots(admin);
144
145
146 HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
147 UTIL.loadTable(table, TEST_FAM);
148 table.close();
149
150 byte[] snapshot1 = Bytes.toBytes("TableSnapshot1");
151 admin.snapshot(snapshot1, TABLE_NAME);
152 LOG.debug("Snapshot1 completed.");
153
154 byte[] snapshot2 = Bytes.toBytes("TableSnapshot2");
155 admin.snapshot(snapshot2, TABLE_NAME);
156 LOG.debug("Snapshot2 completed.");
157
158 String snapshot3 = "3rdTableSnapshot";
159 admin.snapshot(Bytes.toBytes(snapshot3), TABLE_NAME);
160 LOG.debug(snapshot3 + " completed.");
161
162
163 admin.deleteSnapshots("TableSnapshot.*");
164 List<SnapshotDescription> snapshots = admin.listSnapshots();
165 assertEquals(1, snapshots.size());
166 assertEquals(snapshots.get(0).getName(), snapshot3);
167
168 admin.deleteSnapshot(snapshot3);
169 admin.close();
170 }
171
172
173
174
175 @Test (timeout=300000)
176 public void testOfflineTableSnapshot() throws Exception {
177 HBaseAdmin admin = UTIL.getHBaseAdmin();
178
179 SnapshotTestingUtils.assertNoSnapshots(admin);
180
181
182 HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
183 UTIL.loadTable(table, TEST_FAM, false);
184
185
186 Set<String> snapshotServers = new HashSet<String>();
187 List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
188 for (RegionServerThread server : servers) {
189 if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
190 snapshotServers.add(server.getRegionServer().getServerName().toString());
191 }
192 }
193
194 LOG.debug("FS state before disable:");
195 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
196 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
197
198
199 admin.disableTable(TABLE_NAME);
200
201 LOG.debug("FS state before snapshot:");
202 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
203 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
204
205
206 byte[] snapshot = Bytes.toBytes("offlineTableSnapshot");
207 admin.snapshot(snapshot, TABLE_NAME);
208 LOG.debug("Snapshot completed.");
209
210
211 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
212 snapshot, TABLE_NAME);
213
214
215 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
216 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
217 LOG.debug("FS state after snapshot:");
218 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
219 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
220
221 SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
222 admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
223
224 admin.deleteSnapshot(snapshot);
225 snapshots = admin.listSnapshots();
226 SnapshotTestingUtils.assertNoSnapshots(admin);
227 }
228
229 @Test (timeout=300000)
230 public void testSnapshotFailsOnNonExistantTable() throws Exception {
231 HBaseAdmin admin = UTIL.getHBaseAdmin();
232
233 SnapshotTestingUtils.assertNoSnapshots(admin);
234 String tableName = "_not_a_table";
235
236
237 boolean fail = false;
238 do {
239 try {
240 admin.getTableDescriptor(Bytes.toBytes(tableName));
241 fail = true;
242 LOG.error("Table:" + tableName + " already exists, checking a new name");
243 tableName = tableName+"!";
244 } catch (TableNotFoundException e) {
245 fail = false;
246 }
247 } while (fail);
248
249
250 try {
251 admin.snapshot("fail", tableName);
252 fail("Snapshot succeeded even though there is not table.");
253 } catch (SnapshotCreationException e) {
254 LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
255 }
256 }
257
258 @Test (timeout=300000)
259 public void testOfflineTableSnapshotWithEmptyRegions() throws Exception {
260
261
262 HBaseAdmin admin = UTIL.getHBaseAdmin();
263
264 SnapshotTestingUtils.assertNoSnapshots(admin);
265
266
267 Set<String> snapshotServers = new HashSet<String>();
268 List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
269 for (RegionServerThread server : servers) {
270 if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
271 snapshotServers.add(server.getRegionServer().getServerName().toString());
272 }
273 }
274
275 LOG.debug("FS state before disable:");
276 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
277 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
278 admin.disableTable(TABLE_NAME);
279
280 LOG.debug("FS state before snapshot:");
281 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
282 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
283
284
285 byte[] snapshot = Bytes.toBytes("testOfflineTableSnapshotWithEmptyRegions");
286 admin.snapshot(snapshot, TABLE_NAME);
287 LOG.debug("Snapshot completed.");
288
289
290 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
291 snapshot, TABLE_NAME);
292
293
294 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
295 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
296 LOG.debug("FS state after snapshot:");
297 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
298 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
299
300 List<byte[]> emptyCfs = Lists.newArrayList(TEST_FAM);
301 List<byte[]> nonEmptyCfs = Lists.newArrayList();
302 SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, nonEmptyCfs, emptyCfs, rootDir,
303 admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
304
305 admin.deleteSnapshot(snapshot);
306 snapshots = admin.listSnapshots();
307 SnapshotTestingUtils.assertNoSnapshots(admin);
308 }
309 }