View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.client;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.fail;
22  
23  import java.util.HashSet;
24  import java.util.List;
25  import java.util.Set;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.fs.FileSystem;
31  import org.apache.hadoop.fs.Path;
32  import org.apache.hadoop.hbase.TableName;
33  import org.apache.hadoop.hbase.HBaseTestingUtility;
34  import org.apache.hadoop.hbase.HConstants;
35  import org.apache.hadoop.hbase.LargeTests;
36  import org.apache.hadoop.hbase.TableNotFoundException;
37  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
38  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
39  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
40  import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
41  import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
42  import org.apache.hadoop.hbase.util.Bytes;
43  import org.apache.hadoop.hbase.util.FSUtils;
44  import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
45  import org.junit.After;
46  import org.junit.AfterClass;
47  import org.junit.Before;
48  import org.junit.BeforeClass;
49  import org.junit.Test;
50  import org.junit.experimental.categories.Category;
51  
52  import com.google.common.collect.Lists;
53  
54  /**
55   * Test create/using/deleting snapshots from the client
56   * <p>
57   * This is an end-to-end test for the snapshot utility
58   */
59  @Category(LargeTests.class)
60  public class TestSnapshotFromClient {
61    private static final Log LOG = LogFactory.getLog(TestSnapshotFromClient.class);
62    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
63    private static final int NUM_RS = 2;
64    private static final String STRING_TABLE_NAME = "test";
65    private static final byte[] TEST_FAM = Bytes.toBytes("fam");
66    private static final TableName TABLE_NAME =
67        TableName.valueOf(STRING_TABLE_NAME);
68  
69    /**
70     * Setup the config for the cluster
71     * @throws Exception on failure
72     */
73    @BeforeClass
74    public static void setupCluster() throws Exception {
75      setupConf(UTIL.getConfiguration());
76      UTIL.startMiniCluster(NUM_RS);
77    }
78  
79    private static void setupConf(Configuration conf) {
80      // disable the ui
81      conf.setInt("hbase.regionsever.info.port", -1);
82      // change the flush size to a small amount, regulating number of store files
83      conf.setInt("hbase.hregion.memstore.flush.size", 25000);
84      // so make sure we get a compaction when doing a load, but keep around some
85      // files in the store
86      conf.setInt("hbase.hstore.compaction.min", 10);
87      conf.setInt("hbase.hstore.compactionThreshold", 10);
88      // block writes if we get to 12 store files
89      conf.setInt("hbase.hstore.blockingStoreFiles", 12);
90      // Enable snapshot
91      conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
92      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
93        ConstantSizeRegionSplitPolicy.class.getName());
94    }
95  
96    @Before
97    public void setup() throws Exception {
98      UTIL.createTable(TABLE_NAME, TEST_FAM);
99    }
100 
101   @After
102   public void tearDown() throws Exception {
103     UTIL.deleteTable(TABLE_NAME);
104     SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
105     SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
106   }
107 
108   @AfterClass
109   public static void cleanupTest() throws Exception {
110     try {
111       UTIL.shutdownMiniCluster();
112     } catch (Exception e) {
113       LOG.warn("failure shutting down cluster", e);
114     }
115   }
116 
117   /**
118    * Test snapshotting not allowed hbase:meta and -ROOT-
119    * @throws Exception
120    */
121   @Test (timeout=300000)
122   public void testMetaTablesSnapshot() throws Exception {
123     HBaseAdmin admin = UTIL.getHBaseAdmin();
124     byte[] snapshotName = Bytes.toBytes("metaSnapshot");
125 
126     try {
127       admin.snapshot(snapshotName, TableName.META_TABLE_NAME);
128       fail("taking a snapshot of hbase:meta should not be allowed");
129     } catch (IllegalArgumentException e) {
130       // expected
131     }
132   }
133 
134   /**
135    * Test HBaseAdmin#deleteSnapshots(String) which deletes snapshots whose names match the parameter
136    *
137    * @throws Exception
138    */
139   @Test (timeout=300000)
140   public void testSnapshotDeletionWithRegex() throws Exception {
141     HBaseAdmin admin = UTIL.getHBaseAdmin();
142     // make sure we don't fail on listing snapshots
143     SnapshotTestingUtils.assertNoSnapshots(admin);
144 
145     // put some stuff in the table
146     HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
147     UTIL.loadTable(table, TEST_FAM);
148     table.close();
149 
150     byte[] snapshot1 = Bytes.toBytes("TableSnapshot1");
151     admin.snapshot(snapshot1, TABLE_NAME);
152     LOG.debug("Snapshot1 completed.");
153 
154     byte[] snapshot2 = Bytes.toBytes("TableSnapshot2");
155     admin.snapshot(snapshot2, TABLE_NAME);
156     LOG.debug("Snapshot2 completed.");
157 
158     String snapshot3 = "3rdTableSnapshot";
159     admin.snapshot(Bytes.toBytes(snapshot3), TABLE_NAME);
160     LOG.debug(snapshot3 + " completed.");
161 
162     // delete the first two snapshots
163     admin.deleteSnapshots("TableSnapshot.*");
164     List<SnapshotDescription> snapshots = admin.listSnapshots();
165     assertEquals(1, snapshots.size());
166     assertEquals(snapshots.get(0).getName(), snapshot3);
167 
168     admin.deleteSnapshot(snapshot3);
169     admin.close();
170   }
171   /**
172    * Test snapshotting a table that is offline
173    * @throws Exception
174    */
175   @Test (timeout=300000)
176   public void testOfflineTableSnapshot() throws Exception {
177     HBaseAdmin admin = UTIL.getHBaseAdmin();
178     // make sure we don't fail on listing snapshots
179     SnapshotTestingUtils.assertNoSnapshots(admin);
180 
181     // put some stuff in the table
182     HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
183     UTIL.loadTable(table, TEST_FAM, false);
184 
185     // get the name of all the regionservers hosting the snapshotted table
186     Set<String> snapshotServers = new HashSet<String>();
187     List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
188     for (RegionServerThread server : servers) {
189       if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
190         snapshotServers.add(server.getRegionServer().getServerName().toString());
191       }
192     }
193 
194     LOG.debug("FS state before disable:");
195     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
196       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
197     // XXX if this is flakey, might want to consider using the async version and looping as
198     // disableTable can succeed and still timeout.
199     admin.disableTable(TABLE_NAME);
200 
201     LOG.debug("FS state before snapshot:");
202     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
203       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
204 
205     // take a snapshot of the disabled table
206     byte[] snapshot = Bytes.toBytes("offlineTableSnapshot");
207     admin.snapshot(snapshot, TABLE_NAME);
208     LOG.debug("Snapshot completed.");
209 
210     // make sure we have the snapshot
211     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
212       snapshot, TABLE_NAME);
213 
214     // make sure its a valid snapshot
215     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
216     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
217     LOG.debug("FS state after snapshot:");
218     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
219       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
220 
221     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
222       admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
223 
224     admin.deleteSnapshot(snapshot);
225     snapshots = admin.listSnapshots();
226     SnapshotTestingUtils.assertNoSnapshots(admin);
227   }
228 
229   @Test (timeout=300000)
230   public void testSnapshotFailsOnNonExistantTable() throws Exception {
231     HBaseAdmin admin = UTIL.getHBaseAdmin();
232     // make sure we don't fail on listing snapshots
233     SnapshotTestingUtils.assertNoSnapshots(admin);
234     String tableName = "_not_a_table";
235 
236     // make sure the table doesn't exist
237     boolean fail = false;
238     do {
239     try {
240       admin.getTableDescriptor(Bytes.toBytes(tableName));
241       fail = true;
242           LOG.error("Table:" + tableName + " already exists, checking a new name");
243       tableName = tableName+"!";
244     } catch (TableNotFoundException e) {
245       fail = false;
246       }
247     } while (fail);
248 
249     // snapshot the non-existant table
250     try {
251       admin.snapshot("fail", tableName);
252       fail("Snapshot succeeded even though there is not table.");
253     } catch (SnapshotCreationException e) {
254       LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
255     }
256   }
257 
258   @Test (timeout=300000)
259   public void testOfflineTableSnapshotWithEmptyRegions() throws Exception {
260     // test with an empty table with one region
261 
262     HBaseAdmin admin = UTIL.getHBaseAdmin();
263     // make sure we don't fail on listing snapshots
264     SnapshotTestingUtils.assertNoSnapshots(admin);
265 
266     // get the name of all the regionservers hosting the snapshotted table
267     Set<String> snapshotServers = new HashSet<String>();
268     List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
269     for (RegionServerThread server : servers) {
270       if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
271         snapshotServers.add(server.getRegionServer().getServerName().toString());
272       }
273     }
274 
275     LOG.debug("FS state before disable:");
276     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
277       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
278     admin.disableTable(TABLE_NAME);
279 
280     LOG.debug("FS state before snapshot:");
281     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
282       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
283 
284     // take a snapshot of the disabled table
285     byte[] snapshot = Bytes.toBytes("testOfflineTableSnapshotWithEmptyRegions");
286     admin.snapshot(snapshot, TABLE_NAME);
287     LOG.debug("Snapshot completed.");
288 
289     // make sure we have the snapshot
290     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
291       snapshot, TABLE_NAME);
292 
293     // make sure its a valid snapshot
294     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
295     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
296     LOG.debug("FS state after snapshot:");
297     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
298       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
299 
300     List<byte[]> emptyCfs = Lists.newArrayList(TEST_FAM); // no file in the region
301     List<byte[]> nonEmptyCfs = Lists.newArrayList();
302     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, nonEmptyCfs, emptyCfs, rootDir,
303       admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
304 
305     admin.deleteSnapshot(snapshot);
306     snapshots = admin.listSnapshots();
307     SnapshotTestingUtils.assertNoSnapshots(admin);
308   }
309 }