View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.backup;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertTrue;
23  
24  import java.io.IOException;
25  import java.util.ArrayList;
26  import java.util.Collections;
27  import java.util.List;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.fs.FileStatus;
33  import org.apache.hadoop.fs.FileSystem;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.fs.PathFilter;
36  import org.apache.hadoop.hbase.HBaseTestingUtility;
37  import org.apache.hadoop.hbase.HConstants;
38  import org.apache.hadoop.hbase.MediumTests;
39  import org.apache.hadoop.hbase.Stoppable;
40  import org.apache.hadoop.hbase.TableName;
41  import org.apache.hadoop.hbase.client.HBaseAdmin;
42  import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
43  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
44  import org.apache.hadoop.hbase.regionserver.HRegion;
45  import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
46  import org.apache.hadoop.hbase.regionserver.HRegionServer;
47  import org.apache.hadoop.hbase.util.Bytes;
48  import org.apache.hadoop.hbase.util.FSUtils;
49  import org.apache.hadoop.hbase.util.HFileArchiveTestingUtil;
50  import org.apache.hadoop.hbase.util.HFileArchiveUtil;
51  import org.apache.hadoop.hbase.util.StoppableImplementation;
52  import org.junit.After;
53  import org.junit.AfterClass;
54  import org.junit.Assert;
55  import org.junit.BeforeClass;
56  import org.junit.Test;
57  import org.junit.experimental.categories.Category;
58  
59  /**
60   * Test that the {@link HFileArchiver} correctly removes all the parts of a region when cleaning up
61   * a region
62   */
63  @Category(MediumTests.class)
64  public class TestHFileArchiving {
65  
66    private static final Log LOG = LogFactory.getLog(TestHFileArchiving.class);
67    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
68    private static final byte[] TEST_FAM = Bytes.toBytes("fam");
69  
70    /**
71     * Setup the config for the cluster
72     */
73    @BeforeClass
74    public static void setupCluster() throws Exception {
75      setupConf(UTIL.getConfiguration());
76      UTIL.startMiniCluster();
77  
78      // We don't want the cleaner to remove files. The tests do that.
79      UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().interrupt();
80    }
81  
82    private static void setupConf(Configuration conf) {
83      // disable the ui
84      conf.setInt("hbase.regionsever.info.port", -1);
85      // drop the memstore size so we get flushes
86      conf.setInt("hbase.hregion.memstore.flush.size", 25000);
87      // disable major compactions
88      conf.setInt(HConstants.MAJOR_COMPACTION_PERIOD, 0);
89  
90      // prevent aggressive region split
91      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
92        ConstantSizeRegionSplitPolicy.class.getName());
93    }
94  
95    @After
96    public void tearDown() throws Exception {
97      // cleanup the archive directory
98      try {
99        clearArchiveDirectory();
100     } catch (IOException e) {
101       Assert.fail("Failure to delete archive directory:" + e.getMessage());
102     }
103   }
104 
105   @AfterClass
106   public static void cleanupTest() throws Exception {
107     try {
108       UTIL.shutdownMiniCluster();
109     } catch (Exception e) {
110       // NOOP;
111     }
112   }
113 
114   @Test
115   public void testRemovesRegionDirOnArchive() throws Exception {
116     TableName TABLE_NAME =
117         TableName.valueOf("testRemovesRegionDirOnArchive");
118     UTIL.createTable(TABLE_NAME, TEST_FAM);
119 
120     final HBaseAdmin admin = UTIL.getHBaseAdmin();
121 
122     // get the current store files for the region
123     List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
124     // make sure we only have 1 region serving this table
125     assertEquals(1, servingRegions.size());
126     HRegion region = servingRegions.get(0);
127 
128     // and load the table
129     UTIL.loadRegion(region, TEST_FAM);
130 
131     // shutdown the table so we can manipulate the files
132     admin.disableTable(TABLE_NAME);
133 
134     FileSystem fs = UTIL.getTestFileSystem();
135 
136     // now attempt to depose the region
137     Path rootDir = region.getRegionFileSystem().getTableDir().getParent();
138     Path regionDir = HRegion.getRegionDir(rootDir, region.getRegionInfo());
139 
140     HFileArchiver.archiveRegion(UTIL.getConfiguration(), fs, region.getRegionInfo());
141 
142     // check for the existence of the archive directory and some files in it
143     Path archiveDir = HFileArchiveTestingUtil.getRegionArchiveDir(UTIL.getConfiguration(), region);
144     assertTrue(fs.exists(archiveDir));
145 
146     // check to make sure the store directory was copied
147     FileStatus[] stores = fs.listStatus(archiveDir);
148     assertTrue(stores.length == 1);
149 
150     // make sure we archived the store files
151     FileStatus[] storeFiles = fs.listStatus(stores[0].getPath());
152     assertTrue(storeFiles.length > 0);
153 
154     // then ensure the region's directory isn't present
155     assertFalse(fs.exists(regionDir));
156 
157     UTIL.deleteTable(TABLE_NAME);
158   }
159 
160   /**
161    * Test that the region directory is removed when we archive a region without store files, but
162    * still has hidden files.
163    * @throws Exception
164    */
165   @Test
166   public void testDeleteRegionWithNoStoreFiles() throws Exception {
167     TableName TABLE_NAME =
168         TableName.valueOf("testDeleteRegionWithNoStoreFiles");
169     UTIL.createTable(TABLE_NAME, TEST_FAM);
170 
171     // get the current store files for the region
172     List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
173     // make sure we only have 1 region serving this table
174     assertEquals(1, servingRegions.size());
175     HRegion region = servingRegions.get(0);
176 
177     FileSystem fs = region.getRegionFileSystem().getFileSystem();
178 
179     // make sure there are some files in the regiondir
180     Path rootDir = FSUtils.getRootDir(fs.getConf());
181     Path regionDir = HRegion.getRegionDir(rootDir, region.getRegionInfo());
182     FileStatus[] regionFiles = FSUtils.listStatus(fs, regionDir, null);
183     Assert.assertNotNull("No files in the region directory", regionFiles);
184     if (LOG.isDebugEnabled()) {
185       List<Path> files = new ArrayList<Path>();
186       for (FileStatus file : regionFiles) {
187         files.add(file.getPath());
188       }
189       LOG.debug("Current files:" + files);
190     }
191     // delete the visible folders so we just have hidden files/folders
192     final PathFilter dirFilter = new FSUtils.DirFilter(fs);
193     PathFilter nonHidden = new PathFilter() {
194       @Override
195       public boolean accept(Path file) {
196         return dirFilter.accept(file) && !file.getName().toString().startsWith(".");
197       }
198     };
199     FileStatus[] storeDirs = FSUtils.listStatus(fs, regionDir, nonHidden);
200     for (FileStatus store : storeDirs) {
201       LOG.debug("Deleting store for test");
202       fs.delete(store.getPath(), true);
203     }
204 
205     // then archive the region
206     HFileArchiver.archiveRegion(UTIL.getConfiguration(), fs, region.getRegionInfo());
207 
208     // and check to make sure the region directoy got deleted
209     assertFalse("Region directory (" + regionDir + "), still exists.", fs.exists(regionDir));
210 
211     UTIL.deleteTable(TABLE_NAME);
212   }
213 
214   @Test
215   public void testArchiveOnTableDelete() throws Exception {
216     TableName TABLE_NAME =
217         TableName.valueOf("testArchiveOnTableDelete");
218     UTIL.createTable(TABLE_NAME, TEST_FAM);
219 
220     List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
221     // make sure we only have 1 region serving this table
222     assertEquals(1, servingRegions.size());
223     HRegion region = servingRegions.get(0);
224 
225     // get the parent RS and monitor
226     HRegionServer hrs = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
227     FileSystem fs = hrs.getFileSystem();
228 
229     // put some data on the region
230     LOG.debug("-------Loading table");
231     UTIL.loadRegion(region, TEST_FAM);
232 
233     // get the hfiles in the region
234     List<HRegion> regions = hrs.getOnlineRegions(TABLE_NAME);
235     assertEquals("More that 1 region for test table.", 1, regions.size());
236 
237     region = regions.get(0);
238     // wait for all the compactions to complete
239     region.waitForFlushesAndCompactions();
240 
241     // disable table to prevent new updates
242     UTIL.getHBaseAdmin().disableTable(TABLE_NAME);
243     LOG.debug("Disabled table");
244 
245     // remove all the files from the archive to get a fair comparison
246     clearArchiveDirectory();
247 
248     // then get the current store files
249     List<String> storeFiles = getRegionStoreFiles(region);
250 
251     // then delete the table so the hfiles get archived
252     UTIL.deleteTable(TABLE_NAME);
253     LOG.debug("Deleted table");
254 
255     assertArchiveFiles(fs, storeFiles, 30000);
256   }
257 
258   private void assertArchiveFiles(FileSystem fs, List<String> storeFiles, long timeout) throws IOException {
259     long end = System.currentTimeMillis() + timeout;
260     Path archiveDir = HFileArchiveUtil.getArchivePath(UTIL.getConfiguration());
261     List<String> archivedFiles = new ArrayList<String>();
262 
263     // We have to ensure that the DeleteTableHandler is finished. HBaseAdmin.deleteXXX() can return before all files
264     // are archived. We should fix HBASE-5487 and fix synchronous operations from admin.
265     while (System.currentTimeMillis() < end) {
266       archivedFiles = getAllFileNames(fs, archiveDir);
267       if (archivedFiles.size() >= storeFiles.size()) {
268         break;
269       }
270     }
271 
272     Collections.sort(storeFiles);
273     Collections.sort(archivedFiles);
274 
275     LOG.debug("Store files:");
276     for (int i = 0; i < storeFiles.size(); i++) {
277       LOG.debug(i + " - " + storeFiles.get(i));
278     }
279     LOG.debug("Archive files:");
280     for (int i = 0; i < archivedFiles.size(); i++) {
281       LOG.debug(i + " - " + archivedFiles.get(i));
282     }
283 
284     assertTrue("Archived files are missing some of the store files!",
285       archivedFiles.containsAll(storeFiles));
286   }
287 
288 
289   /**
290    * Test that the store files are archived when a column family is removed.
291    * @throws Exception
292    */
293   @Test
294   public void testArchiveOnTableFamilyDelete() throws Exception {
295     TableName TABLE_NAME =
296         TableName.valueOf("testArchiveOnTableFamilyDelete");
297     UTIL.createTable(TABLE_NAME, TEST_FAM);
298 
299     List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
300     // make sure we only have 1 region serving this table
301     assertEquals(1, servingRegions.size());
302     HRegion region = servingRegions.get(0);
303 
304     // get the parent RS and monitor
305     HRegionServer hrs = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
306     FileSystem fs = hrs.getFileSystem();
307 
308     // put some data on the region
309     LOG.debug("-------Loading table");
310     UTIL.loadRegion(region, TEST_FAM);
311 
312     // get the hfiles in the region
313     List<HRegion> regions = hrs.getOnlineRegions(TABLE_NAME);
314     assertEquals("More that 1 region for test table.", 1, regions.size());
315 
316     region = regions.get(0);
317     // wait for all the compactions to complete
318     region.waitForFlushesAndCompactions();
319 
320     // disable table to prevent new updates
321     UTIL.getHBaseAdmin().disableTable(TABLE_NAME);
322     LOG.debug("Disabled table");
323 
324     // remove all the files from the archive to get a fair comparison
325     clearArchiveDirectory();
326 
327     // then get the current store files
328     List<String> storeFiles = getRegionStoreFiles(region);
329 
330     // then delete the table so the hfiles get archived
331     UTIL.getHBaseAdmin().deleteColumn(TABLE_NAME, TEST_FAM);
332 
333     assertArchiveFiles(fs, storeFiles, 30000);
334 
335     UTIL.deleteTable(TABLE_NAME);
336   }
337 
338   /**
339    * Test HFileArchiver.resolveAndArchive() race condition HBASE-7643
340    */
341   @Test
342   public void testCleaningRace() throws Exception {
343     final long TEST_TIME = 20 * 1000;
344 
345     Configuration conf = UTIL.getMiniHBaseCluster().getMaster().getConfiguration();
346     Path rootDir = UTIL.getDataTestDirOnTestFS("testCleaningRace");
347     FileSystem fs = UTIL.getTestFileSystem();
348 
349     Path archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
350     Path regionDir = new Path(FSUtils.getTableDir(new Path("./"),
351         TableName.valueOf("table")), "abcdef");
352     Path familyDir = new Path(regionDir, "cf");
353 
354     Path sourceRegionDir = new Path(rootDir, regionDir);
355     fs.mkdirs(sourceRegionDir);
356 
357     Stoppable stoppable = new StoppableImplementation();
358 
359     // The cleaner should be looping without long pauses to reproduce the race condition.
360     HFileCleaner cleaner = new HFileCleaner(1, stoppable, conf, fs, archiveDir);
361     try {
362       cleaner.start();
363 
364       // Keep creating/archiving new files while the cleaner is running in the other thread
365       long startTime = System.currentTimeMillis();
366       for (long fid = 0; (System.currentTimeMillis() - startTime) < TEST_TIME; ++fid) {
367         Path file = new Path(familyDir,  String.valueOf(fid));
368         Path sourceFile = new Path(rootDir, file);
369         Path archiveFile = new Path(archiveDir, file);
370 
371         fs.createNewFile(sourceFile);
372 
373         try {
374           // Try to archive the file
375           HFileArchiver.archiveRegion(fs, rootDir,
376               sourceRegionDir.getParent(), sourceRegionDir);
377 
378           // The archiver succeded, the file is no longer in the original location
379           // but it's in the archive location.
380           LOG.debug("hfile=" + fid + " should be in the archive");
381           assertTrue(fs.exists(archiveFile));
382           assertFalse(fs.exists(sourceFile));
383         } catch (IOException e) {
384           // The archiver is unable to archive the file. Probably HBASE-7643 race condition.
385           // in this case, the file should not be archived, and we should have the file
386           // in the original location.
387           LOG.debug("hfile=" + fid + " should be in the source location");
388           assertFalse(fs.exists(archiveFile));
389           assertTrue(fs.exists(sourceFile));
390 
391           // Avoid to have this file in the next run
392           fs.delete(sourceFile, false);
393         }
394       }
395     } finally {
396       stoppable.stop("test end");
397       cleaner.join();
398       fs.delete(rootDir, true);
399     }
400   }
401 
402   private void clearArchiveDirectory() throws IOException {
403     UTIL.getTestFileSystem().delete(
404       new Path(UTIL.getDefaultRootDirPath(), HConstants.HFILE_ARCHIVE_DIRECTORY), true);
405   }
406 
407   /**
408    * Get the names of all the files below the given directory
409    * @param fs
410    * @param archiveDir
411    * @return
412    * @throws IOException
413    */
414   private List<String> getAllFileNames(final FileSystem fs, Path archiveDir) throws IOException {
415     FileStatus[] files = FSUtils.listStatus(fs, archiveDir, null);
416     return recurseOnFiles(fs, files, new ArrayList<String>());
417   }
418 
419   /** Recursively lookup all the file names under the file[] array **/
420   private List<String> recurseOnFiles(FileSystem fs, FileStatus[] files, List<String> fileNames)
421       throws IOException {
422     if (files == null || files.length == 0) return fileNames;
423 
424     for (FileStatus file : files) {
425       if (file.isDir()) {
426         recurseOnFiles(fs, FSUtils.listStatus(fs, file.getPath(), null), fileNames);
427       } else fileNames.add(file.getPath().getName());
428     }
429     return fileNames;
430   }
431 
432   private List<String> getRegionStoreFiles(final HRegion region) throws IOException {
433     Path regionDir = region.getRegionFileSystem().getRegionDir();
434     FileSystem fs = region.getRegionFileSystem().getFileSystem();
435     List<String> storeFiles = getAllFileNames(fs, regionDir);
436     // remove all the non-storefile named files for the region
437     for (int i = 0; i < storeFiles.size(); i++) {
438       String file = storeFiles.get(i);
439       if (file.contains(HRegionFileSystem.REGION_INFO_FILE) || file.contains("hlog")) {
440         storeFiles.remove(i--);
441       }
442     }
443     storeFiles.remove(HRegionFileSystem.REGION_INFO_FILE);
444     return storeFiles;
445   }
446 }