View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master.snapshot;
19  
20  import static org.junit.Assert.assertFalse;
21  import static org.junit.Assert.assertTrue;
22  
23  import java.io.IOException;
24  import java.util.Collection;
25  import java.util.HashSet;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.fs.FileSystem;
30  import org.apache.hadoop.fs.Path;
31  import org.apache.hadoop.hbase.HBaseTestingUtility;
32  import org.apache.hadoop.hbase.MediumTests;
33  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
34  import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
35  import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
36  import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
37  import org.apache.hadoop.hbase.util.FSUtils;
38  import org.junit.After;
39  import org.junit.AfterClass;
40  import org.junit.BeforeClass;
41  import org.junit.Test;
42  import org.junit.experimental.categories.Category;
43  
44  /**
45   * Test that we correctly reload the cache, filter directories, etc.
46   */
47  @Category(MediumTests.class)
48  public class TestSnapshotFileCache {
49  
50    private static final Log LOG = LogFactory.getLog(TestSnapshotFileCache.class);
51    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
52    private static FileSystem fs;
53    private static Path rootDir;
54  
55    @BeforeClass
56    public static void startCluster() throws Exception {
57      UTIL.startMiniDFSCluster(1);
58      fs = UTIL.getDFSCluster().getFileSystem();
59      rootDir = UTIL.getDefaultRootDirPath();
60    }
61  
62    @AfterClass
63    public static void stopCluster() throws Exception {
64      UTIL.shutdownMiniDFSCluster();
65    }
66  
67    @After
68    public void cleanupFiles() throws Exception {
69      // cleanup the snapshot directory
70      Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
71      fs.delete(snapshotDir, true);
72    }
73  
74    @Test(timeout = 10000000)
75    public void testLoadAndDelete() throws Exception {
76      // don't refresh the cache unless we tell it to
77      long period = Long.MAX_VALUE;
78      Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
79      SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
80          "test-snapshot-file-cache-refresh", new SnapshotFiles());
81  
82      Path snapshot = new Path(snapshotDir, "snapshot");
83      Path region = new Path(snapshot, "7e91021");
84      Path family = new Path(region, "fam");
85      Path file1 = new Path(family, "file1");
86      Path file2 = new Path(family, "file2");
87  
88      // create two hfiles under the snapshot
89      fs.createNewFile(file1);
90      fs.createNewFile(file2);
91  
92      FSUtils.logFileSystemState(fs, rootDir, LOG);
93  
94      // then make sure the cache finds them
95      assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName()));
96      assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName()));
97      String not = "file-shouldn't-be-found";
98      assertFalse("Cache found '" + not + "', but it shouldn't have.", cache.contains(not));
99  
100     // make sure we get a little bit of separation in the modification times
101     // its okay if we sleep a little longer (b/c of GC pause), as long as we sleep a little
102     Thread.sleep(10);
103 
104     LOG.debug("Deleting snapshot.");
105     // then delete the snapshot and make sure that we can still find the files
106     if (!fs.delete(snapshot, true)) {
107       throw new IOException("Couldn't delete " + snapshot + " for an unknown reason.");
108     }
109     FSUtils.logFileSystemState(fs, rootDir, LOG);
110 
111 
112     LOG.debug("Checking to see if file is deleted.");
113     assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName()));
114     assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName()));
115 
116     // then trigger a refresh
117     cache.triggerCacheRefreshForTesting();
118     // and not it shouldn't find those files
119     assertFalse("Cache found '" + file1 + "', but it shouldn't have.",
120       cache.contains(file1.getName()));
121     assertFalse("Cache found '" + file2 + "', but it shouldn't have.",
122       cache.contains(file2.getName()));
123 
124     fs.delete(snapshotDir, true);
125   }
126 
127   @Test
128   public void testLoadsTmpDir() throws Exception {
129     // don't refresh the cache unless we tell it to
130     long period = Long.MAX_VALUE;
131     Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
132     SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
133         "test-snapshot-file-cache-refresh", new SnapshotFiles());
134 
135     // create a file in a 'completed' snapshot
136     Path snapshot = new Path(snapshotDir, "snapshot");
137     Path region = new Path(snapshot, "7e91021");
138     Path family = new Path(region, "fam");
139     Path file1 = new Path(family, "file1");
140     fs.createNewFile(file1);
141 
142     // create an 'in progress' snapshot
143     SnapshotDescription desc = SnapshotDescription.newBuilder().setName("working").build();
144     snapshot = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
145     region = new Path(snapshot, "7e91021");
146     family = new Path(region, "fam");
147     Path file2 = new Path(family, "file2");
148     fs.createNewFile(file2);
149 
150     FSUtils.logFileSystemState(fs, rootDir, LOG);
151 
152     // then make sure the cache finds both files
153     assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName()));
154     assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName()));
155   }
156 
157   @Test
158   public void testJustFindLogsDirectory() throws Exception {
159     // don't refresh the cache unless we tell it to
160     long period = Long.MAX_VALUE;
161     Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
162     SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
163         "test-snapshot-file-cache-refresh", new SnapshotFileCache.SnapshotFileInspector() {
164             public Collection<String> filesUnderSnapshot(final Path snapshotDir)
165                 throws IOException {
166               return SnapshotReferenceUtil.getHLogNames(fs, snapshotDir);
167             }
168         });
169 
170     // create a file in a 'completed' snapshot
171     Path snapshot = new Path(snapshotDir, "snapshot");
172     Path region = new Path(snapshot, "7e91021");
173     Path family = new Path(region, "fam");
174     Path file1 = new Path(family, "file1");
175     fs.createNewFile(file1);
176 
177     // and another file in the logs directory
178     Path logs = TakeSnapshotUtils.getSnapshotHLogsDir(snapshot, "server");
179     Path log = new Path(logs, "me.hbase.com%2C58939%2C1350424310315.1350424315552");
180     fs.createNewFile(log);
181 
182     FSUtils.logFileSystemState(fs, rootDir, LOG);
183 
184     // then make sure the cache only finds the log files
185     assertFalse("Cache found '" + file1 + "', but it shouldn't have.",
186       cache.contains(file1.getName()));
187     assertTrue("Cache didn't find:" + log, cache.contains(log.getName()));
188   }
189 
190   @Test
191   public void testReloadModifiedDirectory() throws IOException {
192     // don't refresh the cache unless we tell it to
193     long period = Long.MAX_VALUE;
194     Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
195     SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
196         "test-snapshot-file-cache-refresh", new SnapshotFiles());
197 
198     Path snapshot = new Path(snapshotDir, "snapshot");
199     Path region = new Path(snapshot, "7e91021");
200     Path family = new Path(region, "fam");
201     Path file1 = new Path(family, "file1");
202     Path file2 = new Path(family, "file2");
203 
204     // create two hfiles under the snapshot
205     fs.createNewFile(file1);
206     fs.createNewFile(file2);
207 
208     FSUtils.logFileSystemState(fs, rootDir, LOG);
209 
210     assertTrue("Cache didn't find " + file1, cache.contains(file1.getName()));
211 
212     // now delete the snapshot and add a file with a different name
213     fs.delete(snapshot, true);
214     Path file3 = new Path(family, "new_file");
215     fs.createNewFile(file3);
216 
217     FSUtils.logFileSystemState(fs, rootDir, LOG);
218     assertTrue("Cache didn't find new file:" + file3, cache.contains(file3.getName()));
219   }
220 
221   @Test
222   public void testSnapshotTempDirReload() throws IOException {
223     long period = Long.MAX_VALUE;
224     // This doesn't refresh cache until we invoke it explicitly
225     Path snapshotDir = new Path(SnapshotDescriptionUtils.getSnapshotsDir(rootDir),
226         SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME);
227     SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
228         "test-snapshot-file-cache-refresh", new SnapshotFiles());
229 
230     // Add a new snapshot
231     Path snapshot1 = new Path(snapshotDir, "snapshot1");
232     Path file1 = new Path(new Path(new Path(snapshot1, "7e91021"), "fam"), "file1");
233     fs.createNewFile(file1);
234     assertTrue(cache.contains(file1.getName()));
235 
236     // Add another snapshot
237     Path snapshot2 = new Path(snapshotDir, "snapshot2");
238     Path file2 = new Path(new Path(new Path(snapshot2, "7e91021"), "fam2"), "file2");
239     fs.createNewFile(file2);
240     assertTrue(cache.contains(file2.getName()));
241   }
242 
243   class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector {
244     public Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException {
245       Collection<String> files =  new HashSet<String>();
246       files.addAll(SnapshotReferenceUtil.getHLogNames(fs, snapshotDir));
247       files.addAll(SnapshotReferenceUtil.getHFileNames(fs, snapshotDir));
248       return files;
249     }
250   };
251 }