1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master.snapshot;
19
20 import static org.junit.Assert.assertFalse;
21 import static org.junit.Assert.assertTrue;
22
23 import java.io.IOException;
24 import java.util.Collection;
25 import java.util.HashSet;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.fs.FileSystem;
30 import org.apache.hadoop.fs.Path;
31 import org.apache.hadoop.hbase.HBaseTestingUtility;
32 import org.apache.hadoop.hbase.MediumTests;
33 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
34 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
35 import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
36 import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
37 import org.apache.hadoop.hbase.util.FSUtils;
38 import org.junit.After;
39 import org.junit.AfterClass;
40 import org.junit.BeforeClass;
41 import org.junit.Test;
42 import org.junit.experimental.categories.Category;
43
44
45
46
47 @Category(MediumTests.class)
48 public class TestSnapshotFileCache {
49
50 private static final Log LOG = LogFactory.getLog(TestSnapshotFileCache.class);
51 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
52 private static FileSystem fs;
53 private static Path rootDir;
54
55 @BeforeClass
56 public static void startCluster() throws Exception {
57 UTIL.startMiniDFSCluster(1);
58 fs = UTIL.getDFSCluster().getFileSystem();
59 rootDir = UTIL.getDefaultRootDirPath();
60 }
61
62 @AfterClass
63 public static void stopCluster() throws Exception {
64 UTIL.shutdownMiniDFSCluster();
65 }
66
67 @After
68 public void cleanupFiles() throws Exception {
69
70 Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
71 fs.delete(snapshotDir, true);
72 }
73
74 @Test(timeout = 10000000)
75 public void testLoadAndDelete() throws Exception {
76
77 long period = Long.MAX_VALUE;
78 Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
79 SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
80 "test-snapshot-file-cache-refresh", new SnapshotFiles());
81
82 Path snapshot = new Path(snapshotDir, "snapshot");
83 Path region = new Path(snapshot, "7e91021");
84 Path family = new Path(region, "fam");
85 Path file1 = new Path(family, "file1");
86 Path file2 = new Path(family, "file2");
87
88
89 fs.createNewFile(file1);
90 fs.createNewFile(file2);
91
92 FSUtils.logFileSystemState(fs, rootDir, LOG);
93
94
95 assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName()));
96 assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName()));
97 String not = "file-shouldn't-be-found";
98 assertFalse("Cache found '" + not + "', but it shouldn't have.", cache.contains(not));
99
100
101
102 Thread.sleep(10);
103
104 LOG.debug("Deleting snapshot.");
105
106 if (!fs.delete(snapshot, true)) {
107 throw new IOException("Couldn't delete " + snapshot + " for an unknown reason.");
108 }
109 FSUtils.logFileSystemState(fs, rootDir, LOG);
110
111
112 LOG.debug("Checking to see if file is deleted.");
113 assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName()));
114 assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName()));
115
116
117 cache.triggerCacheRefreshForTesting();
118
119 assertFalse("Cache found '" + file1 + "', but it shouldn't have.",
120 cache.contains(file1.getName()));
121 assertFalse("Cache found '" + file2 + "', but it shouldn't have.",
122 cache.contains(file2.getName()));
123
124 fs.delete(snapshotDir, true);
125 }
126
127 @Test
128 public void testLoadsTmpDir() throws Exception {
129
130 long period = Long.MAX_VALUE;
131 Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
132 SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
133 "test-snapshot-file-cache-refresh", new SnapshotFiles());
134
135
136 Path snapshot = new Path(snapshotDir, "snapshot");
137 Path region = new Path(snapshot, "7e91021");
138 Path family = new Path(region, "fam");
139 Path file1 = new Path(family, "file1");
140 fs.createNewFile(file1);
141
142
143 SnapshotDescription desc = SnapshotDescription.newBuilder().setName("working").build();
144 snapshot = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
145 region = new Path(snapshot, "7e91021");
146 family = new Path(region, "fam");
147 Path file2 = new Path(family, "file2");
148 fs.createNewFile(file2);
149
150 FSUtils.logFileSystemState(fs, rootDir, LOG);
151
152
153 assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName()));
154 assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName()));
155 }
156
157 @Test
158 public void testJustFindLogsDirectory() throws Exception {
159
160 long period = Long.MAX_VALUE;
161 Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
162 SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
163 "test-snapshot-file-cache-refresh", new SnapshotFileCache.SnapshotFileInspector() {
164 public Collection<String> filesUnderSnapshot(final Path snapshotDir)
165 throws IOException {
166 return SnapshotReferenceUtil.getHLogNames(fs, snapshotDir);
167 }
168 });
169
170
171 Path snapshot = new Path(snapshotDir, "snapshot");
172 Path region = new Path(snapshot, "7e91021");
173 Path family = new Path(region, "fam");
174 Path file1 = new Path(family, "file1");
175 fs.createNewFile(file1);
176
177
178 Path logs = TakeSnapshotUtils.getSnapshotHLogsDir(snapshot, "server");
179 Path log = new Path(logs, "me.hbase.com%2C58939%2C1350424310315.1350424315552");
180 fs.createNewFile(log);
181
182 FSUtils.logFileSystemState(fs, rootDir, LOG);
183
184
185 assertFalse("Cache found '" + file1 + "', but it shouldn't have.",
186 cache.contains(file1.getName()));
187 assertTrue("Cache didn't find:" + log, cache.contains(log.getName()));
188 }
189
190 @Test
191 public void testReloadModifiedDirectory() throws IOException {
192
193 long period = Long.MAX_VALUE;
194 Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
195 SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
196 "test-snapshot-file-cache-refresh", new SnapshotFiles());
197
198 Path snapshot = new Path(snapshotDir, "snapshot");
199 Path region = new Path(snapshot, "7e91021");
200 Path family = new Path(region, "fam");
201 Path file1 = new Path(family, "file1");
202 Path file2 = new Path(family, "file2");
203
204
205 fs.createNewFile(file1);
206 fs.createNewFile(file2);
207
208 FSUtils.logFileSystemState(fs, rootDir, LOG);
209
210 assertTrue("Cache didn't find " + file1, cache.contains(file1.getName()));
211
212
213 fs.delete(snapshot, true);
214 Path file3 = new Path(family, "new_file");
215 fs.createNewFile(file3);
216
217 FSUtils.logFileSystemState(fs, rootDir, LOG);
218 assertTrue("Cache didn't find new file:" + file3, cache.contains(file3.getName()));
219 }
220
221 @Test
222 public void testSnapshotTempDirReload() throws IOException {
223 long period = Long.MAX_VALUE;
224
225 Path snapshotDir = new Path(SnapshotDescriptionUtils.getSnapshotsDir(rootDir),
226 SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME);
227 SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
228 "test-snapshot-file-cache-refresh", new SnapshotFiles());
229
230
231 Path snapshot1 = new Path(snapshotDir, "snapshot1");
232 Path file1 = new Path(new Path(new Path(snapshot1, "7e91021"), "fam"), "file1");
233 fs.createNewFile(file1);
234 assertTrue(cache.contains(file1.getName()));
235
236
237 Path snapshot2 = new Path(snapshotDir, "snapshot2");
238 Path file2 = new Path(new Path(new Path(snapshot2, "7e91021"), "fam2"), "file2");
239 fs.createNewFile(file2);
240 assertTrue(cache.contains(file2.getName()));
241 }
242
243 class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector {
244 public Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException {
245 Collection<String> files = new HashSet<String>();
246 files.addAll(SnapshotReferenceUtil.getHLogNames(fs, snapshotDir));
247 files.addAll(SnapshotReferenceUtil.getHFileNames(fs, snapshotDir));
248 return files;
249 }
250 };
251 }