1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.snapshot;
20
21 import java.io.IOException;
22 import java.io.FileNotFoundException;
23 import java.util.HashSet;
24 import java.util.LinkedList;
25 import java.util.List;
26 import java.util.Map;
27 import java.util.Set;
28 import java.util.TreeMap;
29
30 import org.apache.hadoop.classification.InterfaceAudience;
31 import org.apache.hadoop.conf.Configuration;
32 import org.apache.hadoop.fs.FileStatus;
33 import org.apache.hadoop.fs.FileSystem;
34 import org.apache.hadoop.fs.Path;
35 import org.apache.hadoop.hbase.TableName;
36 import org.apache.hadoop.hbase.io.HFileLink;
37 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
38 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
39 import org.apache.hadoop.hbase.util.FSUtils;
40 import org.apache.hadoop.hbase.util.FSVisitor;
41
42
43
44
45 @InterfaceAudience.Private
46 public final class SnapshotReferenceUtil {
47 public interface FileVisitor extends FSVisitor.StoreFileVisitor,
48 FSVisitor.RecoveredEditsVisitor, FSVisitor.LogFileVisitor {
49 }
50
51 private SnapshotReferenceUtil() {
52
53 }
54
55
56
57
58
59
60
61
62 public static Path getLogsDir(Path snapshotDir, String serverName) {
63 return new Path(snapshotDir, HLogUtil.getHLogDirectoryName(serverName));
64 }
65
66
67
68
69
70
71
72
73 public static Path getRecoveredEditsDir(Path snapshotDir, String regionName) {
74 return HLogUtil.getRegionDirRecoveredEditsDir(new Path(snapshotDir, regionName));
75 }
76
77
78
79
80
81
82
83
84
85 public static Path getRecoveredEdits(Path snapshotDir, String regionName, String logfile) {
86 return new Path(getRecoveredEditsDir(snapshotDir, regionName), logfile);
87 }
88
89
90
91
92
93
94
95
96
97 public static void visitReferencedFiles(final FileSystem fs, final Path snapshotDir,
98 final FileVisitor visitor) throws IOException {
99 visitTableStoreFiles(fs, snapshotDir, visitor);
100 visitRecoveredEdits(fs, snapshotDir, visitor);
101 visitLogFiles(fs, snapshotDir, visitor);
102 }
103
104
105
106
107
108
109
110
111
112 public static void visitTableStoreFiles(final FileSystem fs, final Path snapshotDir,
113 final FSVisitor.StoreFileVisitor visitor) throws IOException {
114 FSVisitor.visitTableStoreFiles(fs, snapshotDir, visitor);
115 }
116
117
118
119
120
121
122
123
124
125 public static void visitRegionStoreFiles(final FileSystem fs, final Path regionDir,
126 final FSVisitor.StoreFileVisitor visitor) throws IOException {
127 FSVisitor.visitRegionStoreFiles(fs, regionDir, visitor);
128 }
129
130
131
132
133
134
135
136
137
138 public static void visitRecoveredEdits(final FileSystem fs, final Path snapshotDir,
139 final FSVisitor.RecoveredEditsVisitor visitor) throws IOException {
140 FSVisitor.visitTableRecoveredEdits(fs, snapshotDir, visitor);
141 }
142
143
144
145
146
147
148
149
150
151 public static void visitLogFiles(final FileSystem fs, final Path snapshotDir,
152 final FSVisitor.LogFileVisitor visitor) throws IOException {
153 FSVisitor.visitLogFiles(fs, snapshotDir, visitor);
154 }
155
156
157
158
159
160
161
162
163
164
165
166 public static void verifySnapshot(final Configuration conf, final FileSystem fs,
167 final Path snapshotDir, final SnapshotDescription snapshotDesc) throws IOException {
168 final TableName table = TableName.valueOf(snapshotDesc.getTable());
169 visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
170 public void storeFile (final String region, final String family, final String hfile)
171 throws IOException {
172 HFileLink link = HFileLink.create(conf, table, region, family, hfile);
173 try {
174 link.getFileStatus(fs);
175 } catch (FileNotFoundException e) {
176 throw new CorruptedSnapshotException("Corrupted snapshot '" + snapshotDesc + "'", e);
177 }
178 }
179 });
180 }
181
182
183
184
185
186
187
188
189
190 public static Set<String> getSnapshotRegionNames(final FileSystem fs, final Path snapshotDir)
191 throws IOException {
192 FileStatus[] regionDirs = FSUtils.listStatus(fs, snapshotDir, new FSUtils.RegionDirFilter(fs));
193 if (regionDirs == null) return null;
194
195 Set<String> regions = new HashSet<String>();
196 for (FileStatus regionDir: regionDirs) {
197 regions.add(regionDir.getPath().getName());
198 }
199 return regions;
200 }
201
202
203
204
205
206
207
208
209
210
211
212
213 public static Map<String, List<String>> getRegionHFileReferences(final FileSystem fs,
214 final Path snapshotRegionDir) throws IOException {
215 final Map<String, List<String>> familyFiles = new TreeMap<String, List<String>>();
216
217 visitRegionStoreFiles(fs, snapshotRegionDir,
218 new FSVisitor.StoreFileVisitor() {
219 public void storeFile (final String region, final String family, final String hfile)
220 throws IOException {
221 List<String> hfiles = familyFiles.get(family);
222 if (hfiles == null) {
223 hfiles = new LinkedList<String>();
224 familyFiles.put(family, hfiles);
225 }
226 hfiles.add(hfile);
227 }
228 });
229
230 return familyFiles;
231 }
232
233
234
235
236
237
238
239
240
241 public static Set<String> getHFileNames(final FileSystem fs, final Path snapshotDir)
242 throws IOException {
243 final Set<String> names = new HashSet<String>();
244 visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
245 public void storeFile (final String region, final String family, final String hfile)
246 throws IOException {
247 if (HFileLink.isHFileLink(hfile)) {
248 names.add(HFileLink.getReferencedHFileName(hfile));
249 } else {
250 names.add(hfile);
251 }
252 }
253 });
254 return names;
255 }
256
257
258
259
260
261
262
263
264
265 public static Set<String> getHLogNames(final FileSystem fs, final Path snapshotDir)
266 throws IOException {
267 final Set<String> names = new HashSet<String>();
268 visitLogFiles(fs, snapshotDir, new FSVisitor.LogFileVisitor() {
269 public void logFile (final String server, final String logfile) throws IOException {
270 names.add(logfile);
271 }
272 });
273 return names;
274 }
275 }