1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.IOException;
23 import java.util.regex.Matcher;
24 import java.util.regex.Pattern;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.classification.InterfaceAudience;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.fs.FileStatus;
31 import org.apache.hadoop.fs.FileSystem;
32 import org.apache.hadoop.fs.Path;
33 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
34 import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
35 import org.apache.hadoop.hbase.io.HFileLink;
36 import org.apache.hadoop.hbase.io.HalfStoreFileReader;
37 import org.apache.hadoop.hbase.io.Reference;
38 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
39 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
40 import org.apache.hadoop.hbase.util.FSUtils;
41
42
43
44
45 @InterfaceAudience.Private
46 public class StoreFileInfo {
47 public static final Log LOG = LogFactory.getLog(StoreFileInfo.class);
48
49
50
51
52
53 public static final String HFILE_NAME_REGEX = "[0-9a-f]+(?:_SeqId_[0-9]+_)?";
54
55
56 private static final Pattern HFILE_NAME_PATTERN =
57 Pattern.compile("^(" + HFILE_NAME_REGEX + ")");
58
59
60
61
62
63
64
65
66 private static final Pattern REF_NAME_PATTERN =
67 Pattern.compile(String.format("^(%s|%s)\\.(.+)$",
68 HFILE_NAME_REGEX, HFileLink.LINK_NAME_REGEX));
69
70
71 private Configuration conf;
72
73
74 private HDFSBlocksDistribution hdfsBlocksDistribution = null;
75
76
77 private final Reference reference;
78
79
80 private final HFileLink link;
81
82
83 private final FileStatus fileStatus;
84
85 private RegionCoprocessorHost coprocessorHost;
86
87
88
89
90
91
92
93 public StoreFileInfo(final Configuration conf, final FileSystem fs, final Path path)
94 throws IOException {
95 this(conf, fs, fs.getFileStatus(path));
96 }
97
98
99
100
101
102
103
104 public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus)
105 throws IOException {
106 this.conf = conf;
107 this.fileStatus = fileStatus;
108 Path p = fileStatus.getPath();
109 if (HFileLink.isHFileLink(p)) {
110
111 this.reference = null;
112 this.link = new HFileLink(conf, p);
113 if (LOG.isTraceEnabled()) LOG.trace(p + " is a link");
114 } else if (isReference(p)) {
115 this.reference = Reference.read(fs, p);
116 Path referencePath = getReferredToFile(p);
117 if (HFileLink.isHFileLink(referencePath)) {
118
119 this.link = new HFileLink(conf, referencePath);
120 } else {
121
122 this.link = null;
123 }
124 if (LOG.isTraceEnabled()) LOG.trace(p + " is a " + reference.getFileRegion() +
125 " reference to " + referencePath);
126 } else if (isHFile(p)) {
127
128 this.reference = null;
129 this.link = null;
130 } else {
131 throw new IOException("path=" + p + " doesn't look like a valid StoreFile");
132 }
133 }
134
135
136
137
138
139 public void setRegionCoprocessorHost(RegionCoprocessorHost coprocessorHost) {
140 this.coprocessorHost = coprocessorHost;
141 }
142
143
144
145
146
147 Reference getReference() {
148 return this.reference;
149 }
150
151
152 public boolean isReference() {
153 return this.reference != null;
154 }
155
156
157 public boolean isTopReference() {
158 return this.reference != null && Reference.isTopFileRegion(this.reference.getFileRegion());
159 }
160
161
162 public boolean isLink() {
163 return this.link != null && this.reference == null;
164 }
165
166
167 public HDFSBlocksDistribution getHDFSBlockDistribution() {
168 return this.hdfsBlocksDistribution;
169 }
170
171
172
173
174
175
176
177 public StoreFile.Reader open(final FileSystem fs,
178 final CacheConfig cacheConf) throws IOException {
179 FSDataInputStreamWrapper in;
180 FileStatus status;
181
182 if (this.link != null) {
183
184 in = new FSDataInputStreamWrapper(fs, this.link);
185 status = this.link.getFileStatus(fs);
186 } else if (this.reference != null) {
187
188 Path referencePath = getReferredToFile(this.getPath());
189 in = new FSDataInputStreamWrapper(fs, referencePath);
190 status = fs.getFileStatus(referencePath);
191 } else {
192 in = new FSDataInputStreamWrapper(fs, this.getPath());
193 status = fileStatus;
194 }
195 long length = status.getLen();
196 if (this.reference != null) {
197 hdfsBlocksDistribution = computeRefFileHDFSBlockDistribution(fs, reference, status);
198 } else {
199 hdfsBlocksDistribution = FSUtils.computeHDFSBlocksDistribution(fs, status, 0, length);
200 }
201 StoreFile.Reader reader = null;
202 if (this.coprocessorHost != null) {
203 reader = this.coprocessorHost.preStoreFileReaderOpen(fs, this.getPath(), in, length,
204 cacheConf, reference);
205 }
206 if (reader == null) {
207 if (this.reference != null) {
208 reader = new HalfStoreFileReader(fs, this.getPath(), in, length, cacheConf, reference,
209 conf);
210 } else {
211 reader = new StoreFile.Reader(fs, this.getPath(), in, length, cacheConf, conf);
212 }
213 }
214 if (this.coprocessorHost != null) {
215 reader = this.coprocessorHost.postStoreFileReaderOpen(fs, this.getPath(), in, length,
216 cacheConf, reference, reader);
217 }
218 return reader;
219 }
220
221
222
223
224 public HDFSBlocksDistribution computeHDFSBlocksDistribution(final FileSystem fs)
225 throws IOException {
226 FileStatus status;
227 if (this.reference != null) {
228 if (this.link != null) {
229
230 status = link.getFileStatus(fs);
231 } else {
232
233 Path referencePath = getReferredToFile(this.getPath());
234 status = fs.getFileStatus(referencePath);
235 }
236 return computeRefFileHDFSBlockDistribution(fs, reference, status);
237 } else {
238 if (this.link != null) {
239
240 status = link.getFileStatus(fs);
241 } else {
242 status = this.fileStatus;
243 }
244 return FSUtils.computeHDFSBlocksDistribution(fs, status, 0, status.getLen());
245 }
246 }
247
248
249 public Path getPath() {
250 return this.fileStatus.getPath();
251 }
252
253
254 public FileStatus getFileStatus() {
255 return this.fileStatus;
256 }
257
258
259 public long getModificationTime() {
260 return this.fileStatus.getModificationTime();
261 }
262
263 @Override
264 public String toString() {
265 return this.getPath() +
266 (isReference() ? "-" + getReferredToFile(this.getPath()) + "-" + reference : "");
267 }
268
269
270
271
272
273 public static boolean isHFile(final Path path) {
274 return isHFile(path.getName());
275 }
276
277 public static boolean isHFile(final String fileName) {
278 Matcher m = HFILE_NAME_PATTERN.matcher(fileName);
279 return m.matches() && m.groupCount() > 0;
280 }
281
282
283
284
285
286 public static boolean isReference(final Path path) {
287 return isReference(path.getName());
288 }
289
290
291
292
293
294 public static boolean isReference(final String name) {
295 Matcher m = REF_NAME_PATTERN.matcher(name);
296 return m.matches() && m.groupCount() > 1;
297 }
298
299
300
301
302
303
304
305
306 public static Path getReferredToFile(final Path p) {
307 Matcher m = REF_NAME_PATTERN.matcher(p.getName());
308 if (m == null || !m.matches()) {
309 LOG.warn("Failed match of store file name " + p.toString());
310 throw new IllegalArgumentException("Failed match of store file name " +
311 p.toString());
312 }
313
314
315 String otherRegion = m.group(2);
316
317 Path tableDir = p.getParent().getParent().getParent();
318 String nameStrippedOfSuffix = m.group(1);
319 LOG.debug("reference '" + p + "' to region=" + otherRegion + " hfile=" + nameStrippedOfSuffix);
320
321
322
323 return new Path(new Path(new Path(tableDir, otherRegion),
324 p.getParent().getName()), nameStrippedOfSuffix);
325 }
326
327
328
329
330
331
332 public static boolean validateStoreFileName(final String fileName) {
333 if (HFileLink.isHFileLink(fileName) || isReference(fileName))
334 return(true);
335 return !fileName.contains("-");
336 }
337
338
339
340
341
342
343 public static boolean isValid(final FileStatus fileStatus)
344 throws IOException {
345 final Path p = fileStatus.getPath();
346
347 if (fileStatus.isDir())
348 return false;
349
350
351
352
353 if (!HFileLink.isHFileLink(p) && fileStatus.getLen() <= 0) {
354 LOG.warn("Skipping " + p + " because it is empty. HBASE-646 DATA LOSS?");
355 return false;
356 }
357
358 return validateStoreFileName(p.getName());
359 }
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374 private static HDFSBlocksDistribution computeRefFileHDFSBlockDistribution(
375 final FileSystem fs, final Reference reference, final FileStatus status)
376 throws IOException {
377 if (status == null) {
378 return null;
379 }
380
381 long start = 0;
382 long length = 0;
383
384 if (Reference.isTopFileRegion(reference.getFileRegion())) {
385 start = status.getLen()/2;
386 length = status.getLen() - status.getLen()/2;
387 } else {
388 start = 0;
389 length = status.getLen()/2;
390 }
391 return FSUtils.computeHDFSBlocksDistribution(fs, status, start, length);
392 }
393 }