1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.snapshot;
20
21 import java.io.IOException;
22 import java.io.FileNotFoundException;
23 import java.net.URI;
24 import java.text.SimpleDateFormat;
25 import java.util.ArrayList;
26 import java.util.Date;
27 import java.util.List;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31
32 import org.apache.hadoop.fs.Path;
33 import org.apache.hadoop.fs.FileStatus;
34 import org.apache.hadoop.fs.FileSystem;
35 import org.apache.hadoop.classification.InterfaceAudience;
36 import org.apache.hadoop.classification.InterfaceStability;
37 import org.apache.hadoop.conf.Configured;
38 import org.apache.hadoop.hbase.TableName;
39 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
40 import org.apache.hadoop.util.StringUtils;
41 import org.apache.hadoop.util.Tool;
42 import org.apache.hadoop.util.ToolRunner;
43
44 import org.apache.hadoop.conf.Configuration;
45 import org.apache.hadoop.hbase.HBaseConfiguration;
46 import org.apache.hadoop.hbase.HTableDescriptor;
47 import org.apache.hadoop.hbase.io.HFileLink;
48 import org.apache.hadoop.hbase.io.HLogLink;
49 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
50 import org.apache.hadoop.hbase.util.FSUtils;
51 import org.apache.hadoop.hbase.util.FSTableDescriptors;
52
53
54
55
56
57
58
59
60
61
62 @InterfaceAudience.Public
63 @InterfaceStability.Evolving
64 public final class SnapshotInfo extends Configured implements Tool {
65 private static final Log LOG = LogFactory.getLog(SnapshotInfo.class);
66
67
68
69
70
71
72
73
74
75 public static class SnapshotStats {
76
77 static class FileInfo {
78 private final boolean inArchive;
79 private final long size;
80
81 FileInfo(final boolean inArchive, final long size) {
82 this.inArchive = inArchive;
83 this.size = size;
84 }
85
86
87 public boolean inArchive() {
88 return this.inArchive;
89 }
90
91
92 public boolean isMissing() {
93 return this.size < 0;
94 }
95
96
97 public long getSize() {
98 return this.size;
99 }
100
101 String getStateToString() {
102 if (isMissing()) return "NOT FOUND";
103 if (inArchive()) return "archive";
104 return null;
105 }
106 }
107
108 private int hfileArchiveCount = 0;
109 private int hfilesMissing = 0;
110 private int hfilesCount = 0;
111 private int logsMissing = 0;
112 private int logsCount = 0;
113 private long hfileArchiveSize = 0;
114 private long hfileSize = 0;
115 private long logSize = 0;
116
117 private final SnapshotDescription snapshot;
118 private final TableName snapshotTable;
119 private final Configuration conf;
120 private final FileSystem fs;
121
122 SnapshotStats(final Configuration conf, final FileSystem fs, final SnapshotDescription snapshot)
123 {
124 this.snapshot = snapshot;
125 this.snapshotTable = TableName.valueOf(snapshot.getTable());
126 this.conf = conf;
127 this.fs = fs;
128 }
129
130
131 public SnapshotDescription getSnapshotDescription() {
132 return this.snapshot;
133 }
134
135
136 public boolean isSnapshotCorrupted() {
137 return hfilesMissing > 0 || logsMissing > 0;
138 }
139
140
141 public int getStoreFilesCount() {
142 return hfilesCount + hfileArchiveCount;
143 }
144
145
146 public int getArchivedStoreFilesCount() {
147 return hfileArchiveCount;
148 }
149
150
151 public int getLogsCount() {
152 return logsCount;
153 }
154
155
156 public int getMissingStoreFilesCount() {
157 return hfilesMissing;
158 }
159
160
161 public int getMissingLogsCount() {
162 return logsMissing;
163 }
164
165
166 public long getStoreFilesSize() {
167 return hfileSize + hfileArchiveSize;
168 }
169
170
171 public long getSharedStoreFilesSize() {
172 return hfileSize;
173 }
174
175
176 public long getArchivedStoreFileSize() {
177 return hfileArchiveSize;
178 }
179
180
181 public float getSharedStoreFilePercentage() {
182 return ((float)hfileSize / (hfileSize + hfileArchiveSize)) * 100;
183 }
184
185
186 public long getLogsSize() {
187 return logSize;
188 }
189
190
191
192
193
194
195
196
197 FileInfo addStoreFile(final String region, final String family, final String hfile)
198 throws IOException {
199 TableName table = snapshotTable;
200 HFileLink link = HFileLink.create(conf, table, region, family, hfile);
201 boolean inArchive = false;
202 long size = -1;
203 try {
204 if ((inArchive = fs.exists(link.getArchivePath()))) {
205 size = fs.getFileStatus(link.getArchivePath()).getLen();
206 hfileArchiveSize += size;
207 hfileArchiveCount++;
208 } else {
209 size = link.getFileStatus(fs).getLen();
210 hfileSize += size;
211 hfilesCount++;
212 }
213 } catch (FileNotFoundException e) {
214 hfilesMissing++;
215 }
216 return new FileInfo(inArchive, size);
217 }
218
219
220
221
222
223
224
225 FileInfo addRecoveredEdits(final String region, final String logfile) throws IOException {
226 Path rootDir = FSUtils.getRootDir(conf);
227 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
228 Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
229 long size = fs.getFileStatus(path).getLen();
230 logSize += size;
231 logsCount++;
232 return new FileInfo(true, size);
233 }
234
235
236
237
238
239
240
241 FileInfo addLogFile(final String server, final String logfile) throws IOException {
242 HLogLink logLink = new HLogLink(conf, server, logfile);
243 long size = -1;
244 try {
245 size = logLink.getFileStatus(fs).getLen();
246 logSize += size;
247 logsCount++;
248 } catch (FileNotFoundException e) {
249 logsMissing++;
250 }
251 return new FileInfo(false, size);
252 }
253 }
254
255 private boolean printSizeInBytes = false;
256 private FileSystem fs;
257 private Path rootDir;
258
259 private HTableDescriptor snapshotTableDesc;
260 private SnapshotDescription snapshotDesc;
261 private Path snapshotDir;
262
263 @Override
264 public int run(String[] args) throws IOException, InterruptedException {
265 final Configuration conf = getConf();
266 boolean listSnapshots = false;
267 String snapshotName = null;
268 boolean showSchema = false;
269 boolean showFiles = false;
270 boolean showStats = false;
271
272
273 for (int i = 0; i < args.length; i++) {
274 String cmd = args[i];
275 try {
276 if (cmd.equals("-snapshot")) {
277 snapshotName = args[++i];
278 } else if (cmd.equals("-files")) {
279 showFiles = true;
280 showStats = true;
281 } else if (cmd.equals("-stats")) {
282 showStats = true;
283 } else if (cmd.equals("-schema")) {
284 showSchema = true;
285 } else if (cmd.equals("-remote-dir")) {
286 Path sourceDir = new Path(args[++i]);
287 URI defaultFs = sourceDir.getFileSystem(conf).getUri();
288 FSUtils.setFsDefault(conf, new Path(defaultFs));
289 FSUtils.setRootDir(conf, sourceDir);
290 } else if (cmd.equals("-list-snapshots")) {
291 listSnapshots = true;
292 } else if (cmd.equals("-size-in-bytes")) {
293 printSizeInBytes = true;
294 } else if (cmd.equals("-h") || cmd.equals("--help")) {
295 printUsageAndExit();
296 } else {
297 System.err.println("UNEXPECTED: " + cmd);
298 printUsageAndExit();
299 }
300 } catch (Exception e) {
301 printUsageAndExit();
302 }
303 }
304
305
306 if (listSnapshots) {
307 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
308 System.out.printf("%-20s | %-20s | %s%n", "SNAPSHOT", "CREATION TIME", "TABLE NAME");
309 for (SnapshotDescription desc: getSnapshotList(conf)) {
310 System.out.printf("%-20s | %20s | %s%n",
311 desc.getName(),
312 df.format(new Date(desc.getCreationTime())),
313 desc.getTable());
314 }
315 return 0;
316 }
317
318 if (snapshotName == null) {
319 System.err.println("Missing snapshot name!");
320 printUsageAndExit();
321 return 1;
322 }
323
324 rootDir = FSUtils.getRootDir(conf);
325 fs = FileSystem.get(rootDir.toUri(), conf);
326 LOG.debug("fs=" + fs.getUri().toString() + " root=" + rootDir);
327
328
329 if (!loadSnapshotInfo(snapshotName)) {
330 System.err.println("Snapshot '" + snapshotName + "' not found!");
331 return 1;
332 }
333
334 printInfo();
335 if (showSchema) printSchema();
336 printFiles(showFiles, showStats);
337
338 return 0;
339 }
340
341
342
343
344
345
346 private boolean loadSnapshotInfo(final String snapshotName) throws IOException {
347 snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
348 if (!fs.exists(snapshotDir)) {
349 LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir);
350 return false;
351 }
352
353 snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
354 snapshotTableDesc = FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir);
355 return true;
356 }
357
358
359
360
361 private void printInfo() {
362 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
363 System.out.println("Snapshot Info");
364 System.out.println("----------------------------------------");
365 System.out.println(" Name: " + snapshotDesc.getName());
366 System.out.println(" Type: " + snapshotDesc.getType());
367 System.out.println(" Table: " + snapshotTableDesc.getTableName().getNameAsString());
368 System.out.println(" Format: " + snapshotDesc.getVersion());
369 System.out.println("Created: " + df.format(new Date(snapshotDesc.getCreationTime())));
370 System.out.println();
371 }
372
373
374
375
376 private void printSchema() {
377 System.out.println("Table Descriptor");
378 System.out.println("----------------------------------------");
379 System.out.println(snapshotTableDesc.toString());
380 System.out.println();
381 }
382
383
384
385
386
387 private void printFiles(final boolean showFiles, final boolean showStats) throws IOException {
388 if (showFiles) {
389 System.out.println("Snapshot Files");
390 System.out.println("----------------------------------------");
391 }
392
393
394 final String table = snapshotTableDesc.getTableName().getNameAsString();
395 final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, this.snapshotDesc);
396 SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
397 new SnapshotReferenceUtil.FileVisitor() {
398 public void storeFile (final String region, final String family, final String hfile)
399 throws IOException {
400 SnapshotStats.FileInfo info = stats.addStoreFile(region, family, hfile);
401
402 if (showFiles) {
403 String state = info.getStateToString();
404 System.out.printf("%8s %s/%s/%s/%s %s%n",
405 (info.isMissing() ? "-" : fileSizeToString(info.getSize())),
406 table, region, family, hfile,
407 state == null ? "" : "(" + state + ")");
408 }
409 }
410
411 public void recoveredEdits (final String region, final String logfile)
412 throws IOException {
413 SnapshotStats.FileInfo info = stats.addRecoveredEdits(region, logfile);
414
415 if (showFiles) {
416 System.out.printf("%8s recovered.edits %s on region %s%n",
417 fileSizeToString(info.getSize()), logfile, region);
418 }
419 }
420
421 public void logFile (final String server, final String logfile)
422 throws IOException {
423 SnapshotStats.FileInfo info = stats.addLogFile(server, logfile);
424
425 if (showFiles) {
426 String state = info.getStateToString();
427 System.out.printf("%8s log %s on server %s %s%n",
428 (info.isMissing() ? "-" : fileSizeToString(info.getSize())),
429 logfile, server,
430 state == null ? "" : "(" + state + ")");
431 }
432 }
433 });
434
435
436 System.out.println();
437 if (stats.isSnapshotCorrupted()) {
438 System.out.println("**************************************************************");
439 System.out.printf("BAD SNAPSHOT: %d hfile(s) and %d log(s) missing.%n",
440 stats.getMissingStoreFilesCount(), stats.getMissingLogsCount());
441 System.out.println("**************************************************************");
442 }
443
444 if (showStats) {
445 System.out.printf("%d HFiles (%d in archive), total size %s (%.2f%% %s shared with the source table)%n",
446 stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(),
447 fileSizeToString(stats.getStoreFilesSize()),
448 stats.getSharedStoreFilePercentage(),
449 fileSizeToString(stats.getSharedStoreFilesSize())
450 );
451 System.out.printf("%d Logs, total size %s%n",
452 stats.getLogsCount(), fileSizeToString(stats.getLogsSize()));
453 System.out.println();
454 }
455 }
456
457 private String fileSizeToString(long size) {
458 return printSizeInBytes ? Long.toString(size) : StringUtils.humanReadableInt(size);
459 }
460
461 private void printUsageAndExit() {
462 System.err.printf("Usage: bin/hbase %s [options]%n", getClass().getName());
463 System.err.println(" where [options] are:");
464 System.err.println(" -h|-help Show this help and exit.");
465 System.err.println(" -remote-dir Root directory that contains the snapshots.");
466 System.err.println(" -list-snapshots List all the available snapshots and exit.");
467 System.err.println(" -snapshot NAME Snapshot to examine.");
468 System.err.println(" -files Files and logs list.");
469 System.err.println(" -stats Files and logs stats.");
470 System.err.println(" -schema Describe the snapshotted table.");
471 System.err.println();
472 System.err.println("Examples:");
473 System.err.println(" hbase " + getClass() + " \\");
474 System.err.println(" -snapshot MySnapshot -files");
475 System.exit(1);
476 }
477
478
479
480
481
482
483
484 public static SnapshotStats getSnapshotStats(final Configuration conf,
485 final SnapshotDescription snapshot) throws IOException {
486 Path rootDir = FSUtils.getRootDir(conf);
487 FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
488 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
489 final SnapshotStats stats = new SnapshotStats(conf, fs, snapshot);
490 SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
491 new SnapshotReferenceUtil.FileVisitor() {
492 public void storeFile (final String region, final String family, final String hfile)
493 throws IOException {
494 stats.addStoreFile(region, family, hfile);
495 }
496
497 public void recoveredEdits (final String region, final String logfile) throws IOException {
498 stats.addRecoveredEdits(region, logfile);
499 }
500
501 public void logFile (final String server, final String logfile) throws IOException {
502 stats.addLogFile(server, logfile);
503 }
504 });
505 return stats;
506 }
507
508
509
510
511
512
513 public static List<SnapshotDescription> getSnapshotList(final Configuration conf)
514 throws IOException {
515 Path rootDir = FSUtils.getRootDir(conf);
516 FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
517 Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
518 FileStatus[] snapshots = fs.listStatus(snapshotDir,
519 new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
520 List<SnapshotDescription> snapshotLists =
521 new ArrayList<SnapshotDescription>(snapshots.length);
522 for (FileStatus snapshotDirStat: snapshots) {
523 snapshotLists.add(SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDirStat.getPath()));
524 }
525 return snapshotLists;
526 }
527
528
529
530
531
532
533
534
535 static int innerMain(final String [] args) throws Exception {
536 return ToolRunner.run(HBaseConfiguration.create(), new SnapshotInfo(), args);
537 }
538
539 public static void main(String[] args) throws Exception {
540 System.exit(innerMain(args));
541 }
542 }