View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.snapshot;
20  
21  import java.io.IOException;
22  import java.io.FileNotFoundException;
23  import java.net.URI;
24  import java.text.SimpleDateFormat;
25  import java.util.ArrayList;
26  import java.util.Date;
27  import java.util.List;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  
32  import org.apache.hadoop.fs.Path;
33  import org.apache.hadoop.fs.FileStatus;
34  import org.apache.hadoop.fs.FileSystem;
35  import org.apache.hadoop.classification.InterfaceAudience;
36  import org.apache.hadoop.classification.InterfaceStability;
37  import org.apache.hadoop.conf.Configured;
38  import org.apache.hadoop.hbase.TableName;
39  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
40  import org.apache.hadoop.util.StringUtils;
41  import org.apache.hadoop.util.Tool;
42  import org.apache.hadoop.util.ToolRunner;
43  
44  import org.apache.hadoop.conf.Configuration;
45  import org.apache.hadoop.hbase.HBaseConfiguration;
46  import org.apache.hadoop.hbase.HTableDescriptor;
47  import org.apache.hadoop.hbase.io.HFileLink;
48  import org.apache.hadoop.hbase.io.HLogLink;
49  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
50  import org.apache.hadoop.hbase.util.FSUtils;
51  import org.apache.hadoop.hbase.util.FSTableDescriptors;
52  
53  /**
54   * Tool for dumping snapshot information.
55   * <ol>
56   * <li> Table Descriptor
57   * <li> Snapshot creation time, type, format version, ...
58   * <li> List of hfiles and hlogs
59   * <li> Stats about hfiles and logs sizes, percentage of shared with the source table, ...
60   * </ol>
61   */
62  @InterfaceAudience.Public
63  @InterfaceStability.Evolving
64  public final class SnapshotInfo extends Configured implements Tool {
65    private static final Log LOG = LogFactory.getLog(SnapshotInfo.class);
66  
67    /**
68     * Statistics about the snapshot
69     * <ol>
70     * <li> How many store files and logs are in the archive
71     * <li> How many store files and logs are shared with the table
72     * <li> Total store files and logs size and shared amount
73     * </ol>
74     */
75    public static class SnapshotStats {
76      /** Information about the file referenced by the snapshot */
77      static class FileInfo {
78        private final boolean inArchive;
79        private final long size;
80  
81        FileInfo(final boolean inArchive, final long size) {
82          this.inArchive = inArchive;
83          this.size = size;
84        }
85  
86        /** @return true if the file is in the archive */
87        public boolean inArchive() {
88          return this.inArchive;
89        }
90  
91        /** @return true if the file is missing */
92        public boolean isMissing() {
93          return this.size < 0;
94        }
95  
96        /** @return the file size */
97        public long getSize() {
98          return this.size;
99        }
100 
101       String getStateToString() {
102         if (isMissing()) return "NOT FOUND";
103         if (inArchive()) return "archive";
104         return null;
105       }
106     }
107 
108     private int hfileArchiveCount = 0;
109     private int hfilesMissing = 0;
110     private int hfilesCount = 0;
111     private int logsMissing = 0;
112     private int logsCount = 0;
113     private long hfileArchiveSize = 0;
114     private long hfileSize = 0;
115     private long logSize = 0;
116 
117     private final SnapshotDescription snapshot;
118     private final TableName snapshotTable;
119     private final Configuration conf;
120     private final FileSystem fs;
121 
122     SnapshotStats(final Configuration conf, final FileSystem fs, final SnapshotDescription snapshot)
123     {
124       this.snapshot = snapshot;
125       this.snapshotTable = TableName.valueOf(snapshot.getTable());
126       this.conf = conf;
127       this.fs = fs;
128     }
129 
130     /** @return the snapshot descriptor */
131     public SnapshotDescription getSnapshotDescription() {
132       return this.snapshot;
133     }
134 
135     /** @return true if the snapshot is corrupted */
136     public boolean isSnapshotCorrupted() {
137       return hfilesMissing > 0 || logsMissing > 0;
138     }
139 
140     /** @return the number of available store files */
141     public int getStoreFilesCount() {
142       return hfilesCount + hfileArchiveCount;
143     }
144 
145     /** @return the number of available store files in the archive */
146     public int getArchivedStoreFilesCount() {
147       return hfileArchiveCount;
148     }
149 
150     /** @return the number of available log files */
151     public int getLogsCount() {
152       return logsCount;
153     }
154 
155     /** @return the number of missing store files */
156     public int getMissingStoreFilesCount() {
157       return hfilesMissing;
158     }
159 
160     /** @return the number of missing log files */
161     public int getMissingLogsCount() {
162       return logsMissing;
163     }
164 
165     /** @return the total size of the store files referenced by the snapshot */
166     public long getStoreFilesSize() {
167       return hfileSize + hfileArchiveSize;
168     }
169 
170     /** @return the total size of the store files shared */
171     public long getSharedStoreFilesSize() {
172       return hfileSize;
173     }
174 
175     /** @return the total size of the store files in the archive */
176     public long getArchivedStoreFileSize() {
177       return hfileArchiveSize;
178     }
179 
180     /** @return the percentage of the shared store files */
181     public float getSharedStoreFilePercentage() {
182       return ((float)hfileSize / (hfileSize + hfileArchiveSize)) * 100;
183     }
184 
185     /** @return the total log size */
186     public long getLogsSize() {
187       return logSize;
188     }
189 
190     /**
191      * Add the specified store file to the stats
192      * @param region region encoded Name
193      * @param family family name
194      * @param hfile store file name
195      * @return the store file information
196      */
197     FileInfo addStoreFile(final String region, final String family, final String hfile)
198           throws IOException {
199       TableName table = snapshotTable;
200       HFileLink link = HFileLink.create(conf, table, region, family, hfile);
201       boolean inArchive = false;
202       long size = -1;
203       try {
204         if ((inArchive = fs.exists(link.getArchivePath()))) {
205           size = fs.getFileStatus(link.getArchivePath()).getLen();
206           hfileArchiveSize += size;
207           hfileArchiveCount++;
208         } else {
209           size = link.getFileStatus(fs).getLen();
210           hfileSize += size;
211           hfilesCount++;
212         }
213       } catch (FileNotFoundException e) {
214         hfilesMissing++;
215       }
216       return new FileInfo(inArchive, size);
217     }
218 
219     /**
220      * Add the specified recovered.edits file to the stats
221      * @param region region encoded name
222      * @param logfile log file name
223      * @return the recovered.edits information
224      */
225     FileInfo addRecoveredEdits(final String region, final String logfile) throws IOException {
226       Path rootDir = FSUtils.getRootDir(conf);
227       Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
228       Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
229       long size = fs.getFileStatus(path).getLen();
230       logSize += size;
231       logsCount++;
232       return new FileInfo(true, size);
233     }
234 
235     /**
236      * Add the specified log file to the stats
237      * @param server server name
238      * @param logfile log file name
239      * @return the log information
240      */
241     FileInfo addLogFile(final String server, final String logfile) throws IOException {
242       HLogLink logLink = new HLogLink(conf, server, logfile);
243       long size = -1;
244       try {
245         size = logLink.getFileStatus(fs).getLen();
246         logSize += size;
247         logsCount++;
248       } catch (FileNotFoundException e) {
249         logsMissing++;
250       }
251       return new FileInfo(false, size);
252     }
253   }
254 
255   private boolean printSizeInBytes = false;
256   private FileSystem fs;
257   private Path rootDir;
258 
259   private HTableDescriptor snapshotTableDesc;
260   private SnapshotDescription snapshotDesc;
261   private Path snapshotDir;
262 
263   @Override
264   public int run(String[] args) throws IOException, InterruptedException {
265     final Configuration conf = getConf();
266     boolean listSnapshots = false;
267     String snapshotName = null;
268     boolean showSchema = false;
269     boolean showFiles = false;
270     boolean showStats = false;
271 
272     // Process command line args
273     for (int i = 0; i < args.length; i++) {
274       String cmd = args[i];
275       try {
276         if (cmd.equals("-snapshot")) {
277           snapshotName = args[++i];
278         } else if (cmd.equals("-files")) {
279           showFiles = true;
280           showStats = true;
281         } else if (cmd.equals("-stats")) {
282           showStats = true;
283         } else if (cmd.equals("-schema")) {
284           showSchema = true;
285         } else if (cmd.equals("-remote-dir")) {
286           Path sourceDir = new Path(args[++i]);
287           URI defaultFs = sourceDir.getFileSystem(conf).getUri();
288           FSUtils.setFsDefault(conf, new Path(defaultFs));
289           FSUtils.setRootDir(conf, sourceDir);
290         } else if (cmd.equals("-list-snapshots")) {
291           listSnapshots = true;
292         } else if (cmd.equals("-size-in-bytes")) {
293           printSizeInBytes = true;
294         } else if (cmd.equals("-h") || cmd.equals("--help")) {
295           printUsageAndExit();
296         } else {
297           System.err.println("UNEXPECTED: " + cmd);
298           printUsageAndExit();
299         }
300       } catch (Exception e) {
301         printUsageAndExit();
302       }
303     }
304 
305     // List Available Snapshots
306     if (listSnapshots) {
307       SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
308       System.out.printf("%-20s | %-20s | %s%n", "SNAPSHOT", "CREATION TIME", "TABLE NAME");
309       for (SnapshotDescription desc: getSnapshotList(conf)) {
310         System.out.printf("%-20s | %20s | %s%n",
311                           desc.getName(),
312                           df.format(new Date(desc.getCreationTime())),
313                           desc.getTable());
314       }
315       return 0;
316     }
317 
318     if (snapshotName == null) {
319       System.err.println("Missing snapshot name!");
320       printUsageAndExit();
321       return 1;
322     }
323 
324     rootDir = FSUtils.getRootDir(conf);
325     fs = FileSystem.get(rootDir.toUri(), conf);
326     LOG.debug("fs=" + fs.getUri().toString() + " root=" + rootDir);
327 
328     // Load snapshot information
329     if (!loadSnapshotInfo(snapshotName)) {
330       System.err.println("Snapshot '" + snapshotName + "' not found!");
331       return 1;
332     }
333 
334     printInfo();
335     if (showSchema) printSchema();
336     printFiles(showFiles, showStats);
337 
338     return 0;
339   }
340 
341   /**
342    * Load snapshot info and table descriptor for the specified snapshot
343    * @param snapshotName name of the snapshot to load
344    * @return false if snapshot is not found
345    */
346   private boolean loadSnapshotInfo(final String snapshotName) throws IOException {
347     snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
348     if (!fs.exists(snapshotDir)) {
349       LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir);
350       return false;
351     }
352 
353     snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
354     snapshotTableDesc = FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir);
355     return true;
356   }
357 
358   /**
359    * Dump the {@link SnapshotDescription}
360    */
361   private void printInfo() {
362     SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
363     System.out.println("Snapshot Info");
364     System.out.println("----------------------------------------");
365     System.out.println("   Name: " + snapshotDesc.getName());
366     System.out.println("   Type: " + snapshotDesc.getType());
367     System.out.println("  Table: " + snapshotTableDesc.getTableName().getNameAsString());
368     System.out.println(" Format: " + snapshotDesc.getVersion());
369     System.out.println("Created: " + df.format(new Date(snapshotDesc.getCreationTime())));
370     System.out.println();
371   }
372 
373   /**
374    * Dump the {@link HTableDescriptor}
375    */
376   private void printSchema() {
377     System.out.println("Table Descriptor");
378     System.out.println("----------------------------------------");
379     System.out.println(snapshotTableDesc.toString());
380     System.out.println();
381   }
382 
383   /**
384    * Collect the hfiles and logs statistics of the snapshot and
385    * dump the file list if requested and the collected information.
386    */
387   private void printFiles(final boolean showFiles, final boolean showStats) throws IOException {
388     if (showFiles) {
389       System.out.println("Snapshot Files");
390       System.out.println("----------------------------------------");
391     }
392 
393     // Collect information about hfiles and logs in the snapshot
394     final String table = snapshotTableDesc.getTableName().getNameAsString();
395     final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, this.snapshotDesc);
396     SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
397       new SnapshotReferenceUtil.FileVisitor() {
398         public void storeFile (final String region, final String family, final String hfile)
399             throws IOException {
400           SnapshotStats.FileInfo info = stats.addStoreFile(region, family, hfile);
401 
402           if (showFiles) {
403             String state = info.getStateToString();
404             System.out.printf("%8s %s/%s/%s/%s %s%n",
405               (info.isMissing() ? "-" : fileSizeToString(info.getSize())),
406               table, region, family, hfile,
407               state == null ? "" : "(" + state + ")");
408           }
409         }
410 
411         public void recoveredEdits (final String region, final String logfile)
412             throws IOException {
413           SnapshotStats.FileInfo info = stats.addRecoveredEdits(region, logfile);
414 
415           if (showFiles) {
416             System.out.printf("%8s recovered.edits %s on region %s%n",
417               fileSizeToString(info.getSize()), logfile, region);
418           }
419         }
420 
421         public void logFile (final String server, final String logfile)
422             throws IOException {
423           SnapshotStats.FileInfo info = stats.addLogFile(server, logfile);
424 
425           if (showFiles) {
426             String state = info.getStateToString();
427             System.out.printf("%8s log %s on server %s %s%n",
428               (info.isMissing() ? "-" : fileSizeToString(info.getSize())),
429               logfile, server,
430               state == null ? "" : "(" + state + ")");
431           }
432         }
433     });
434 
435     // Dump the stats
436     System.out.println();
437     if (stats.isSnapshotCorrupted()) {
438       System.out.println("**************************************************************");
439       System.out.printf("BAD SNAPSHOT: %d hfile(s) and %d log(s) missing.%n",
440         stats.getMissingStoreFilesCount(), stats.getMissingLogsCount());
441       System.out.println("**************************************************************");
442     }
443 
444     if (showStats) {
445       System.out.printf("%d HFiles (%d in archive), total size %s (%.2f%% %s shared with the source table)%n",
446         stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(),
447         fileSizeToString(stats.getStoreFilesSize()),
448         stats.getSharedStoreFilePercentage(),
449         fileSizeToString(stats.getSharedStoreFilesSize())
450       );
451       System.out.printf("%d Logs, total size %s%n",
452         stats.getLogsCount(), fileSizeToString(stats.getLogsSize()));
453       System.out.println();
454     }
455   }
456 
457   private String fileSizeToString(long size) {
458     return printSizeInBytes ? Long.toString(size) : StringUtils.humanReadableInt(size);
459   }
460 
461   private void printUsageAndExit() {
462     System.err.printf("Usage: bin/hbase %s [options]%n", getClass().getName());
463     System.err.println(" where [options] are:");
464     System.err.println("  -h|-help                Show this help and exit.");
465     System.err.println("  -remote-dir             Root directory that contains the snapshots.");
466     System.err.println("  -list-snapshots         List all the available snapshots and exit.");
467     System.err.println("  -snapshot NAME          Snapshot to examine.");
468     System.err.println("  -files                  Files and logs list.");
469     System.err.println("  -stats                  Files and logs stats.");
470     System.err.println("  -schema                 Describe the snapshotted table.");
471     System.err.println();
472     System.err.println("Examples:");
473     System.err.println("  hbase " + getClass() + " \\");
474     System.err.println("    -snapshot MySnapshot -files");
475     System.exit(1);
476   }
477 
478   /**
479    * Returns the snapshot stats
480    * @param conf the {@link Configuration} to use
481    * @param snapshot {@link SnapshotDescription} to get stats from
482    * @return the snapshot stats
483    */
484   public static SnapshotStats getSnapshotStats(final Configuration conf,
485       final SnapshotDescription snapshot) throws IOException {
486     Path rootDir = FSUtils.getRootDir(conf);
487     FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
488     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
489     final SnapshotStats stats = new SnapshotStats(conf, fs, snapshot);
490     SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
491       new SnapshotReferenceUtil.FileVisitor() {
492         public void storeFile (final String region, final String family, final String hfile)
493             throws IOException {
494           stats.addStoreFile(region, family, hfile);
495         }
496 
497         public void recoveredEdits (final String region, final String logfile) throws IOException {
498           stats.addRecoveredEdits(region, logfile);
499         }
500 
501         public void logFile (final String server, final String logfile) throws IOException {
502           stats.addLogFile(server, logfile);
503         }
504     });
505     return stats;
506   }
507 
508   /**
509    * Returns the list of available snapshots in the specified location
510    * @param conf the {@link Configuration} to use
511    * @return the list of snapshots
512    */
513   public static List<SnapshotDescription> getSnapshotList(final Configuration conf)
514       throws IOException {
515     Path rootDir = FSUtils.getRootDir(conf);
516     FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
517     Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
518     FileStatus[] snapshots = fs.listStatus(snapshotDir,
519       new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
520     List<SnapshotDescription> snapshotLists =
521       new ArrayList<SnapshotDescription>(snapshots.length);
522     for (FileStatus snapshotDirStat: snapshots) {
523       snapshotLists.add(SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDirStat.getPath()));
524     }
525     return snapshotLists;
526   }
527 
528   /**
529    * The guts of the {@link #main} method.
530    * Call this method to avoid the {@link #main(String[])} System.exit.
531    * @param args
532    * @return errCode
533    * @throws Exception
534    */
535   static int innerMain(final String [] args) throws Exception {
536     return ToolRunner.run(HBaseConfiguration.create(), new SnapshotInfo(), args);
537   }
538 
539   public static void main(String[] args) throws Exception {
540      System.exit(innerMain(args));
541   }
542 }