View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.snapshot;
20  
21  import java.io.IOException;
22  import java.io.InterruptedIOException;
23  import java.util.ArrayList;
24  import java.util.List;
25  import java.util.concurrent.Callable;
26  import java.util.concurrent.Executor;
27  import java.util.concurrent.ExecutionException;
28  import java.util.concurrent.ExecutorCompletionService;
29  
30  import org.apache.commons.logging.Log;
31  import org.apache.commons.logging.LogFactory;
32  import org.apache.hadoop.hbase.classification.InterfaceAudience;
33  import org.apache.hadoop.conf.Configuration;
34  import org.apache.hadoop.fs.FSDataInputStream;
35  import org.apache.hadoop.fs.FSDataOutputStream;
36  import org.apache.hadoop.fs.FileStatus;
37  import org.apache.hadoop.fs.FileSystem;
38  import org.apache.hadoop.fs.Path;
39  import org.apache.hadoop.fs.PathFilter;
40  import org.apache.hadoop.hbase.HRegionInfo;
41  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
42  import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
43  import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
44  import org.apache.hadoop.hbase.util.ByteStringer;
45  import org.apache.hadoop.hbase.util.FSUtils;
46  
47  /**
48   * DO NOT USE DIRECTLY. USE {@link SnapshotManifest}.
49   *
50   * Snapshot v2 layout format
51   *  - Single Manifest file containing all the information of regions
52   *  - In the online-snapshot case each region will write a "region manifest"
53   *      /snapshotName/manifest.regionName
54   */
55  @InterfaceAudience.Private
56  public class SnapshotManifestV2 {
57    private static final Log LOG = LogFactory.getLog(SnapshotManifestV2.class);
58  
59    public static final int DESCRIPTOR_VERSION = 2;
60  
61    private static final String SNAPSHOT_MANIFEST_PREFIX = "region-manifest.";
62  
63    static class ManifestBuilder implements SnapshotManifest.RegionVisitor<
64                      SnapshotRegionManifest.Builder, SnapshotRegionManifest.FamilyFiles.Builder> {
65      private final Configuration conf;
66      private final Path snapshotDir;
67      private final FileSystem fs;
68  
69      public ManifestBuilder(final Configuration conf, final FileSystem fs, final Path snapshotDir) {
70        this.snapshotDir = snapshotDir;
71        this.conf = conf;
72        this.fs = fs;
73      }
74  
75      public SnapshotRegionManifest.Builder regionOpen(final HRegionInfo regionInfo) {
76        SnapshotRegionManifest.Builder manifest = SnapshotRegionManifest.newBuilder();
77        manifest.setRegionInfo(HRegionInfo.convert(regionInfo));
78        return manifest;
79      }
80  
81      public void regionClose(final SnapshotRegionManifest.Builder region) throws IOException {
82        SnapshotRegionManifest manifest = region.build();
83        FSDataOutputStream stream = fs.create(getRegionManifestPath(snapshotDir, manifest));
84        try {
85          manifest.writeTo(stream);
86        } finally {
87          stream.close();
88        }
89      }
90  
91      public SnapshotRegionManifest.FamilyFiles.Builder familyOpen(
92          final SnapshotRegionManifest.Builder region, final byte[] familyName) {
93        SnapshotRegionManifest.FamilyFiles.Builder family =
94            SnapshotRegionManifest.FamilyFiles.newBuilder();
95        family.setFamilyName(ByteStringer.wrap(familyName));
96        return family;
97      }
98  
99      public void familyClose(final SnapshotRegionManifest.Builder region,
100         final SnapshotRegionManifest.FamilyFiles.Builder family) {
101       region.addFamilyFiles(family.build());
102     }
103 
104     public void storeFile(final SnapshotRegionManifest.Builder region,
105         final SnapshotRegionManifest.FamilyFiles.Builder family, final StoreFileInfo storeFile)
106         throws IOException {
107       SnapshotRegionManifest.StoreFile.Builder sfManifest =
108             SnapshotRegionManifest.StoreFile.newBuilder();
109       sfManifest.setName(storeFile.getPath().getName());
110       if (storeFile.isReference()) {
111         sfManifest.setReference(storeFile.getReference().convert());
112       }
113       sfManifest.setFileSize(storeFile.getReferencedFileStatus(fs).getLen());
114       family.addStoreFiles(sfManifest.build());
115     }
116   }
117 
118   static List<SnapshotRegionManifest> loadRegionManifests(final Configuration conf,
119       final Executor executor,final FileSystem fs, final Path snapshotDir,
120       final SnapshotDescription desc) throws IOException {
121     FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir, new PathFilter() {
122       @Override
123       public boolean accept(Path path) {
124         return path.getName().startsWith(SNAPSHOT_MANIFEST_PREFIX);
125       }
126     });
127 
128     if (manifestFiles == null || manifestFiles.length == 0) return null;
129 
130     final ExecutorCompletionService<SnapshotRegionManifest> completionService =
131       new ExecutorCompletionService<SnapshotRegionManifest>(executor);
132     for (final FileStatus st: manifestFiles) {
133       completionService.submit(new Callable<SnapshotRegionManifest>() {
134         @Override
135         public SnapshotRegionManifest call() throws IOException {
136           FSDataInputStream stream = fs.open(st.getPath());
137           try {
138             return SnapshotRegionManifest.parseFrom(stream);
139           } finally {
140             stream.close();
141           }
142         }
143       });
144     }
145 
146     ArrayList<SnapshotRegionManifest> regionsManifest =
147         new ArrayList<SnapshotRegionManifest>(manifestFiles.length);
148     try {
149       for (int i = 0; i < manifestFiles.length; ++i) {
150         regionsManifest.add(completionService.take().get());
151       }
152     } catch (InterruptedException e) {
153       throw new InterruptedIOException(e.getMessage());
154     } catch (ExecutionException e) {
155       IOException ex = new IOException();
156       ex.initCause(e.getCause());
157       throw ex;
158     }
159     return regionsManifest;
160   }
161 
162   static void deleteRegionManifest(final FileSystem fs, final Path snapshotDir,
163       final SnapshotRegionManifest manifest) throws IOException {
164     fs.delete(getRegionManifestPath(snapshotDir, manifest), true);
165   }
166 
167   private static Path getRegionManifestPath(final Path snapshotDir,
168       final SnapshotRegionManifest manifest) {
169     String regionName = SnapshotManifest.getRegionNameFromManifest(manifest);
170     return new Path(snapshotDir, SNAPSHOT_MANIFEST_PREFIX + regionName);
171   }
172 }