View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.util;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertNotNull;
24  import static org.junit.Assert.assertNull;
25  import static org.junit.Assert.assertTrue;
26  
27  import java.io.IOException;
28  import java.util.UUID;
29  import java.util.Set;
30  import java.util.HashSet;
31  
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.fs.FSDataOutputStream;
35  import org.apache.hadoop.fs.FileSystem;
36  import org.apache.hadoop.fs.Path;
37  import org.apache.hadoop.hbase.HBaseTestingUtility;
38  import org.apache.hadoop.hbase.HConstants;
39  import org.apache.hadoop.hbase.MediumTests;
40  import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
41  import org.junit.*;
42  import org.junit.experimental.categories.Category;
43  
44  /**
45   * Test {@link FSUtils}.
46   */
47  @Category(MediumTests.class)
48  public class TestFSVisitor {
49    final Log LOG = LogFactory.getLog(getClass());
50  
51    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
52  
53    private final String TABLE_NAME = "testtb";
54  
55    private Set<String> tableFamilies;
56    private Set<String> tableRegions;
57    private Set<String> recoveredEdits;
58    private Set<String> tableHFiles;
59    private Set<String> regionServers;
60    private Set<String> serverLogs;
61  
62    private FileSystem fs;
63    private Path tableDir;
64    private Path logsDir;
65    private Path rootDir;
66  
67    @Before
68    public void setUp() throws Exception {
69      fs = FileSystem.get(TEST_UTIL.getConfiguration());
70      rootDir = TEST_UTIL.getDataTestDir("hbase");
71      logsDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);
72  
73      tableFamilies = new HashSet<String>();
74      tableRegions = new HashSet<String>();
75      recoveredEdits = new HashSet<String>();
76      tableHFiles = new HashSet<String>();
77      regionServers = new HashSet<String>();
78      serverLogs = new HashSet<String>();
79      tableDir = createTableFiles(rootDir, TABLE_NAME, tableRegions, tableFamilies, tableHFiles);
80      createRecoverEdits(tableDir, tableRegions, recoveredEdits);
81      createLogs(logsDir, regionServers, serverLogs);
82      FSUtils.logFileSystemState(fs, rootDir, LOG);
83    }
84  
85    @After
86    public void tearDown() throws Exception {
87      fs.delete(rootDir);
88    }
89  
90    @Test
91    public void testVisitStoreFiles() throws IOException {
92      final Set<String> regions = new HashSet<String>();
93      final Set<String> families = new HashSet<String>();
94      final Set<String> hfiles = new HashSet<String>();
95      FSVisitor.visitTableStoreFiles(fs, tableDir, new FSVisitor.StoreFileVisitor() {
96        public void storeFile(final String region, final String family, final String hfileName)
97            throws IOException {
98          regions.add(region);
99          families.add(family);
100         hfiles.add(hfileName);
101       }
102     });
103     assertEquals(tableRegions, regions);
104     assertEquals(tableFamilies, families);
105     assertEquals(tableHFiles, hfiles);
106   }
107 
108   @Test
109   public void testVisitRecoveredEdits() throws IOException {
110     final Set<String> regions = new HashSet<String>();
111     final Set<String> edits = new HashSet<String>();
112     FSVisitor.visitTableRecoveredEdits(fs, tableDir, new FSVisitor.RecoveredEditsVisitor() {
113       public void recoveredEdits (final String region, final String logfile)
114           throws IOException {
115         regions.add(region);
116         edits.add(logfile);
117       }
118     });
119     assertEquals(tableRegions, regions);
120     assertEquals(recoveredEdits, edits);
121   }
122 
123   @Test
124   public void testVisitLogFiles() throws IOException {
125     final Set<String> servers = new HashSet<String>();
126     final Set<String> logs = new HashSet<String>();
127     FSVisitor.visitLogFiles(fs, rootDir, new FSVisitor.LogFileVisitor() {
128       public void logFile (final String server, final String logfile) throws IOException {
129         servers.add(server);
130         logs.add(logfile);
131       }
132     });
133     assertEquals(regionServers, servers);
134     assertEquals(serverLogs, logs);
135   }
136 
137 
138   /*
139    * |-testtb/
140    * |----f1d3ff8443297732862df21dc4e57262/
141    * |-------f1/
142    * |----------d0be84935ba84b66b1e866752ec5d663
143    * |----------9fc9d481718f4878b29aad0a597ecb94
144    * |-------f2/
145    * |----------4b0fe6068c564737946bcf4fd4ab8ae1
146    */
147   private Path createTableFiles(final Path rootDir, final String tableName,
148       final Set<String> tableRegions, final Set<String> tableFamilies,
149       final Set<String> tableHFiles) throws IOException {
150     Path tableDir = new Path(rootDir, tableName);
151     for (int r = 0; r < 10; ++r) {
152       String regionName = MD5Hash.getMD5AsHex(Bytes.toBytes(r));
153       tableRegions.add(regionName);
154       Path regionDir = new Path(tableDir, regionName);
155       for (int f = 0; f < 3; ++f) {
156         String familyName = "f" + f;
157         tableFamilies.add(familyName);
158         Path familyDir = new Path(regionDir, familyName);
159         fs.mkdirs(familyDir);
160         for (int h = 0; h < 5; ++h) {
161          String hfileName = UUID.randomUUID().toString().replaceAll("-", "");
162          tableHFiles.add(hfileName);
163          fs.createNewFile(new Path(familyDir, hfileName));
164         }
165       }
166     }
167     return tableDir;
168   }
169 
170   /*
171    * |-testtb/
172    * |----f1d3ff8443297732862df21dc4e57262/
173    * |-------recovered.edits/
174    * |----------0000001351969633479
175    * |----------0000001351969633481
176    */
177   private void createRecoverEdits(final Path tableDir, final Set<String> tableRegions,
178       final Set<String> recoverEdits) throws IOException {
179     for (String region: tableRegions) {
180       Path regionEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(new Path(tableDir, region));
181       long seqId = System.currentTimeMillis();
182       for (int i = 0; i < 3; ++i) {
183         String editName = String.format("%019d", seqId + i);
184         recoverEdits.add(editName);
185         FSDataOutputStream stream = fs.create(new Path(regionEditsDir, editName));
186         stream.write(Bytes.toBytes("test"));
187         stream.close();
188       }
189     }
190   }
191 
192   /*
193    * |-.logs/
194    * |----server5,5,1351969633508/
195    * |-------server5,5,1351969633508.0
196    * |----server6,6,1351969633512/
197    * |-------server6,6,1351969633512.0
198    * |-------server6,6,1351969633512.3
199    */
200   private void createLogs(final Path logDir, final Set<String> servers,
201       final Set<String> logs) throws IOException {
202     for (int s = 0; s < 7; ++s) {
203       String server = String.format("server%d,%d,%d", s, s, System.currentTimeMillis());
204       servers.add(server);
205       Path serverLogDir = new Path(logDir, server);
206       fs.mkdirs(serverLogDir);
207       for (int i = 0; i < 5; ++i) {
208         String logfile = server + '.' + i;
209         logs.add(logfile);
210         FSDataOutputStream stream = fs.create(new Path(serverLogDir, logfile));
211         stream.write(Bytes.toBytes("test"));
212         stream.close();
213       }
214     }
215   }
216 }