1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.util;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertNotNull;
24 import static org.junit.Assert.assertNull;
25 import static org.junit.Assert.assertTrue;
26
27 import java.io.IOException;
28 import java.util.UUID;
29 import java.util.Set;
30 import java.util.HashSet;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.fs.FSDataOutputStream;
35 import org.apache.hadoop.fs.FileSystem;
36 import org.apache.hadoop.fs.Path;
37 import org.apache.hadoop.hbase.HBaseTestingUtility;
38 import org.apache.hadoop.hbase.HConstants;
39 import org.apache.hadoop.hbase.MediumTests;
40 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
41 import org.junit.*;
42 import org.junit.experimental.categories.Category;
43
44
45
46
47 @Category(MediumTests.class)
48 public class TestFSVisitor {
49 final Log LOG = LogFactory.getLog(getClass());
50
51 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
52
53 private final String TABLE_NAME = "testtb";
54
55 private Set<String> tableFamilies;
56 private Set<String> tableRegions;
57 private Set<String> recoveredEdits;
58 private Set<String> tableHFiles;
59 private Set<String> regionServers;
60 private Set<String> serverLogs;
61
62 private FileSystem fs;
63 private Path tableDir;
64 private Path logsDir;
65 private Path rootDir;
66
67 @Before
68 public void setUp() throws Exception {
69 fs = FileSystem.get(TEST_UTIL.getConfiguration());
70 rootDir = TEST_UTIL.getDataTestDir("hbase");
71 logsDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);
72
73 tableFamilies = new HashSet<String>();
74 tableRegions = new HashSet<String>();
75 recoveredEdits = new HashSet<String>();
76 tableHFiles = new HashSet<String>();
77 regionServers = new HashSet<String>();
78 serverLogs = new HashSet<String>();
79 tableDir = createTableFiles(rootDir, TABLE_NAME, tableRegions, tableFamilies, tableHFiles);
80 createRecoverEdits(tableDir, tableRegions, recoveredEdits);
81 createLogs(logsDir, regionServers, serverLogs);
82 FSUtils.logFileSystemState(fs, rootDir, LOG);
83 }
84
85 @After
86 public void tearDown() throws Exception {
87 fs.delete(rootDir);
88 }
89
90 @Test
91 public void testVisitStoreFiles() throws IOException {
92 final Set<String> regions = new HashSet<String>();
93 final Set<String> families = new HashSet<String>();
94 final Set<String> hfiles = new HashSet<String>();
95 FSVisitor.visitTableStoreFiles(fs, tableDir, new FSVisitor.StoreFileVisitor() {
96 public void storeFile(final String region, final String family, final String hfileName)
97 throws IOException {
98 regions.add(region);
99 families.add(family);
100 hfiles.add(hfileName);
101 }
102 });
103 assertEquals(tableRegions, regions);
104 assertEquals(tableFamilies, families);
105 assertEquals(tableHFiles, hfiles);
106 }
107
108 @Test
109 public void testVisitRecoveredEdits() throws IOException {
110 final Set<String> regions = new HashSet<String>();
111 final Set<String> edits = new HashSet<String>();
112 FSVisitor.visitTableRecoveredEdits(fs, tableDir, new FSVisitor.RecoveredEditsVisitor() {
113 public void recoveredEdits (final String region, final String logfile)
114 throws IOException {
115 regions.add(region);
116 edits.add(logfile);
117 }
118 });
119 assertEquals(tableRegions, regions);
120 assertEquals(recoveredEdits, edits);
121 }
122
123 @Test
124 public void testVisitLogFiles() throws IOException {
125 final Set<String> servers = new HashSet<String>();
126 final Set<String> logs = new HashSet<String>();
127 FSVisitor.visitLogFiles(fs, rootDir, new FSVisitor.LogFileVisitor() {
128 public void logFile (final String server, final String logfile) throws IOException {
129 servers.add(server);
130 logs.add(logfile);
131 }
132 });
133 assertEquals(regionServers, servers);
134 assertEquals(serverLogs, logs);
135 }
136
137
138
139
140
141
142
143
144
145
146
147 private Path createTableFiles(final Path rootDir, final String tableName,
148 final Set<String> tableRegions, final Set<String> tableFamilies,
149 final Set<String> tableHFiles) throws IOException {
150 Path tableDir = new Path(rootDir, tableName);
151 for (int r = 0; r < 10; ++r) {
152 String regionName = MD5Hash.getMD5AsHex(Bytes.toBytes(r));
153 tableRegions.add(regionName);
154 Path regionDir = new Path(tableDir, regionName);
155 for (int f = 0; f < 3; ++f) {
156 String familyName = "f" + f;
157 tableFamilies.add(familyName);
158 Path familyDir = new Path(regionDir, familyName);
159 fs.mkdirs(familyDir);
160 for (int h = 0; h < 5; ++h) {
161 String hfileName = UUID.randomUUID().toString().replaceAll("-", "");
162 tableHFiles.add(hfileName);
163 fs.createNewFile(new Path(familyDir, hfileName));
164 }
165 }
166 }
167 return tableDir;
168 }
169
170
171
172
173
174
175
176
177 private void createRecoverEdits(final Path tableDir, final Set<String> tableRegions,
178 final Set<String> recoverEdits) throws IOException {
179 for (String region: tableRegions) {
180 Path regionEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(new Path(tableDir, region));
181 long seqId = System.currentTimeMillis();
182 for (int i = 0; i < 3; ++i) {
183 String editName = String.format("%019d", seqId + i);
184 recoverEdits.add(editName);
185 FSDataOutputStream stream = fs.create(new Path(regionEditsDir, editName));
186 stream.write(Bytes.toBytes("test"));
187 stream.close();
188 }
189 }
190 }
191
192
193
194
195
196
197
198
199
200 private void createLogs(final Path logDir, final Set<String> servers,
201 final Set<String> logs) throws IOException {
202 for (int s = 0; s < 7; ++s) {
203 String server = String.format("server%d,%d,%d", s, s, System.currentTimeMillis());
204 servers.add(server);
205 Path serverLogDir = new Path(logDir, server);
206 fs.mkdirs(serverLogDir);
207 for (int i = 0; i < 5; ++i) {
208 String logfile = server + '.' + i;
209 logs.add(logfile);
210 FSDataOutputStream stream = fs.create(new Path(serverLogDir, logfile));
211 stream.write(Bytes.toBytes("test"));
212 stream.close();
213 }
214 }
215 }
216 }