1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver.wal;
21
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.NavigableSet;
25 import java.util.TreeSet;
26 import java.util.UUID;
27 import java.util.concurrent.atomic.AtomicLong;
28 import java.util.regex.Matcher;
29 import java.util.regex.Pattern;
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.conf.Configuration;
34 import org.apache.hadoop.fs.FileStatus;
35 import org.apache.hadoop.fs.FileSystem;
36 import org.apache.hadoop.fs.Path;
37 import org.apache.hadoop.fs.PathFilter;
38 import org.apache.hadoop.hbase.TableName;
39 import org.apache.hadoop.hbase.HConstants;
40 import org.apache.hadoop.hbase.HRegionInfo;
41 import org.apache.hadoop.hbase.HTableDescriptor;
42 import org.apache.hadoop.hbase.ServerName;
43 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
44 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
45 import org.apache.hadoop.hbase.util.FSUtils;
46
47 import com.google.protobuf.TextFormat;
48
49 public class HLogUtil {
50 static final Log LOG = LogFactory.getLog(HLogUtil.class);
51
52
53
54
55 private static final Pattern pattern =
56 Pattern.compile(".*\\.\\d*("+HLog.META_HLOG_FILE_EXTN+")*");
57
58
59
60
61
62
63
64 public static boolean validateHLogFilename(String filename) {
65 return pattern.matcher(filename).matches();
66 }
67
68
69
70
71
72
73
74
75
76
77
78 public static String getHLogDirectoryName(final String serverName) {
79 StringBuilder dirName = new StringBuilder(HConstants.HREGION_LOGDIR_NAME);
80 dirName.append("/");
81 dirName.append(serverName);
82 return dirName.toString();
83 }
84
85
86
87
88
89
90
91 public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {
92 return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);
93 }
94
95
96
97
98
99
100
101
102
103
104 public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)
105 throws IOException {
106 Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."
107 + System.currentTimeMillis());
108 if (!fs.rename(edits, moveAsideName)) {
109 LOG.warn("Rename failed from " + edits + " to " + moveAsideName);
110 }
111 return moveAsideName;
112 }
113
114
115
116
117
118
119
120
121
122 public static ServerName getServerNameFromHLogDirectoryName(
123 Configuration conf, String path) throws IOException {
124 if (path == null
125 || path.length() <= HConstants.HREGION_LOGDIR_NAME.length()) {
126 return null;
127 }
128
129 if (conf == null) {
130 throw new IllegalArgumentException("parameter conf must be set");
131 }
132
133 final String rootDir = conf.get(HConstants.HBASE_DIR);
134 if (rootDir == null || rootDir.isEmpty()) {
135 throw new IllegalArgumentException(HConstants.HBASE_DIR
136 + " key not found in conf.");
137 }
138
139 final StringBuilder startPathSB = new StringBuilder(rootDir);
140 if (!rootDir.endsWith("/"))
141 startPathSB.append('/');
142 startPathSB.append(HConstants.HREGION_LOGDIR_NAME);
143 if (!HConstants.HREGION_LOGDIR_NAME.endsWith("/"))
144 startPathSB.append('/');
145 final String startPath = startPathSB.toString();
146
147 String fullPath;
148 try {
149 fullPath = FileSystem.get(conf).makeQualified(new Path(path)).toString();
150 } catch (IllegalArgumentException e) {
151 LOG.info("Call to makeQualified failed on " + path + " " + e.getMessage());
152 return null;
153 }
154
155 if (!fullPath.startsWith(startPath)) {
156 return null;
157 }
158
159 final String serverNameAndFile = fullPath.substring(startPath.length());
160
161 if (serverNameAndFile.indexOf('/') < "a,0,0".length()) {
162
163 return null;
164 }
165
166 Path p = new Path(path);
167 return getServerNameFromHLogDirectoryName(p);
168 }
169
170
171
172
173
174
175
176
177 public static ServerName getServerNameFromHLogDirectoryName(Path logFile) {
178 Path logDir = logFile.getParent();
179 String logDirName = logDir.getName();
180 if (logDirName.equals(HConstants.HREGION_LOGDIR_NAME)) {
181 logDir = logFile;
182 logDirName = logDir.getName();
183 }
184 ServerName serverName = null;
185 if (logDirName.endsWith(HLog.SPLITTING_EXT)) {
186 logDirName = logDirName.substring(0, logDirName.length() - HLog.SPLITTING_EXT.length());
187 }
188 try {
189 serverName = ServerName.parseServerName(logDirName);
190 } catch (IllegalArgumentException ex) {
191 serverName = null;
192 LOG.warn("Invalid log file path=" + logFile, ex);
193 }
194 if (serverName != null && serverName.getStartcode() < 0) {
195 LOG.warn("Invalid log file path=" + logFile);
196 return null;
197 }
198 return serverName;
199 }
200
201
202
203
204
205
206
207
208
209
210 public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,
211 final Path regiondir) throws IOException {
212 NavigableSet<Path> filesSorted = new TreeSet<Path>();
213 Path editsdir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
214 if (!fs.exists(editsdir))
215 return filesSorted;
216 FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {
217 @Override
218 public boolean accept(Path p) {
219 boolean result = false;
220 try {
221
222
223
224
225 Matcher m = HLog.EDITFILES_NAME_PATTERN.matcher(p.getName());
226 result = fs.isFile(p) && m.matches();
227
228
229 if (p.getName().endsWith(HLog.RECOVERED_LOG_TMPFILE_SUFFIX)) {
230 result = false;
231 }
232 } catch (IOException e) {
233 LOG.warn("Failed isFile check on " + p);
234 }
235 return result;
236 }
237 });
238 if (files == null)
239 return filesSorted;
240 for (FileStatus status : files) {
241 filesSorted.add(status.getPath());
242 }
243 return filesSorted;
244 }
245
246 public static boolean isMetaFile(Path p) {
247 return isMetaFile(p.getName());
248 }
249
250 public static boolean isMetaFile(String p) {
251 if (p != null && p.endsWith(HLog.META_HLOG_FILE_EXTN)) {
252 return true;
253 }
254 return false;
255 }
256
257
258
259
260
261
262
263
264 public static void writeCompactionMarker(HLog log, HTableDescriptor htd, HRegionInfo info,
265 final CompactionDescriptor c, AtomicLong sequenceId) throws IOException {
266 WALEdit e = WALEdit.createCompaction(info, c);
267 long now = EnvironmentEdgeManager.currentTimeMillis();
268 TableName tn = TableName.valueOf(c.getTableName().toByteArray());
269 long txid = log.appendNoSync(info, tn, e, new ArrayList<UUID>(), now, htd, sequenceId,
270 false, HConstants.NO_NONCE, HConstants.NO_NONCE);
271 log.sync(txid);
272
273 if (LOG.isTraceEnabled()) {
274 LOG.trace("Appended compaction marker " + TextFormat.shortDebugString(c));
275 }
276 }
277 }