1 /** 2 * 3 * Licensed to the Apache Software Foundation (ASF) under one 4 * or more contributor license agreements. See the NOTICE file 5 * distributed with this work for additional information 6 * regarding copyright ownership. The ASF licenses this file 7 * to you under the Apache License, Version 2.0 (the 8 * "License"); you may not use this file except in compliance 9 * with the License. You may obtain a copy of the License at 10 * 11 * http://www.apache.org/licenses/LICENSE-2.0 12 * 13 * Unless required by applicable law or agreed to in writing, software 14 * distributed under the License is distributed on an "AS IS" BASIS, 15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 * See the License for the specific language governing permissions and 17 * limitations under the License. 18 */ 19 20 package org.apache.hadoop.hbase.regionserver.wal; 21 22 import java.io.DataInput; 23 import java.io.DataOutput; 24 import java.io.IOException; 25 import java.util.List; 26 import java.util.UUID; 27 import java.util.concurrent.atomic.AtomicLong; 28 import java.util.regex.Pattern; 29 30 import org.apache.commons.logging.Log; 31 import org.apache.commons.logging.LogFactory; 32 import org.apache.hadoop.classification.InterfaceAudience; 33 import org.apache.hadoop.conf.Configuration; 34 import org.apache.hadoop.fs.FSDataInputStream; 35 import org.apache.hadoop.fs.FileSystem; 36 import org.apache.hadoop.fs.Path; 37 import org.apache.hadoop.hbase.HRegionInfo; 38 import org.apache.hadoop.hbase.HTableDescriptor; 39 import org.apache.hadoop.hbase.TableName; 40 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.WALTrailer; 41 import org.apache.hadoop.io.Writable; 42 43 import com.google.common.annotations.VisibleForTesting; 44 45 46 @InterfaceAudience.Private 47 // TODO: Rename interface to WAL 48 public interface HLog { 49 Log LOG = LogFactory.getLog(HLog.class); 50 51 /** File Extension used while splitting an HLog into regions (HBASE-2312) */ 52 // TODO: this seems like an implementation detail that does not belong here. 53 String SPLITTING_EXT = "-splitting"; 54 boolean SPLIT_SKIP_ERRORS_DEFAULT = false; 55 /** The hbase:meta region's HLog filename extension */ 56 String META_HLOG_FILE_EXTN = ".meta"; 57 58 /** 59 * Configuration name of HLog Trailer's warning size. If a waltrailer's size is greater than the 60 * configured size, a warning is logged. This is used with Protobuf reader/writer. 61 */ 62 // TODO: Implementation detail. Why in here? 63 String WAL_TRAILER_WARN_SIZE = "hbase.regionserver.waltrailer.warn.size"; 64 int DEFAULT_WAL_TRAILER_WARN_SIZE = 1024 * 1024; // 1MB 65 66 // TODO: Implemenation detail. Why in here? 67 Pattern EDITFILES_NAME_PATTERN = Pattern.compile("-?[0-9]+"); 68 String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp"; 69 70 interface Reader { 71 72 /** 73 * @param fs File system. 74 * @param path Path. 75 * @param c Configuration. 76 * @param s Input stream that may have been pre-opened by the caller; may be null. 77 */ 78 void init(FileSystem fs, Path path, Configuration c, FSDataInputStream s) throws IOException; 79 80 void close() throws IOException; 81 82 Entry next() throws IOException; 83 84 Entry next(Entry reuse) throws IOException; 85 86 void seek(long pos) throws IOException; 87 88 long getPosition() throws IOException; 89 void reset() throws IOException; 90 91 /** 92 * @return the WALTrailer of the current HLog. It may be null in case of legacy or corrupt WAL 93 * files. 94 */ 95 // TODO: What we need a trailer on WAL for? 96 WALTrailer getWALTrailer(); 97 } 98 99 interface Writer { 100 void init(FileSystem fs, Path path, Configuration c, boolean overwritable) throws IOException; 101 102 void close() throws IOException; 103 104 void sync() throws IOException; 105 106 void append(Entry entry) throws IOException; 107 108 long getLength() throws IOException; 109 110 /** 111 * Sets HLog's WALTrailer. This trailer is appended at the end of WAL on closing. 112 * @param walTrailer trailer to append to WAL. 113 */ 114 void setWALTrailer(WALTrailer walTrailer); 115 } 116 117 /** 118 * Utility class that lets us keep track of the edit with it's key. 119 * Only used when splitting logs. 120 */ 121 // TODO: Remove this Writable. 122 class Entry implements Writable { 123 private WALEdit edit; 124 private HLogKey key; 125 126 public Entry() { 127 edit = new WALEdit(); 128 key = new HLogKey(); 129 } 130 131 /** 132 * Constructor for both params 133 * 134 * @param edit log's edit 135 * @param key log's key 136 */ 137 public Entry(HLogKey key, WALEdit edit) { 138 super(); 139 this.key = key; 140 this.edit = edit; 141 } 142 143 /** 144 * Gets the edit 145 * 146 * @return edit 147 */ 148 public WALEdit getEdit() { 149 return edit; 150 } 151 152 /** 153 * Gets the key 154 * 155 * @return key 156 */ 157 public HLogKey getKey() { 158 return key; 159 } 160 161 /** 162 * Set compression context for this entry. 163 * 164 * @param compressionContext 165 * Compression context 166 */ 167 public void setCompressionContext(CompressionContext compressionContext) { 168 edit.setCompressionContext(compressionContext); 169 key.setCompressionContext(compressionContext); 170 } 171 172 @Override 173 public String toString() { 174 return this.key + "=" + this.edit; 175 } 176 177 @Override 178 @SuppressWarnings("deprecation") 179 public void write(DataOutput dataOutput) throws IOException { 180 this.key.write(dataOutput); 181 this.edit.write(dataOutput); 182 } 183 184 @Override 185 public void readFields(DataInput dataInput) throws IOException { 186 this.key.readFields(dataInput); 187 this.edit.readFields(dataInput); 188 } 189 } 190 191 /** 192 * registers WALActionsListener 193 * 194 * @param listener 195 */ 196 void registerWALActionsListener(final WALActionsListener listener); 197 198 /** 199 * unregisters WALActionsListener 200 * 201 * @param listener 202 */ 203 boolean unregisterWALActionsListener(final WALActionsListener listener); 204 205 /** 206 * @return Current state of the monotonically increasing file id. 207 */ 208 // TODO: Remove. Implementation detail. 209 long getFilenum(); 210 211 /** 212 * @return the number of HLog files 213 */ 214 int getNumLogFiles(); 215 216 /** 217 * @return the size of HLog files 218 */ 219 long getLogFileSize(); 220 221 // TODO: Log rolling should not be in this interface. 222 /** 223 * Roll the log writer. That is, start writing log messages to a new file. 224 * 225 * <p> 226 * The implementation is synchronized in order to make sure there's one rollWriter 227 * running at any given time. 228 * 229 * @return If lots of logs, flush the returned regions so next time through we 230 * can clean logs. Returns null if nothing to flush. Names are actual 231 * region names as returned by {@link HRegionInfo#getEncodedName()} 232 * @throws org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException 233 * @throws IOException 234 */ 235 byte[][] rollWriter() throws FailedLogCloseException, IOException; 236 237 /** 238 * Roll the log writer. That is, start writing log messages to a new file. 239 * 240 * <p> 241 * The implementation is synchronized in order to make sure there's one rollWriter 242 * running at any given time. 243 * 244 * @param force 245 * If true, force creation of a new writer even if no entries have 246 * been written to the current writer 247 * @return If lots of logs, flush the returned regions so next time through we 248 * can clean logs. Returns null if nothing to flush. Names are actual 249 * region names as returned by {@link HRegionInfo#getEncodedName()} 250 * @throws org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException 251 * @throws IOException 252 */ 253 byte[][] rollWriter(boolean force) throws FailedLogCloseException, 254 IOException; 255 256 /** 257 * Shut down the log. 258 * 259 * @throws IOException 260 */ 261 void close() throws IOException; 262 263 /** 264 * Shut down the log and delete the log directory 265 * 266 * @throws IOException 267 */ 268 void closeAndDelete() throws IOException; 269 270 /** 271 * Same as appendNoSync(HRegionInfo, TableName, WALEdit, List, long, HTableDescriptor), 272 * except it causes a sync on the log 273 * @param sequenceId of the region. 274 */ 275 @VisibleForTesting 276 public void append(HRegionInfo info, TableName tableName, WALEdit edits, 277 final long now, HTableDescriptor htd, AtomicLong sequenceId) throws IOException; 278 279 /** 280 * For notification post append to the writer. 281 * @param entries 282 */ 283 void postAppend(final List<Entry> entries); 284 285 /** 286 * For notification post writer sync. 287 */ 288 void postSync(); 289 290 /** 291 * Append a set of edits to the log. Log edits are keyed by (encoded) regionName, rowname, and 292 * log-sequence-id. The HLog is not flushed after this transaction is written to the log. 293 * @param info 294 * @param tableName 295 * @param edits 296 * @param clusterIds The clusters that have consumed the change (for replication) 297 * @param now 298 * @param htd 299 * @param sequenceId of the region 300 * @return txid of this transaction 301 * @throws IOException 302 */ 303 long appendNoSync(HRegionInfo info, TableName tableName, WALEdit edits, 304 List<UUID> clusterIds, final long now, HTableDescriptor htd, AtomicLong sequenceId, 305 boolean isInMemstore, long nonceGroup, long nonce) throws IOException; 306 307 // TODO: Do we need all these versions of sync? 308 void hsync() throws IOException; 309 310 void hflush() throws IOException; 311 312 void sync() throws IOException; 313 314 void sync(long txid) throws IOException; 315 316 /** 317 * WAL keeps track of the sequence numbers that were not yet flushed from memstores 318 * in order to be able to do cleanup. This method tells WAL that some region is about 319 * to flush memstore. 320 * 321 * We stash the oldest seqNum for the region, and let the the next edit inserted in this 322 * region be recorded in {@link #append(HRegionInfo, TableName, WALEdit, long, HTableDescriptor, 323 * AtomicLong)} as new oldest seqnum. 324 * In case of flush being aborted, we put the stashed value back; in case of flush succeeding, 325 * the seqNum of that first edit after start becomes the valid oldest seqNum for this region. 326 * 327 * @return true if the flush can proceed, false in case wal is closing (ususally, when server is 328 * closing) and flush couldn't be started. 329 */ 330 boolean startCacheFlush(final byte[] encodedRegionName); 331 332 /** 333 * Complete the cache flush. 334 * @param encodedRegionName Encoded region name. 335 */ 336 void completeCacheFlush(final byte[] encodedRegionName); 337 338 /** 339 * Abort a cache flush. Call if the flush fails. Note that the only recovery 340 * for an aborted flush currently is a restart of the regionserver so the 341 * snapshot content dropped by the failure gets restored to the memstore.v 342 * @param encodedRegionName Encoded region name. 343 */ 344 void abortCacheFlush(byte[] encodedRegionName); 345 346 /** 347 * @return Coprocessor host. 348 */ 349 WALCoprocessorHost getCoprocessorHost(); 350 351 /** 352 * Get LowReplication-Roller status 353 * 354 * @return lowReplicationRollEnabled 355 */ 356 // TODO: This is implementation detail? 357 boolean isLowReplicationRollEnabled(); 358 359 /** Gets the earliest sequence number in the memstore for this particular region. 360 * This can serve as best-effort "recent" WAL number for this region. 361 * @param encodedRegionName The region to get the number for. 362 * @return The number if present, HConstants.NO_SEQNUM if absent. 363 */ 364 long getEarliestMemstoreSeqNum(byte[] encodedRegionName); 365 }