View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver.wal;
21  
22  import java.io.DataInput;
23  import java.io.DataOutput;
24  import java.io.IOException;
25  import java.util.List;
26  import java.util.UUID;
27  import java.util.concurrent.atomic.AtomicLong;
28  import java.util.regex.Pattern;
29  
30  import org.apache.commons.logging.Log;
31  import org.apache.commons.logging.LogFactory;
32  import org.apache.hadoop.classification.InterfaceAudience;
33  import org.apache.hadoop.conf.Configuration;
34  import org.apache.hadoop.fs.FSDataInputStream;
35  import org.apache.hadoop.fs.FileSystem;
36  import org.apache.hadoop.fs.Path;
37  import org.apache.hadoop.hbase.HRegionInfo;
38  import org.apache.hadoop.hbase.HTableDescriptor;
39  import org.apache.hadoop.hbase.TableName;
40  import org.apache.hadoop.hbase.protobuf.generated.WALProtos.WALTrailer;
41  import org.apache.hadoop.io.Writable;
42  
43  import com.google.common.annotations.VisibleForTesting;
44  
45  
46  @InterfaceAudience.Private
47  // TODO: Rename interface to WAL
48  public interface HLog {
49    Log LOG = LogFactory.getLog(HLog.class);
50  
51    /** File Extension used while splitting an HLog into regions (HBASE-2312) */
52    // TODO: this seems like an implementation detail that does not belong here.
53    String SPLITTING_EXT = "-splitting";
54    boolean SPLIT_SKIP_ERRORS_DEFAULT = false;
55    /** The hbase:meta region's HLog filename extension */
56    String META_HLOG_FILE_EXTN = ".meta";
57  
58    /**
59     * Configuration name of HLog Trailer's warning size. If a waltrailer's size is greater than the
60     * configured size, a warning is logged. This is used with Protobuf reader/writer.
61     */
62    // TODO: Implementation detail.  Why in here?
63    String WAL_TRAILER_WARN_SIZE = "hbase.regionserver.waltrailer.warn.size";
64    int DEFAULT_WAL_TRAILER_WARN_SIZE = 1024 * 1024; // 1MB
65  
66    // TODO: Implemenation detail.  Why in here?
67    Pattern EDITFILES_NAME_PATTERN = Pattern.compile("-?[0-9]+");
68    String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp";
69  
70    interface Reader {
71  
72      /**
73       * @param fs File system.
74       * @param path Path.
75       * @param c Configuration.
76       * @param s Input stream that may have been pre-opened by the caller; may be null.
77       */
78      void init(FileSystem fs, Path path, Configuration c, FSDataInputStream s) throws IOException;
79  
80      void close() throws IOException;
81  
82      Entry next() throws IOException;
83  
84      Entry next(Entry reuse) throws IOException;
85  
86      void seek(long pos) throws IOException;
87  
88      long getPosition() throws IOException;
89      void reset() throws IOException;
90  
91      /**
92       * @return the WALTrailer of the current HLog. It may be null in case of legacy or corrupt WAL
93       *         files.
94       */
95      // TODO: What we need a trailer on WAL for?
96      WALTrailer getWALTrailer();
97    }
98  
99    interface Writer {
100     void init(FileSystem fs, Path path, Configuration c, boolean overwritable) throws IOException;
101 
102     void close() throws IOException;
103 
104     void sync() throws IOException;
105 
106     void append(Entry entry) throws IOException;
107 
108     long getLength() throws IOException;
109 
110     /**
111      * Sets HLog's WALTrailer. This trailer is appended at the end of WAL on closing.
112      * @param walTrailer trailer to append to WAL.
113      */
114     void setWALTrailer(WALTrailer walTrailer);
115   }
116 
117   /**
118    * Utility class that lets us keep track of the edit with it's key.
119    * Only used when splitting logs.
120    */
121   // TODO: Remove this Writable.
122   class Entry implements Writable {
123     private WALEdit edit;
124     private HLogKey key;
125 
126     public Entry() {
127       edit = new WALEdit();
128       key = new HLogKey();
129     }
130 
131     /**
132      * Constructor for both params
133      *
134      * @param edit log's edit
135      * @param key log's key
136      */
137     public Entry(HLogKey key, WALEdit edit) {
138       super();
139       this.key = key;
140       this.edit = edit;
141     }
142 
143     /**
144      * Gets the edit
145      *
146      * @return edit
147      */
148     public WALEdit getEdit() {
149       return edit;
150     }
151 
152     /**
153      * Gets the key
154      *
155      * @return key
156      */
157     public HLogKey getKey() {
158       return key;
159     }
160 
161     /**
162      * Set compression context for this entry.
163      *
164      * @param compressionContext
165      *          Compression context
166      */
167     public void setCompressionContext(CompressionContext compressionContext) {
168       edit.setCompressionContext(compressionContext);
169       key.setCompressionContext(compressionContext);
170     }
171 
172     @Override
173     public String toString() {
174       return this.key + "=" + this.edit;
175     }
176 
177     @Override
178     @SuppressWarnings("deprecation")
179     public void write(DataOutput dataOutput) throws IOException {
180       this.key.write(dataOutput);
181       this.edit.write(dataOutput);
182     }
183 
184     @Override
185     public void readFields(DataInput dataInput) throws IOException {
186       this.key.readFields(dataInput);
187       this.edit.readFields(dataInput);
188     }
189   }
190 
191   /**
192    * registers WALActionsListener
193    *
194    * @param listener
195    */
196   void registerWALActionsListener(final WALActionsListener listener);
197 
198   /**
199    * unregisters WALActionsListener
200    *
201    * @param listener
202    */
203   boolean unregisterWALActionsListener(final WALActionsListener listener);
204 
205   /**
206    * @return Current state of the monotonically increasing file id.
207    */
208   // TODO: Remove.  Implementation detail.
209   long getFilenum();
210 
211   /**
212    * @return the number of HLog files
213    */
214   int getNumLogFiles();
215   
216   /**
217    * @return the size of HLog files
218    */
219   long getLogFileSize();
220   
221   // TODO: Log rolling should not be in this interface.
222   /**
223    * Roll the log writer. That is, start writing log messages to a new file.
224    *
225    * <p>
226    * The implementation is synchronized in order to make sure there's one rollWriter
227    * running at any given time.
228    *
229    * @return If lots of logs, flush the returned regions so next time through we
230    *         can clean logs. Returns null if nothing to flush. Names are actual
231    *         region names as returned by {@link HRegionInfo#getEncodedName()}
232    * @throws org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException
233    * @throws IOException
234    */
235   byte[][] rollWriter() throws FailedLogCloseException, IOException;
236 
237   /**
238    * Roll the log writer. That is, start writing log messages to a new file.
239    *
240    * <p>
241    * The implementation is synchronized in order to make sure there's one rollWriter
242    * running at any given time.
243    *
244    * @param force
245    *          If true, force creation of a new writer even if no entries have
246    *          been written to the current writer
247    * @return If lots of logs, flush the returned regions so next time through we
248    *         can clean logs. Returns null if nothing to flush. Names are actual
249    *         region names as returned by {@link HRegionInfo#getEncodedName()}
250    * @throws org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException
251    * @throws IOException
252    */
253   byte[][] rollWriter(boolean force) throws FailedLogCloseException,
254       IOException;
255 
256   /**
257    * Shut down the log.
258    *
259    * @throws IOException
260    */
261   void close() throws IOException;
262 
263   /**
264    * Shut down the log and delete the log directory
265    *
266    * @throws IOException
267    */
268   void closeAndDelete() throws IOException;
269 
270   /**
271    * Same as appendNoSync(HRegionInfo, TableName, WALEdit, List, long, HTableDescriptor),
272    * except it causes a sync on the log
273    * @param sequenceId of the region.
274    */
275   @VisibleForTesting
276   public void append(HRegionInfo info, TableName tableName, WALEdit edits,
277       final long now, HTableDescriptor htd, AtomicLong sequenceId) throws IOException;
278 
279   /**
280    * For notification post append to the writer.
281    * @param entries
282    */
283   void postAppend(final List<Entry> entries);
284 
285   /**
286    * For notification post writer sync.
287    */
288   void postSync();
289 
290   /**
291    * Append a set of edits to the log. Log edits are keyed by (encoded) regionName, rowname, and
292    * log-sequence-id. The HLog is not flushed after this transaction is written to the log.
293    * @param info
294    * @param tableName
295    * @param edits
296    * @param clusterIds The clusters that have consumed the change (for replication)
297    * @param now
298    * @param htd
299    * @param sequenceId of the region
300    * @return txid of this transaction
301    * @throws IOException
302    */
303   long appendNoSync(HRegionInfo info, TableName tableName, WALEdit edits,
304       List<UUID> clusterIds, final long now, HTableDescriptor htd, AtomicLong sequenceId,
305       boolean isInMemstore, long nonceGroup, long nonce) throws IOException;
306 
307   // TODO: Do we need all these versions of sync?
308   void hsync() throws IOException;
309 
310   void hflush() throws IOException;
311 
312   void sync() throws IOException;
313 
314   void sync(long txid) throws IOException;
315 
316   /**
317    * WAL keeps track of the sequence numbers that were not yet flushed from memstores
318    * in order to be able to do cleanup. This method tells WAL that some region is about
319    * to flush memstore.
320    *
321    * We stash the oldest seqNum for the region, and let the the next edit inserted in this
322    * region be recorded in {@link #append(HRegionInfo, TableName, WALEdit, long, HTableDescriptor,
323    * AtomicLong)} as new oldest seqnum.
324    * In case of flush being aborted, we put the stashed value back; in case of flush succeeding,
325    * the seqNum of that first edit after start becomes the valid oldest seqNum for this region.
326    *
327    * @return true if the flush can proceed, false in case wal is closing (ususally, when server is
328    * closing) and flush couldn't be started.
329    */
330   boolean startCacheFlush(final byte[] encodedRegionName);
331 
332   /**
333    * Complete the cache flush.
334    * @param encodedRegionName Encoded region name.
335    */
336   void completeCacheFlush(final byte[] encodedRegionName);
337 
338   /**
339    * Abort a cache flush. Call if the flush fails. Note that the only recovery
340    * for an aborted flush currently is a restart of the regionserver so the
341    * snapshot content dropped by the failure gets restored to the memstore.v
342    * @param encodedRegionName Encoded region name.
343    */
344   void abortCacheFlush(byte[] encodedRegionName);
345 
346   /**
347    * @return Coprocessor host.
348    */
349   WALCoprocessorHost getCoprocessorHost();
350 
351   /**
352    * Get LowReplication-Roller status
353    *
354    * @return lowReplicationRollEnabled
355    */
356   // TODO: This is implementation detail?
357   boolean isLowReplicationRollEnabled();
358 
359   /** Gets the earliest sequence number in the memstore for this particular region.
360    * This can serve as best-effort "recent" WAL number for this region.
361    * @param encodedRegionName The region to get the number for.
362    * @return The number if present, HConstants.NO_SEQNUM if absent.
363    */
364   long getEarliestMemstoreSeqNum(byte[] encodedRegionName);
365 }