View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.io.hfile;
19  
20  import java.io.DataOutputStream;
21  import java.io.IOException;
22  
23  import org.apache.commons.logging.Log;
24  import org.apache.commons.logging.LogFactory;
25  import org.apache.hadoop.classification.InterfaceAudience;
26  import org.apache.hadoop.conf.Configuration;
27  import org.apache.hadoop.fs.FSDataOutputStream;
28  import org.apache.hadoop.fs.FileSystem;
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.hbase.HConstants;
31  import org.apache.hadoop.hbase.KeyValue;
32  import org.apache.hadoop.hbase.KeyValue.KVComparator;
33  import org.apache.hadoop.hbase.io.crypto.Encryption;
34  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
35  import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
36  import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
37  import org.apache.hadoop.hbase.security.EncryptionUtil;
38  import org.apache.hadoop.hbase.security.User;
39  import org.apache.hadoop.hbase.util.Bytes;
40  import org.apache.hadoop.io.WritableUtils;
41  
42  /**
43   * {@link HFile} writer for version 3.
44   */
45  @InterfaceAudience.Private
46  public class HFileWriterV3 extends HFileWriterV2 {
47  
48    private static final Log LOG = LogFactory.getLog(HFileWriterV3.class);
49  
50    private int maxTagsLength = 0;
51  
52    static class WriterFactoryV3 extends HFile.WriterFactory {
53      WriterFactoryV3(Configuration conf, CacheConfig cacheConf) {
54        super(conf, cacheConf);
55      }
56  
57      @Override
58      public Writer createWriter(FileSystem fs, Path path, FSDataOutputStream ostream,
59          final KVComparator comparator, HFileContext fileContext)
60          throws IOException {
61        return new HFileWriterV3(conf, cacheConf, fs, path, ostream, comparator, fileContext);
62      }
63    }
64  
65    /** Constructor that takes a path, creates and closes the output stream. */
66    public HFileWriterV3(Configuration conf, CacheConfig cacheConf, FileSystem fs, Path path,
67        FSDataOutputStream ostream, final KVComparator comparator,
68        final HFileContext fileContext) throws IOException {
69      super(conf, cacheConf, fs, path, ostream, comparator, fileContext);
70      if (LOG.isTraceEnabled()) {
71        LOG.trace("Writer" + (path != null ? " for " + path : "") +
72          " initialized with cacheConf: " + cacheConf +
73          " comparator: " + comparator.getClass().getSimpleName() +
74          " fileContext: " + fileContext);
75      }
76    }
77  
78    /**
79     * Add key/value to file. Keys must be added in an order that agrees with the
80     * Comparator passed on construction.
81     * 
82     * @param kv
83     *          KeyValue to add. Cannot be empty nor null.
84     * @throws IOException
85     */
86    @Override
87    public void append(final KeyValue kv) throws IOException {
88      // Currently get the complete arrays
89      append(kv.getMvccVersion(), kv.getBuffer(), kv.getKeyOffset(), kv.getKeyLength(),
90          kv.getBuffer(), kv.getValueOffset(), kv.getValueLength(), kv.getBuffer(),
91          kv.getTagsOffset(), kv.getTagsLengthUnsigned());
92      this.maxMemstoreTS = Math.max(this.maxMemstoreTS, kv.getMvccVersion());
93    }
94    
95    /**
96     * Add key/value to file. Keys must be added in an order that agrees with the
97     * Comparator passed on construction.
98     * @param key
99     *          Key to add. Cannot be empty nor null.
100    * @param value
101    *          Value to add. Cannot be empty nor null.
102    * @throws IOException
103    */
104   @Override
105   public void append(final byte[] key, final byte[] value) throws IOException {
106     append(key, value, HConstants.EMPTY_BYTE_ARRAY);
107   }
108 
109   /**
110    * Add key/value to file. Keys must be added in an order that agrees with the
111    * Comparator passed on construction.
112    * @param key
113    *          Key to add. Cannot be empty nor null.
114    * @param value
115    *          Value to add. Cannot be empty nor null.
116    * @param tag
117    *          Tag t add. Cannot be empty or null.
118    * @throws IOException
119    */
120   @Override
121   public void append(final byte[] key, final byte[] value, byte[] tag) throws IOException {
122     append(0, key, 0, key.length, value, 0, value.length, tag, 0, tag.length);
123   }
124 
125   /**
126    * Add key/value to file. Keys must be added in an order that agrees with the
127    * Comparator passed on construction.
128    * @param key
129    * @param koffset
130    * @param klength
131    * @param value
132    * @param voffset
133    * @param vlength
134    * @param tag
135    * @param tagsOffset
136    * @param tagLength
137    * @throws IOException
138    */
139   private void append(final long memstoreTS, final byte[] key, final int koffset,
140       final int klength, final byte[] value, final int voffset, final int vlength,
141       final byte[] tag, final int tagsOffset, final int tagsLength) throws IOException {
142     boolean dupKey = checkKey(key, koffset, klength);
143     checkValue(value, voffset, vlength);
144     if (!dupKey) {
145       checkBlockBoundary();
146     }
147 
148     if (!fsBlockWriter.isWriting())
149       newBlock();
150 
151     // Write length of key and value and then actual key and value bytes.
152     // Additionally, we may also write down the memstoreTS.
153     {
154       DataOutputStream out = fsBlockWriter.getUserDataStream();
155       out.writeInt(klength);
156       totalKeyLength += klength;
157       out.writeInt(vlength);
158       totalValueLength += vlength;
159       out.write(key, koffset, klength);
160       out.write(value, voffset, vlength);
161       // Write the additional tag into the stream
162       if (hFileContext.isIncludesTags()) {
163         out.writeShort(tagsLength);
164         if (tagsLength > 0) {
165           out.write(tag, tagsOffset, tagsLength);
166           if (tagsLength > maxTagsLength) {
167             maxTagsLength = tagsLength;
168           }
169         }
170       }
171       if (this.hFileContext.isIncludesMvcc()) {
172         WritableUtils.writeVLong(out, memstoreTS);
173       }
174     }
175 
176     // Are we the first key in this block?
177     if (firstKeyInBlock == null) {
178       // Copy the key.
179       firstKeyInBlock = new byte[klength];
180       System.arraycopy(key, koffset, firstKeyInBlock, 0, klength);
181     }
182 
183     lastKeyBuffer = key;
184     lastKeyOffset = koffset;
185     lastKeyLength = klength;
186     entryCount++;
187   }
188   
189   protected void finishFileInfo() throws IOException {
190     super.finishFileInfo();
191     if (hFileContext.getDataBlockEncoding() == DataBlockEncoding.PREFIX_TREE) {
192       // In case of Prefix Tree encoding, we always write tags information into HFiles even if all
193       // KVs are having no tags.
194       fileInfo.append(FileInfo.MAX_TAGS_LEN, Bytes.toBytes(this.maxTagsLength), false);
195     } else if (hFileContext.isIncludesTags()) {
196       // When tags are not being written in this file, MAX_TAGS_LEN is excluded
197       // from the FileInfo
198       fileInfo.append(FileInfo.MAX_TAGS_LEN, Bytes.toBytes(this.maxTagsLength), false);
199       boolean tagsCompressed = (hFileContext.getDataBlockEncoding() != DataBlockEncoding.NONE)
200         && hFileContext.isCompressTags();
201       fileInfo.append(FileInfo.TAGS_COMPRESSED, Bytes.toBytes(tagsCompressed), false);
202     }
203   }
204 
205   @Override
206   protected int getMajorVersion() {
207     return 3;
208   }
209 
210   @Override
211   protected int getMinorVersion() {
212     return HFileReaderV3.MAX_MINOR_VERSION;
213   }
214 
215   @Override
216   protected void finishClose(FixedFileTrailer trailer) throws IOException {
217     // Write out encryption metadata before finalizing if we have a valid crypto context
218     Encryption.Context cryptoContext = hFileContext.getEncryptionContext();
219     if (cryptoContext != Encryption.Context.NONE) {
220       // Wrap the context's key and write it as the encryption metadata, the wrapper includes
221       // all information needed for decryption
222       trailer.setEncryptionKey(EncryptionUtil.wrapKey(cryptoContext.getConf(),
223         cryptoContext.getConf().get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY,
224           User.getCurrent().getShortName()),
225         cryptoContext.getKey()));
226     }
227     // Now we can finish the close
228     super.finishClose(trailer);
229   }
230 
231 }