1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.io.hfile;
19
20 import java.io.IOException;
21 import java.security.Key;
22 import java.security.KeyException;
23
24 import org.apache.commons.logging.Log;
25 import org.apache.commons.logging.LogFactory;
26 import org.apache.hadoop.classification.InterfaceAudience;
27 import org.apache.hadoop.conf.Configuration;
28 import org.apache.hadoop.fs.Path;
29 import org.apache.hadoop.hbase.HConstants;
30 import org.apache.hadoop.hbase.fs.HFileSystem;
31 import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
32 import org.apache.hadoop.hbase.io.crypto.Cipher;
33 import org.apache.hadoop.hbase.io.crypto.Encryption;
34 import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
35 import org.apache.hadoop.hbase.security.EncryptionUtil;
36 import org.apache.hadoop.hbase.security.User;
37 import org.apache.hadoop.hbase.util.ByteBufferUtils;
38 import org.apache.hadoop.hbase.util.Bytes;
39 import org.apache.hadoop.io.WritableUtils;
40
41
42
43
44 @InterfaceAudience.Private
45 public class HFileReaderV3 extends HFileReaderV2 {
46
47 private static final Log LOG = LogFactory.getLog(HFileReaderV3.class);
48
49 public static final int MAX_MINOR_VERSION = 0;
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69 public HFileReaderV3(final Path path, FixedFileTrailer trailer, final FSDataInputStreamWrapper fsdis,
70 final long size, final CacheConfig cacheConf, final HFileSystem hfs,
71 final Configuration conf) throws IOException {
72 super(path, trailer, fsdis, size, cacheConf, hfs, conf);
73 byte[] tmp = fileInfo.get(FileInfo.MAX_TAGS_LEN);
74
75 if (tmp != null) {
76 hfileContext.setIncludesTags(true);
77 tmp = fileInfo.get(FileInfo.TAGS_COMPRESSED);
78 if (tmp != null && Bytes.toBoolean(tmp)) {
79 hfileContext.setCompressTags(true);
80 }
81 }
82 }
83
84 @Override
85 protected HFileContext createHFileContext(FSDataInputStreamWrapper fsdis, long fileSize,
86 HFileSystem hfs, Path path, FixedFileTrailer trailer) throws IOException {
87 trailer.expectMajorVersion(3);
88 HFileContextBuilder builder = new HFileContextBuilder()
89 .withIncludesMvcc(this.includesMemstoreTS)
90 .withHBaseCheckSum(true)
91 .withCompression(this.compressAlgo);
92
93
94 byte[] keyBytes = trailer.getEncryptionKey();
95 if (keyBytes != null) {
96 Encryption.Context cryptoContext = Encryption.newContext(conf);
97 Key key;
98 String masterKeyName = conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY,
99 User.getCurrent().getShortName());
100 try {
101
102 key = EncryptionUtil.unwrapKey(conf, masterKeyName, keyBytes);
103 } catch (KeyException e) {
104
105
106 if (LOG.isDebugEnabled()) {
107 LOG.debug("Unable to unwrap key with current master key '" + masterKeyName + "'");
108 }
109 String alternateKeyName =
110 conf.get(HConstants.CRYPTO_MASTERKEY_ALTERNATE_NAME_CONF_KEY);
111 if (alternateKeyName != null) {
112 try {
113 key = EncryptionUtil.unwrapKey(conf, alternateKeyName, keyBytes);
114 } catch (KeyException ex) {
115 throw new IOException(ex);
116 }
117 } else {
118 throw new IOException(e);
119 }
120 }
121
122 Cipher cipher = Encryption.getCipher(conf, key.getAlgorithm());
123 if (cipher == null) {
124 throw new IOException("Cipher '" + key.getAlgorithm() + "' is not available");
125 }
126 cryptoContext.setCipher(cipher);
127 cryptoContext.setKey(key);
128 builder.withEncryptionContext(cryptoContext);
129 }
130
131 HFileContext context = builder.build();
132
133 if (LOG.isTraceEnabled()) {
134 LOG.trace("Reader" + (path != null ? " for " + path : "" ) +
135 " initialized with cacheConf: " + cacheConf +
136 " comparator: " + comparator.getClass().getSimpleName() +
137 " fileContext: " + context);
138 }
139
140 return context;
141 }
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157 @Override
158 public HFileScanner getScanner(boolean cacheBlocks, final boolean pread,
159 final boolean isCompaction) {
160 if (dataBlockEncoder.useEncodedScanner()) {
161 return new EncodedScannerV3(this, cacheBlocks, pread, isCompaction, this.hfileContext);
162 }
163 return new ScannerV3(this, cacheBlocks, pread, isCompaction);
164 }
165
166
167
168
169 protected static class ScannerV3 extends ScannerV2 {
170
171 private HFileReaderV3 reader;
172 private int currTagsLen;
173
174 public ScannerV3(HFileReaderV3 r, boolean cacheBlocks, final boolean pread,
175 final boolean isCompaction) {
176 super(r, cacheBlocks, pread, isCompaction);
177 this.reader = r;
178 }
179
180 @Override
181 protected int getCellBufSize() {
182 int kvBufSize = super.getCellBufSize();
183 if (reader.hfileContext.isIncludesTags()) {
184 kvBufSize += Bytes.SIZEOF_SHORT + currTagsLen;
185 }
186 return kvBufSize;
187 }
188
189 protected void setNonSeekedState() {
190 super.setNonSeekedState();
191 currTagsLen = 0;
192 }
193
194 @Override
195 protected int getNextCellStartPosition() {
196 int nextKvPos = super.getNextCellStartPosition();
197 if (reader.hfileContext.isIncludesTags()) {
198 nextKvPos += Bytes.SIZEOF_SHORT + currTagsLen;
199 }
200 return nextKvPos;
201 }
202
203 protected void readKeyValueLen() {
204 blockBuffer.mark();
205 currKeyLen = blockBuffer.getInt();
206 currValueLen = blockBuffer.getInt();
207 if (currKeyLen < 0 || currValueLen < 0 || currKeyLen > blockBuffer.limit()
208 || currValueLen > blockBuffer.limit()) {
209 throw new IllegalStateException("Invalid currKeyLen " + currKeyLen + " or currValueLen "
210 + currValueLen + ". Block offset: "
211 + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
212 + blockBuffer.position() + " (without header).");
213 }
214 ByteBufferUtils.skip(blockBuffer, currKeyLen + currValueLen);
215 if (reader.hfileContext.isIncludesTags()) {
216
217 currTagsLen = ((blockBuffer.get() & 0xff) << 8) ^ (blockBuffer.get() & 0xff);
218 if (currTagsLen < 0 || currTagsLen > blockBuffer.limit()) {
219 throw new IllegalStateException("Invalid currTagsLen " + currTagsLen + ". Block offset: "
220 + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
221 + blockBuffer.position() + " (without header).");
222 }
223 ByteBufferUtils.skip(blockBuffer, currTagsLen);
224 }
225 readMvccVersion();
226 blockBuffer.reset();
227 }
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248 protected int blockSeek(byte[] key, int offset, int length, boolean seekBefore) {
249 int klen, vlen, tlen = 0;
250 long memstoreTS = 0;
251 int memstoreTSLen = 0;
252 int lastKeyValueSize = -1;
253 do {
254 blockBuffer.mark();
255 klen = blockBuffer.getInt();
256 vlen = blockBuffer.getInt();
257 if (klen < 0 || vlen < 0 || klen > blockBuffer.limit()
258 || vlen > blockBuffer.limit()) {
259 throw new IllegalStateException("Invalid klen " + klen + " or vlen "
260 + vlen + ". Block offset: "
261 + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
262 + blockBuffer.position() + " (without header).");
263 }
264 ByteBufferUtils.skip(blockBuffer, klen + vlen);
265 if (reader.hfileContext.isIncludesTags()) {
266
267 tlen = ((blockBuffer.get() & 0xff) << 8) ^ (blockBuffer.get() & 0xff);
268 if (tlen < 0 || tlen > blockBuffer.limit()) {
269 throw new IllegalStateException("Invalid tlen " + tlen + ". Block offset: "
270 + block.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
271 + blockBuffer.position() + " (without header).");
272 }
273 ByteBufferUtils.skip(blockBuffer, tlen);
274 }
275 if (this.reader.shouldIncludeMemstoreTS()) {
276 if (this.reader.decodeMemstoreTS) {
277 try {
278 memstoreTS = Bytes.readVLong(blockBuffer.array(), blockBuffer.arrayOffset()
279 + blockBuffer.position());
280 memstoreTSLen = WritableUtils.getVIntSize(memstoreTS);
281 } catch (Exception e) {
282 throw new RuntimeException("Error reading memstore timestamp", e);
283 }
284 } else {
285 memstoreTS = 0;
286 memstoreTSLen = 1;
287 }
288 }
289 blockBuffer.reset();
290 int keyOffset = blockBuffer.arrayOffset() + blockBuffer.position() + (Bytes.SIZEOF_INT * 2);
291 int comp = reader.getComparator().compare(key, offset, length, blockBuffer.array(),
292 keyOffset, klen);
293
294 if (comp == 0) {
295 if (seekBefore) {
296 if (lastKeyValueSize < 0) {
297 throw new IllegalStateException("blockSeek with seekBefore "
298 + "at the first key of the block: key=" + Bytes.toStringBinary(key)
299 + ", blockOffset=" + block.getOffset() + ", onDiskSize="
300 + block.getOnDiskSizeWithHeader());
301 }
302 blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
303 readKeyValueLen();
304 return 1;
305 }
306 currKeyLen = klen;
307 currValueLen = vlen;
308 currTagsLen = tlen;
309 if (this.reader.shouldIncludeMemstoreTS()) {
310 currMemstoreTS = memstoreTS;
311 currMemstoreTSLen = memstoreTSLen;
312 }
313 return 0;
314 } else if (comp < 0) {
315 if (lastKeyValueSize > 0)
316 blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
317 readKeyValueLen();
318 if (lastKeyValueSize == -1 && blockBuffer.position() == 0) {
319 return HConstants.INDEX_KEY_MAGIC;
320 }
321 return 1;
322 }
323
324
325 lastKeyValueSize = klen + vlen + memstoreTSLen + KEY_VALUE_LEN_SIZE;
326
327 if (reader.hfileContext.isIncludesTags()) {
328 lastKeyValueSize += tlen + Bytes.SIZEOF_SHORT;
329 }
330 blockBuffer.position(blockBuffer.position() + lastKeyValueSize);
331 } while (blockBuffer.remaining() > 0);
332
333
334
335
336 blockBuffer.position(blockBuffer.position() - lastKeyValueSize);
337 readKeyValueLen();
338 return 1;
339 }
340
341 }
342
343
344
345
346 protected static class EncodedScannerV3 extends EncodedScannerV2 {
347 public EncodedScannerV3(HFileReaderV3 reader, boolean cacheBlocks, boolean pread,
348 boolean isCompaction, HFileContext context) {
349 super(reader, cacheBlocks, pread, isCompaction, context);
350 }
351 }
352
353 @Override
354 public int getMajorVersion() {
355 return 3;
356 }
357 }