1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.codec.prefixtree;
20
21 import java.io.DataInputStream;
22 import java.io.DataOutputStream;
23 import java.io.IOException;
24 import java.nio.ByteBuffer;
25
26 import org.apache.hadoop.hbase.classification.InterfaceAudience;
27 import org.apache.hadoop.hbase.Cell;
28 import org.apache.hadoop.hbase.KeyValue;
29 import org.apache.hadoop.hbase.KeyValue.KVComparator;
30 import org.apache.hadoop.hbase.KeyValue.MetaComparator;
31 import org.apache.hadoop.hbase.KeyValue.RawBytesComparator;
32 import org.apache.hadoop.hbase.KeyValueUtil;
33 import org.apache.hadoop.hbase.codec.prefixtree.decode.DecoderFactory;
34 import org.apache.hadoop.hbase.codec.prefixtree.decode.PrefixTreeArraySearcher;
35 import org.apache.hadoop.hbase.codec.prefixtree.encode.EncoderFactory;
36 import org.apache.hadoop.hbase.codec.prefixtree.encode.PrefixTreeEncoder;
37 import org.apache.hadoop.hbase.codec.prefixtree.scanner.CellSearcher;
38 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
39 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
40 import org.apache.hadoop.hbase.io.encoding.EncodingState;
41 import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext;
42 import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultDecodingContext;
43 import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext;
44 import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext;
45 import org.apache.hadoop.hbase.io.hfile.BlockType;
46 import org.apache.hadoop.hbase.io.hfile.HFileContext;
47 import org.apache.hadoop.hbase.util.ByteBufferUtils;
48 import org.apache.hadoop.io.WritableUtils;
49
50
51
52
53
54
55
56
57
58
59
60
61 @InterfaceAudience.Private
62 public class PrefixTreeCodec implements DataBlockEncoder{
63
64
65
66
67 public PrefixTreeCodec() {
68 }
69
70 @Override
71 public ByteBuffer decodeKeyValues(DataInputStream source, HFileBlockDecodingContext decodingCtx)
72 throws IOException {
73 return decodeKeyValues(source, 0, 0, decodingCtx);
74 }
75
76
77
78
79
80
81 public ByteBuffer decodeKeyValues(DataInputStream source, int allocateHeaderLength,
82 int skipLastBytes, HFileBlockDecodingContext decodingCtx) throws IOException {
83 ByteBuffer sourceAsBuffer = ByteBufferUtils.drainInputStreamToBuffer(source);
84 sourceAsBuffer.mark();
85 PrefixTreeBlockMeta blockMeta = new PrefixTreeBlockMeta(sourceAsBuffer);
86 sourceAsBuffer.rewind();
87 int numV1BytesWithHeader = allocateHeaderLength + blockMeta.getNumKeyValueBytes();
88 byte[] keyValueBytesWithHeader = new byte[numV1BytesWithHeader];
89 ByteBuffer result = ByteBuffer.wrap(keyValueBytesWithHeader);
90 result.rewind();
91 CellSearcher searcher = null;
92 try {
93 boolean includesMvcc = decodingCtx.getHFileContext().isIncludesMvcc();
94 searcher = DecoderFactory.checkOut(sourceAsBuffer, includesMvcc);
95 while (searcher.advance()) {
96 KeyValue currentCell = KeyValueUtil.copyToNewKeyValue(searcher.current());
97
98
99 int offset = result.arrayOffset() + result.position();
100 System.arraycopy(currentCell.getBuffer(), currentCell.getOffset(), result.array(), offset,
101 currentCell.getLength());
102 int keyValueLength = KeyValueUtil.length(currentCell);
103 ByteBufferUtils.skip(result, keyValueLength);
104 offset += keyValueLength;
105 if (includesMvcc) {
106 ByteBufferUtils.writeVLong(result, currentCell.getMvccVersion());
107 }
108 }
109 result.position(result.limit());
110 return result;
111 } finally {
112 DecoderFactory.checkIn(searcher);
113 }
114 }
115
116
117 @Override
118 public ByteBuffer getFirstKeyInBlock(ByteBuffer block) {
119 block.rewind();
120 PrefixTreeArraySearcher searcher = null;
121 try {
122
123 searcher = DecoderFactory.checkOut(block, true);
124 if (!searcher.positionAtFirstCell()) {
125 return null;
126 }
127 return KeyValueUtil.copyKeyToNewByteBuffer(searcher.current());
128 } finally {
129 DecoderFactory.checkIn(searcher);
130 }
131 }
132
133 @Override
134 public HFileBlockEncodingContext newDataBlockEncodingContext(
135 DataBlockEncoding encoding, byte[] header, HFileContext meta) {
136 if(DataBlockEncoding.PREFIX_TREE != encoding){
137
138
139 throw new IllegalArgumentException("only DataBlockEncoding.PREFIX_TREE supported");
140 }
141 return new HFileBlockDefaultEncodingContext(encoding, header, meta);
142 }
143
144 @Override
145 public HFileBlockDecodingContext newDataBlockDecodingContext(HFileContext meta) {
146 return new HFileBlockDefaultDecodingContext(meta);
147 }
148
149
150
151
152
153 @Override
154 public EncodedSeeker createSeeker(KVComparator comparator, HFileBlockDecodingContext decodingCtx) {
155 if (comparator instanceof RawBytesComparator){
156 throw new IllegalArgumentException("comparator must be KeyValue.KeyComparator");
157 } else if (comparator instanceof MetaComparator){
158 throw new IllegalArgumentException("DataBlockEncoding.PREFIX_TREE not compatible with hbase:meta "
159 +"table");
160 }
161
162 return new PrefixTreeSeeker(decodingCtx.getHFileContext().isIncludesMvcc());
163 }
164
165 @Override
166 public int encode(Cell cell, HFileBlockEncodingContext encodingCtx, DataOutputStream out)
167 throws IOException {
168 PrefixTreeEncodingState state = (PrefixTreeEncodingState) encodingCtx.getEncodingState();
169 PrefixTreeEncoder builder = state.builder;
170 builder.write(cell);
171 int size = KeyValueUtil.length(cell);
172 if (encodingCtx.getHFileContext().isIncludesMvcc()) {
173 size += WritableUtils.getVIntSize(cell.getSequenceId());
174 }
175 return size;
176 }
177
178 private static class PrefixTreeEncodingState extends EncodingState {
179 PrefixTreeEncoder builder = null;
180 }
181
182 @Override
183 public void startBlockEncoding(HFileBlockEncodingContext blkEncodingCtx, DataOutputStream out)
184 throws IOException {
185 if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) {
186 throw new IOException(this.getClass().getName() + " only accepts "
187 + HFileBlockDefaultEncodingContext.class.getName() + " as the " + "encoding context.");
188 }
189
190 HFileBlockDefaultEncodingContext encodingCtx =
191 (HFileBlockDefaultEncodingContext) blkEncodingCtx;
192 encodingCtx.prepareEncoding(out);
193
194 PrefixTreeEncoder builder = EncoderFactory.checkOut(out, encodingCtx.getHFileContext()
195 .isIncludesMvcc());
196 PrefixTreeEncodingState state = new PrefixTreeEncodingState();
197 state.builder = builder;
198 blkEncodingCtx.setEncodingState(state);
199 }
200
201 @Override
202 public void endBlockEncoding(HFileBlockEncodingContext encodingCtx, DataOutputStream out,
203 byte[] uncompressedBytesWithHeader) throws IOException {
204 PrefixTreeEncodingState state = (PrefixTreeEncodingState) encodingCtx.getEncodingState();
205 PrefixTreeEncoder builder = state.builder;
206 builder.flush();
207 EncoderFactory.checkIn(builder);
208
209 if (encodingCtx.getDataBlockEncoding() != DataBlockEncoding.NONE) {
210 encodingCtx.postEncoding(BlockType.ENCODED_DATA);
211 } else {
212 encodingCtx.postEncoding(BlockType.DATA);
213 }
214 }
215 }