1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.io.hfile;
20
21 import java.io.DataInput;
22 import java.io.DataOutput;
23 import java.io.IOException;
24 import java.nio.ByteBuffer;
25 import java.util.Arrays;
26 import java.util.Map;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.fs.FSDataInputStream;
31 import org.apache.hadoop.fs.FSDataOutputStream;
32 import org.apache.hadoop.fs.FileStatus;
33 import org.apache.hadoop.fs.FileSystem;
34 import org.apache.hadoop.fs.Path;
35 import org.apache.hadoop.hbase.HBaseTestCase;
36 import org.apache.hadoop.hbase.HBaseTestingUtility;
37 import org.apache.hadoop.hbase.HConstants;
38 import org.apache.hadoop.hbase.KeyValue;
39 import org.apache.hadoop.hbase.KeyValue.Type;
40 import org.apache.hadoop.hbase.testclassification.SmallTests;
41 import org.apache.hadoop.hbase.Tag;
42 import org.apache.hadoop.hbase.io.compress.Compression;
43 import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
44 import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
45 import org.apache.hadoop.hbase.util.Bytes;
46 import org.apache.hadoop.io.Writable;
47 import org.junit.experimental.categories.Category;
48
49
50
51
52
53
54
55
56
57 @Category(SmallTests.class)
58 public class TestHFile extends HBaseTestCase {
59 private static final Log LOG = LogFactory.getLog(TestHFile.class);
60
61 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
62 private static String ROOT_DIR =
63 TEST_UTIL.getDataTestDir("TestHFile").toString();
64 private final int minBlockSize = 512;
65 private static String localFormatter = "%010d";
66 private static CacheConfig cacheConf = null;
67 private Map<String, Long> startingMetrics;
68
69 @Override
70 public void setUp() throws Exception {
71 super.setUp();
72 }
73
74 @Override
75 public void tearDown() throws Exception {
76 super.tearDown();
77 }
78
79
80
81
82
83
84
85 public void testEmptyHFile() throws IOException {
86 if (cacheConf == null) cacheConf = new CacheConfig(conf);
87 Path f = new Path(ROOT_DIR, getName());
88 HFileContext context = new HFileContextBuilder().withIncludesTags(false).build();
89 Writer w =
90 HFile.getWriterFactory(conf, cacheConf).withPath(fs, f).withFileContext(context).create();
91 w.close();
92 Reader r = HFile.createReader(fs, f, cacheConf, conf);
93 r.loadFileInfo();
94 assertNull(r.getFirstKey());
95 assertNull(r.getLastKey());
96 }
97
98
99
100
101 public void testCorrupt0LengthHFile() throws IOException {
102 if (cacheConf == null) cacheConf = new CacheConfig(conf);
103 Path f = new Path(ROOT_DIR, getName());
104 FSDataOutputStream fsos = fs.create(f);
105 fsos.close();
106
107 try {
108 Reader r = HFile.createReader(fs, f, cacheConf, conf);
109 } catch (CorruptHFileException che) {
110
111 return;
112 }
113 fail("Should have thrown exception");
114 }
115
116 public static void truncateFile(FileSystem fs, Path src, Path dst) throws IOException {
117 FileStatus fst = fs.getFileStatus(src);
118 long len = fst.getLen();
119 len = len / 2 ;
120
121
122 FSDataOutputStream fdos = fs.create(dst);
123 byte[] buf = new byte[(int)len];
124 FSDataInputStream fdis = fs.open(src);
125 fdis.read(buf);
126 fdos.write(buf);
127 fdis.close();
128 fdos.close();
129 }
130
131
132
133
134 public void testCorruptTruncatedHFile() throws IOException {
135 if (cacheConf == null) cacheConf = new CacheConfig(conf);
136 Path f = new Path(ROOT_DIR, getName());
137 HFileContext context = new HFileContextBuilder().build();
138 Writer w = HFile.getWriterFactory(conf, cacheConf).withPath(this.fs, f)
139 .withFileContext(context).create();
140 writeSomeRecords(w, 0, 100, false);
141 w.close();
142
143 Path trunc = new Path(f.getParent(), "trucated");
144 truncateFile(fs, w.getPath(), trunc);
145
146 try {
147 Reader r = HFile.createReader(fs, trunc, cacheConf, conf);
148 } catch (CorruptHFileException che) {
149
150 return;
151 }
152 fail("Should have thrown exception");
153 }
154
155
156
157 private int writeSomeRecords(Writer writer, int start, int n, boolean useTags)
158 throws IOException {
159 String value = "value";
160 KeyValue kv;
161 for (int i = start; i < (start + n); i++) {
162 String key = String.format(localFormatter, Integer.valueOf(i));
163 if (useTags) {
164 Tag t = new Tag((byte) 1, "myTag1");
165 Tag[] tags = new Tag[1];
166 tags[0] = t;
167 kv = new KeyValue(Bytes.toBytes(key), Bytes.toBytes("family"), Bytes.toBytes("qual"),
168 HConstants.LATEST_TIMESTAMP, Bytes.toBytes(value + key), tags);
169 writer.append(kv);
170 } else {
171 kv = new KeyValue(Bytes.toBytes(key), Bytes.toBytes("family"), Bytes.toBytes("qual"),
172 Bytes.toBytes(value + key));
173 writer.append(kv);
174 }
175 }
176 return (start + n);
177 }
178
179 private void readAllRecords(HFileScanner scanner) throws IOException {
180 readAndCheckbytes(scanner, 0, 100);
181 }
182
183
184 private int readAndCheckbytes(HFileScanner scanner, int start, int n)
185 throws IOException {
186 String value = "value";
187 int i = start;
188 for (; i < (start + n); i++) {
189 ByteBuffer key = scanner.getKey();
190 ByteBuffer val = scanner.getValue();
191 String keyStr = String.format(localFormatter, Integer.valueOf(i));
192 String valStr = value + keyStr;
193 KeyValue kv = new KeyValue(Bytes.toBytes(keyStr), Bytes.toBytes("family"),
194 Bytes.toBytes("qual"), Bytes.toBytes(valStr));
195 byte[] keyBytes = new KeyValue.KeyOnlyKeyValue(Bytes.toBytes(key), 0,
196 Bytes.toBytes(key).length).getKey();
197 assertTrue("bytes for keys do not match " + keyStr + " " +
198 Bytes.toString(Bytes.toBytes(key)),
199 Arrays.equals(kv.getKey(), keyBytes));
200 byte [] valBytes = Bytes.toBytes(val);
201 assertTrue("bytes for vals do not match " + valStr + " " +
202 Bytes.toString(valBytes),
203 Arrays.equals(Bytes.toBytes(valStr), valBytes));
204 if (!scanner.next()) {
205 break;
206 }
207 }
208 assertEquals(i, start + n - 1);
209 return (start + n);
210 }
211
212 private byte[] getSomeKey(int rowId) {
213 KeyValue kv = new KeyValue(String.format(localFormatter, Integer.valueOf(rowId)).getBytes(),
214 Bytes.toBytes("family"), Bytes.toBytes("qual"), HConstants.LATEST_TIMESTAMP, Type.Put);
215 return kv.getKey();
216 }
217
218 private void writeRecords(Writer writer, boolean useTags) throws IOException {
219 writeSomeRecords(writer, 0, 100, useTags);
220 writer.close();
221 }
222
223 private FSDataOutputStream createFSOutput(Path name) throws IOException {
224
225 FSDataOutputStream fout = fs.create(name);
226 return fout;
227 }
228
229
230
231
232
233 void basicWithSomeCodec(String codec, boolean useTags) throws IOException {
234 if (useTags) {
235 conf.setInt("hfile.format.version", 3);
236 }
237 if (cacheConf == null) cacheConf = new CacheConfig(conf);
238 Path ncTFile = new Path(ROOT_DIR, "basic.hfile." + codec.toString() + useTags);
239 FSDataOutputStream fout = createFSOutput(ncTFile);
240 HFileContext meta = new HFileContextBuilder()
241 .withBlockSize(minBlockSize)
242 .withCompression(AbstractHFileWriter.compressionByName(codec))
243 .build();
244 Writer writer = HFile.getWriterFactory(conf, cacheConf)
245 .withOutputStream(fout)
246 .withFileContext(meta)
247 .withComparator(new KeyValue.KVComparator())
248 .create();
249 LOG.info(writer);
250 writeRecords(writer, useTags);
251 fout.close();
252 FSDataInputStream fin = fs.open(ncTFile);
253 Reader reader = HFile.createReaderFromStream(ncTFile, fs.open(ncTFile),
254 fs.getFileStatus(ncTFile).getLen(), cacheConf, conf);
255 System.out.println(cacheConf.toString());
256
257 reader.loadFileInfo();
258
259 HFileScanner scanner = reader.getScanner(true, false);
260
261 scanner.seekTo();
262 readAllRecords(scanner);
263 int seekTo = scanner.seekTo(KeyValue.createKeyValueFromKey(getSomeKey(50)));
264 System.out.println(seekTo);
265 assertTrue("location lookup failed",
266 scanner.seekTo(KeyValue.createKeyValueFromKey(getSomeKey(50))) == 0);
267
268 ByteBuffer readKey = scanner.getKey();
269 assertTrue("seeked key does not match", Arrays.equals(getSomeKey(50),
270 Bytes.toBytes(readKey)));
271
272 scanner.seekTo(KeyValue.createKeyValueFromKey(getSomeKey(0)));
273 ByteBuffer val1 = scanner.getValue();
274 scanner.seekTo(KeyValue.createKeyValueFromKey(getSomeKey(0)));
275 ByteBuffer val2 = scanner.getValue();
276 assertTrue(Arrays.equals(Bytes.toBytes(val1), Bytes.toBytes(val2)));
277
278 reader.close();
279 fin.close();
280 fs.delete(ncTFile, true);
281 }
282
283 public void testTFileFeatures() throws IOException {
284 testTFilefeaturesInternals(false);
285 testTFilefeaturesInternals(true);
286 }
287
288 protected void testTFilefeaturesInternals(boolean useTags) throws IOException {
289 basicWithSomeCodec("none", useTags);
290 basicWithSomeCodec("gz", useTags);
291 }
292
293 private void writeNumMetablocks(Writer writer, int n) {
294 for (int i = 0; i < n; i++) {
295 writer.appendMetaBlock("HFileMeta" + i, new Writable() {
296 private int val;
297 public Writable setVal(int val) { this.val = val; return this; }
298
299 @Override
300 public void write(DataOutput out) throws IOException {
301 out.write(("something to test" + val).getBytes());
302 }
303
304 @Override
305 public void readFields(DataInput in) throws IOException { }
306 }.setVal(i));
307 }
308 }
309
310 private void someTestingWithMetaBlock(Writer writer) {
311 writeNumMetablocks(writer, 10);
312 }
313
314 private void readNumMetablocks(Reader reader, int n) throws IOException {
315 for (int i = 0; i < n; i++) {
316 ByteBuffer actual = reader.getMetaBlock("HFileMeta" + i, false);
317 ByteBuffer expected =
318 ByteBuffer.wrap(("something to test" + i).getBytes());
319 assertEquals("failed to match metadata",
320 Bytes.toStringBinary(expected), Bytes.toStringBinary(actual));
321 }
322 }
323
324 private void someReadingWithMetaBlock(Reader reader) throws IOException {
325 readNumMetablocks(reader, 10);
326 }
327
328 private void metablocks(final String compress) throws Exception {
329 if (cacheConf == null) cacheConf = new CacheConfig(conf);
330 Path mFile = new Path(ROOT_DIR, "meta.hfile");
331 FSDataOutputStream fout = createFSOutput(mFile);
332 HFileContext meta = new HFileContextBuilder()
333 .withCompression(AbstractHFileWriter.compressionByName(compress))
334 .withBlockSize(minBlockSize).build();
335 Writer writer = HFile.getWriterFactory(conf, cacheConf)
336 .withOutputStream(fout)
337 .withFileContext(meta)
338 .create();
339 someTestingWithMetaBlock(writer);
340 writer.close();
341 fout.close();
342 FSDataInputStream fin = fs.open(mFile);
343 Reader reader = HFile.createReaderFromStream(mFile, fs.open(mFile),
344 this.fs.getFileStatus(mFile).getLen(), cacheConf, conf);
345 reader.loadFileInfo();
346
347 assertFalse(reader.getScanner(false, false).seekTo());
348 someReadingWithMetaBlock(reader);
349 fs.delete(mFile, true);
350 reader.close();
351 fin.close();
352 }
353
354
355 public void testMetaBlocks() throws Exception {
356 metablocks("none");
357 metablocks("gz");
358 }
359
360 public void testNullMetaBlocks() throws Exception {
361 if (cacheConf == null) cacheConf = new CacheConfig(conf);
362 for (Compression.Algorithm compressAlgo :
363 HBaseTestingUtility.COMPRESSION_ALGORITHMS) {
364 Path mFile = new Path(ROOT_DIR, "nometa_" + compressAlgo + ".hfile");
365 FSDataOutputStream fout = createFSOutput(mFile);
366 HFileContext meta = new HFileContextBuilder().withCompression(compressAlgo)
367 .withBlockSize(minBlockSize).build();
368 Writer writer = HFile.getWriterFactory(conf, cacheConf)
369 .withOutputStream(fout)
370 .withFileContext(meta)
371 .create();
372 KeyValue kv = new KeyValue("foo".getBytes(), "f1".getBytes(), null, "value".getBytes());
373 writer.append(kv);
374 writer.close();
375 fout.close();
376 Reader reader = HFile.createReader(fs, mFile, cacheConf, conf);
377 reader.loadFileInfo();
378 assertNull(reader.getMetaBlock("non-existant", false));
379 }
380 }
381
382
383
384
385 public void testCompressionOrdinance() {
386 assertTrue(Compression.Algorithm.LZO.ordinal() == 0);
387 assertTrue(Compression.Algorithm.GZ.ordinal() == 1);
388 assertTrue(Compression.Algorithm.NONE.ordinal() == 2);
389 assertTrue(Compression.Algorithm.SNAPPY.ordinal() == 3);
390 assertTrue(Compression.Algorithm.LZ4.ordinal() == 4);
391 }
392
393 }
394