1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.io.hfile;
21
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertTrue;
24
25 import java.io.IOException;
26 import java.util.Random;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.fs.FileSystem;
32 import org.apache.hadoop.fs.Path;
33 import org.apache.hadoop.hbase.Cell;
34 import org.apache.hadoop.hbase.CellUtil;
35 import org.apache.hadoop.hbase.HBaseTestingUtility;
36 import org.apache.hadoop.hbase.HConstants;
37 import org.apache.hadoop.hbase.KeyValue;
38 import org.apache.hadoop.hbase.fs.HFileSystem;
39 import org.apache.hadoop.hbase.regionserver.BloomType;
40 import org.apache.hadoop.hbase.regionserver.StoreFile;
41 import org.apache.hadoop.hbase.testclassification.MediumTests;
42 import org.apache.hadoop.hbase.util.BloomFilterFactory;
43 import org.apache.hadoop.hbase.util.Bytes;
44 import org.junit.Test;
45 import org.junit.experimental.categories.Category;
46
47 @Category({MediumTests.class})
48 public class TestSeekBeforeWithInlineBlocks {
49
50 private static final Log LOG = LogFactory.getLog(TestSeekBeforeWithInlineBlocks.class);
51
52 private static final HBaseTestingUtility TEST_UTIL =
53 new HBaseTestingUtility();
54
55 private static final int NUM_KV = 10000;
56
57 private static final int DATA_BLOCK_SIZE = 4096;
58 private static final int BLOOM_BLOCK_SIZE = 1024;
59 private static final int[] INDEX_CHUNK_SIZES = { 65536, 4096, 1024 };
60 private static final int[] EXPECTED_NUM_LEVELS = { 1, 2, 3 };
61
62 private static final Random RAND = new Random(192537);
63 private static final byte[] FAM = Bytes.toBytes("family");
64
65 private FileSystem fs;
66 private Configuration conf;
67
68
69
70
71
72
73
74
75
76 @Test
77 public void testMultiIndexLevelRandomHFileWithBlooms() throws IOException {
78 conf = TEST_UTIL.getConfiguration();
79
80
81 for (int hfileVersion = HFile.MIN_FORMAT_VERSION_WITH_TAGS;
82 hfileVersion <= HFile.MAX_FORMAT_VERSION; hfileVersion++) {
83
84 conf.setInt(HFile.FORMAT_VERSION_KEY, hfileVersion);
85 fs = HFileSystem.get(conf);
86
87
88 for (BloomType bloomType : BloomType.values()) {
89
90
91 for (int testI = 0; testI < INDEX_CHUNK_SIZES.length; testI++) {
92 int indexBlockSize = INDEX_CHUNK_SIZES[testI];
93 int expectedNumLevels = EXPECTED_NUM_LEVELS[testI];
94
95 LOG.info(String.format("Testing HFileVersion: %s, BloomType: %s, Index Levels: %s",
96 hfileVersion, bloomType, expectedNumLevels));
97
98 conf.setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, indexBlockSize);
99 conf.setInt(BloomFilterFactory.IO_STOREFILE_BLOOM_BLOCK_SIZE, BLOOM_BLOCK_SIZE);
100
101 Cell[] cells = new Cell[NUM_KV];
102
103 Path hfilePath = new Path(TEST_UTIL.getDataTestDir(),
104 String.format("testMultiIndexLevelRandomHFileWithBlooms-%s-%s-%s",
105 hfileVersion, bloomType, testI));
106
107
108 conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f);
109 CacheConfig cacheConf = new CacheConfig(conf);
110
111
112 {
113 HFileContext meta = new HFileContextBuilder()
114 .withBlockSize(DATA_BLOCK_SIZE)
115 .build();
116
117 StoreFile.Writer storeFileWriter =
118 new StoreFile.WriterBuilder(conf, cacheConf, fs)
119 .withFilePath(hfilePath)
120 .withFileContext(meta)
121 .withBloomType(bloomType)
122 .build();
123
124 for (int i = 0; i < NUM_KV; i++) {
125 byte[] row = TestHFileWriterV2.randomOrderedKey(RAND, i);
126 byte[] qual = TestHFileWriterV2.randomRowOrQualifier(RAND);
127 byte[] value = TestHFileWriterV2.randomValue(RAND);
128 KeyValue kv = new KeyValue(row, FAM, qual, value);
129
130 storeFileWriter.append(kv);
131 cells[i] = kv;
132 }
133
134 storeFileWriter.close();
135 }
136
137
138 HFile.Reader reader = HFile.createReader(fs, hfilePath, cacheConf, conf);
139
140
141 assertEquals(expectedNumLevels, reader.getTrailer().getNumDataIndexLevels());
142
143
144
145 for (boolean pread : new boolean[] { false, true }) {
146 HFileScanner scanner = reader.getScanner(true, pread);
147 checkNoSeekBefore(cells, scanner, 0);
148 for (int i = 1; i < NUM_KV; i++) {
149 checkSeekBefore(cells, scanner, i);
150 checkCell(cells[i-1], scanner.getKeyValue());
151 }
152 assertTrue(scanner.seekTo());
153 for (int i = NUM_KV - 1; i >= 1; i--) {
154 checkSeekBefore(cells, scanner, i);
155 checkCell(cells[i-1], scanner.getKeyValue());
156 }
157 checkNoSeekBefore(cells, scanner, 0);
158 }
159
160 reader.close();
161 }
162 }
163 }
164 }
165
166 private void checkSeekBefore(Cell[] cells, HFileScanner scanner, int i)
167 throws IOException {
168 assertEquals("Failed to seek to the key before #" + i + " ("
169 + CellUtil.getCellKeyAsString(cells[i]) + ")", true,
170 scanner.seekBefore(cells[i]));
171 }
172
173 private void checkNoSeekBefore(Cell[] cells, HFileScanner scanner, int i)
174 throws IOException {
175 assertEquals("Incorrectly succeeded in seeking to before first key ("
176 + CellUtil.getCellKeyAsString(cells[i]) + ")", false,
177 scanner.seekBefore(cells[i]));
178 }
179
180
181 private void checkCell(Cell expected, Cell actual) {
182 assertTrue(String.format("Expected key %s, but was %s",
183 CellUtil.getCellKeyAsString(expected), CellUtil.getCellKeyAsString(actual)),
184 CellUtil.equals(expected, actual));
185 }
186 }
187