1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.io;
21
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertNull;
24 import static org.junit.Assert.assertTrue;
25
26 import java.io.IOException;
27 import java.util.ArrayList;
28 import java.util.List;
29
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.fs.FileSystem;
32 import org.apache.hadoop.fs.Path;
33 import org.apache.hadoop.hbase.HBaseTestingUtility;
34 import org.apache.hadoop.hbase.KeyValue;
35 import org.apache.hadoop.hbase.SmallTests;
36 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
37 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
38 import org.apache.hadoop.hbase.io.hfile.HFile;
39 import org.apache.hadoop.hbase.io.hfile.HFileContext;
40 import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
41 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
42 import org.apache.hadoop.hbase.util.Bytes;
43 import org.junit.AfterClass;
44 import org.junit.BeforeClass;
45 import org.junit.Test;
46 import org.junit.experimental.categories.Category;
47
48 @Category(SmallTests.class)
49 public class TestHalfStoreFileReader {
50 private static HBaseTestingUtility TEST_UTIL;
51
52 @BeforeClass
53 public static void setupBeforeClass() throws Exception {
54 TEST_UTIL = new HBaseTestingUtility();
55 }
56
57 @AfterClass
58 public static void tearDownAfterClass() throws Exception {
59 TEST_UTIL.cleanupTestDir();
60 }
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78 @Test
79 public void testHalfScanAndReseek() throws IOException {
80 String root_dir = TEST_UTIL.getDataTestDir().toString();
81 Path p = new Path(root_dir, "test");
82
83 Configuration conf = TEST_UTIL.getConfiguration();
84 FileSystem fs = FileSystem.get(conf);
85 CacheConfig cacheConf = new CacheConfig(conf);
86 HFileContext meta = new HFileContextBuilder().withBlockSize(1024).build();
87 HFile.Writer w = HFile.getWriterFactory(conf, cacheConf)
88 .withPath(fs, p)
89 .withFileContext(meta)
90 .create();
91
92
93 List<KeyValue> items = genSomeKeys();
94 for (KeyValue kv : items) {
95 w.append(kv);
96 }
97 w.close();
98
99 HFile.Reader r = HFile.createReader(fs, p, cacheConf, conf);
100 r.loadFileInfo();
101 byte [] midkey = r.midkey();
102 KeyValue midKV = KeyValue.createKeyValueFromKey(midkey);
103 midkey = midKV.getRow();
104
105
106
107 Reference bottom = new Reference(midkey, Reference.Range.bottom);
108 doTestOfScanAndReseek(p, fs, bottom, cacheConf);
109
110 Reference top = new Reference(midkey, Reference.Range.top);
111 doTestOfScanAndReseek(p, fs, top, cacheConf);
112
113 r.close();
114 }
115
116 private void doTestOfScanAndReseek(Path p, FileSystem fs, Reference bottom,
117 CacheConfig cacheConf)
118 throws IOException {
119 final HalfStoreFileReader halfreader = new HalfStoreFileReader(fs, p,
120 cacheConf, bottom, TEST_UTIL.getConfiguration());
121 halfreader.loadFileInfo();
122 final HFileScanner scanner = halfreader.getScanner(false, false);
123
124 scanner.seekTo();
125 KeyValue curr;
126 do {
127 curr = scanner.getKeyValue();
128 KeyValue reseekKv =
129 getLastOnCol(curr);
130 int ret = scanner.reseekTo(reseekKv.getKey());
131 assertTrue("reseek to returned: " + ret, ret > 0);
132
133 } while (scanner.next());
134
135 int ret = scanner.reseekTo(getLastOnCol(curr).getKey());
136
137 assertTrue( ret > 0 );
138
139 halfreader.close(true);
140 }
141
142
143
144 @Test
145 public void testHalfScanner() throws IOException {
146 String root_dir = TEST_UTIL.getDataTestDir().toString();
147 Path p = new Path(root_dir, "test");
148 Configuration conf = TEST_UTIL.getConfiguration();
149 FileSystem fs = FileSystem.get(conf);
150 CacheConfig cacheConf = new CacheConfig(conf);
151 HFileContext meta = new HFileContextBuilder().withBlockSize(1024).build();
152 HFile.Writer w = HFile.getWriterFactory(conf, cacheConf)
153 .withPath(fs, p)
154 .withFileContext(meta)
155 .create();
156
157
158 List<KeyValue> items = genSomeKeys();
159 for (KeyValue kv : items) {
160 w.append(kv);
161 }
162 w.close();
163
164
165 HFile.Reader r = HFile.createReader(fs, p, cacheConf, conf);
166 r.loadFileInfo();
167 byte[] midkey = r.midkey();
168 KeyValue midKV = KeyValue.createKeyValueFromKey(midkey);
169 midkey = midKV.getRow();
170
171 Reference bottom = new Reference(midkey, Reference.Range.bottom);
172 Reference top = new Reference(midkey, Reference.Range.top);
173
174
175 KeyValue beforeMidKey = null;
176 for (KeyValue item : items) {
177 if (KeyValue.COMPARATOR.compare(item, midKV) >= 0) {
178 break;
179 }
180 beforeMidKey = item;
181 }
182 System.out.println("midkey: " + midKV + " or: " + Bytes.toStringBinary(midkey));
183 System.out.println("beforeMidKey: " + beforeMidKey);
184
185
186
187 KeyValue foundKeyValue = doTestOfSeekBefore(p, fs, bottom, midKV, cacheConf);
188 assertEquals(beforeMidKey, foundKeyValue);
189
190
191 foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(items.size() - 1), cacheConf);
192 assertEquals(items.get(items.size() - 2), foundKeyValue);
193
194 foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(items.size() - 1), cacheConf);
195 assertEquals(beforeMidKey, foundKeyValue);
196
197
198 foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(0), cacheConf);
199 assertNull(foundKeyValue);
200
201
202 foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(0), cacheConf);
203 assertNull(foundKeyValue);
204
205
206 foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(1), cacheConf);
207 assertNull(foundKeyValue);
208
209 foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(1), cacheConf);
210 assertEquals(items.get(0), foundKeyValue);
211
212
213 foundKeyValue = doTestOfSeekBefore(p, fs, top, midKV, cacheConf);
214 assertNull(foundKeyValue);
215 }
216
217 private KeyValue doTestOfSeekBefore(Path p, FileSystem fs, Reference bottom, KeyValue seekBefore,
218 CacheConfig cacheConfig)
219 throws IOException {
220 final HalfStoreFileReader halfreader = new HalfStoreFileReader(fs, p,
221 cacheConfig, bottom, TEST_UTIL.getConfiguration());
222 halfreader.loadFileInfo();
223 final HFileScanner scanner = halfreader.getScanner(false, false);
224 scanner.seekBefore(seekBefore.getKey());
225 return scanner.getKeyValue();
226 }
227
228 private KeyValue getLastOnCol(KeyValue curr) {
229 return KeyValue.createLastOnRow(
230 curr.getBuffer(), curr.getRowOffset(), curr.getRowLength(),
231 curr.getBuffer(), curr.getFamilyOffset(), curr.getFamilyLength(),
232 curr.getBuffer(), curr.getQualifierOffset(), curr.getQualifierLength());
233 }
234
235 static final int SIZE = 1000;
236
237 static byte[] _b(String s) {
238 return Bytes.toBytes(s);
239 }
240
241 List<KeyValue> genSomeKeys() {
242 List<KeyValue> ret = new ArrayList<KeyValue>(SIZE);
243 for (int i = 0; i < SIZE; i++) {
244 KeyValue kv =
245 new KeyValue(
246 _b(String.format("row_%04d", i)),
247 _b("family"),
248 _b("qualifier"),
249 1000,
250 _b("value"));
251 ret.add(kv);
252 }
253 return ret;
254 }
255
256
257
258 }
259