View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.io;
21  
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertNull;
24  import static org.junit.Assert.assertTrue;
25  
26  import java.io.IOException;
27  import java.util.ArrayList;
28  import java.util.List;
29  
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.fs.FileSystem;
32  import org.apache.hadoop.fs.Path;
33  import org.apache.hadoop.hbase.HBaseTestingUtility;
34  import org.apache.hadoop.hbase.KeyValue;
35  import org.apache.hadoop.hbase.SmallTests;
36  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
37  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
38  import org.apache.hadoop.hbase.io.hfile.HFile;
39  import org.apache.hadoop.hbase.io.hfile.HFileContext;
40  import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
41  import org.apache.hadoop.hbase.io.hfile.HFileScanner;
42  import org.apache.hadoop.hbase.util.Bytes;
43  import org.junit.AfterClass;
44  import org.junit.BeforeClass;
45  import org.junit.Test;
46  import org.junit.experimental.categories.Category;
47  
48  @Category(SmallTests.class)
49  public class TestHalfStoreFileReader {
50    private static HBaseTestingUtility TEST_UTIL;
51  
52    @BeforeClass
53    public static void setupBeforeClass() throws Exception {
54      TEST_UTIL = new HBaseTestingUtility();
55    }
56  
57    @AfterClass
58    public static void tearDownAfterClass() throws Exception {
59      TEST_UTIL.cleanupTestDir();
60    }
61  
62    /**
63     * Test the scanner and reseek of a half hfile scanner. The scanner API
64     * demands that seekTo and reseekTo() only return < 0 if the key lies
65     * before the start of the file (with no position on the scanner). Returning
66     * 0 if perfect match (rare), and return > 1 if we got an imperfect match.
67     *
68     * The latter case being the most common, we should generally be returning 1,
69     * and if we do, there may or may not be a 'next' in the scanner/file.
70     *
71     * A bug in the half file scanner was returning -1 at the end of the bottom
72     * half, and that was causing the infrastructure above to go null causing NPEs
73     * and other problems.  This test reproduces that failure, and also tests
74     * both the bottom and top of the file while we are at it.
75     *
76     * @throws IOException
77     */
78    @Test
79    public void testHalfScanAndReseek() throws IOException {
80      String root_dir = TEST_UTIL.getDataTestDir().toString();
81      Path p = new Path(root_dir, "test");
82  
83      Configuration conf = TEST_UTIL.getConfiguration();
84      FileSystem fs = FileSystem.get(conf);
85      CacheConfig cacheConf = new CacheConfig(conf);
86      HFileContext meta = new HFileContextBuilder().withBlockSize(1024).build();
87      HFile.Writer w = HFile.getWriterFactory(conf, cacheConf)
88          .withPath(fs, p)
89          .withFileContext(meta)
90          .create();
91  
92      // write some things.
93      List<KeyValue> items = genSomeKeys();
94      for (KeyValue kv : items) {
95        w.append(kv);
96      }
97      w.close();
98  
99      HFile.Reader r = HFile.createReader(fs, p, cacheConf, conf);
100     r.loadFileInfo();
101     byte [] midkey = r.midkey();
102     KeyValue midKV = KeyValue.createKeyValueFromKey(midkey);
103     midkey = midKV.getRow();
104 
105     //System.out.println("midkey: " + midKV + " or: " + Bytes.toStringBinary(midkey));
106 
107     Reference bottom = new Reference(midkey, Reference.Range.bottom);
108     doTestOfScanAndReseek(p, fs, bottom, cacheConf);
109 
110     Reference top = new Reference(midkey, Reference.Range.top);
111     doTestOfScanAndReseek(p, fs, top, cacheConf);
112 
113     r.close();
114   }
115 
116   private void doTestOfScanAndReseek(Path p, FileSystem fs, Reference bottom,
117       CacheConfig cacheConf)
118       throws IOException {
119     final HalfStoreFileReader halfreader = new HalfStoreFileReader(fs, p,
120       cacheConf, bottom, TEST_UTIL.getConfiguration());
121     halfreader.loadFileInfo();
122     final HFileScanner scanner = halfreader.getScanner(false, false);
123 
124     scanner.seekTo();
125     KeyValue curr;
126     do {
127       curr = scanner.getKeyValue();
128       KeyValue reseekKv =
129           getLastOnCol(curr);
130       int ret = scanner.reseekTo(reseekKv.getKey());
131       assertTrue("reseek to returned: " + ret, ret > 0);
132       //System.out.println(curr + ": " + ret);
133     } while (scanner.next());
134 
135     int ret = scanner.reseekTo(getLastOnCol(curr).getKey());
136     //System.out.println("Last reseek: " + ret);
137     assertTrue( ret > 0 );
138 
139     halfreader.close(true);
140   }
141 
142 
143   // Tests the scanner on an HFile that is backed by HalfStoreFiles
144   @Test
145   public void testHalfScanner() throws IOException {
146       String root_dir = TEST_UTIL.getDataTestDir().toString();
147       Path p = new Path(root_dir, "test");
148       Configuration conf = TEST_UTIL.getConfiguration();
149       FileSystem fs = FileSystem.get(conf);
150       CacheConfig cacheConf = new CacheConfig(conf);
151       HFileContext meta = new HFileContextBuilder().withBlockSize(1024).build();
152       HFile.Writer w = HFile.getWriterFactory(conf, cacheConf)
153               .withPath(fs, p)
154               .withFileContext(meta)
155               .create();
156 
157       // write some things.
158       List<KeyValue> items = genSomeKeys();
159       for (KeyValue kv : items) {
160           w.append(kv);
161       }
162       w.close();
163 
164 
165       HFile.Reader r = HFile.createReader(fs, p, cacheConf, conf);
166       r.loadFileInfo();
167       byte[] midkey = r.midkey();
168       KeyValue midKV = KeyValue.createKeyValueFromKey(midkey);
169       midkey = midKV.getRow();
170 
171       Reference bottom = new Reference(midkey, Reference.Range.bottom);
172       Reference top = new Reference(midkey, Reference.Range.top);
173 
174       // Ugly code to get the item before the midkey
175       KeyValue beforeMidKey = null;
176       for (KeyValue item : items) {
177           if (KeyValue.COMPARATOR.compare(item, midKV) >= 0) {
178               break;
179           }
180           beforeMidKey = item;
181       }
182       System.out.println("midkey: " + midKV + " or: " + Bytes.toStringBinary(midkey));
183       System.out.println("beforeMidKey: " + beforeMidKey);
184 
185 
186       // Seek on the splitKey, should be in top, not in bottom
187       KeyValue foundKeyValue = doTestOfSeekBefore(p, fs, bottom, midKV, cacheConf);
188       assertEquals(beforeMidKey, foundKeyValue);
189 
190       // Seek tot the last thing should be the penultimate on the top, the one before the midkey on the bottom.
191       foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(items.size() - 1), cacheConf);
192       assertEquals(items.get(items.size() - 2), foundKeyValue);
193 
194       foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(items.size() - 1), cacheConf);
195       assertEquals(beforeMidKey, foundKeyValue);
196 
197       // Try and seek before something that is in the bottom.
198       foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(0), cacheConf);
199       assertNull(foundKeyValue);
200 
201       // Try and seek before the first thing.
202       foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(0), cacheConf);
203       assertNull(foundKeyValue);
204 
205       // Try and seek before the second thing in the top and bottom.
206       foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(1), cacheConf);
207       assertNull(foundKeyValue);
208 
209       foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(1), cacheConf);
210       assertEquals(items.get(0), foundKeyValue);
211 
212       // Try to seek before the splitKey in the top file
213       foundKeyValue = doTestOfSeekBefore(p, fs, top, midKV, cacheConf);
214       assertNull(foundKeyValue);
215     }
216 
217   private KeyValue doTestOfSeekBefore(Path p, FileSystem fs, Reference bottom, KeyValue seekBefore,
218                                         CacheConfig cacheConfig)
219             throws IOException {
220       final HalfStoreFileReader halfreader = new HalfStoreFileReader(fs, p,
221               cacheConfig, bottom, TEST_UTIL.getConfiguration());
222       halfreader.loadFileInfo();
223       final HFileScanner scanner = halfreader.getScanner(false, false);
224       scanner.seekBefore(seekBefore.getKey());
225       return scanner.getKeyValue();
226   }
227 
228   private KeyValue getLastOnCol(KeyValue curr) {
229     return KeyValue.createLastOnRow(
230         curr.getBuffer(), curr.getRowOffset(), curr.getRowLength(),
231         curr.getBuffer(), curr.getFamilyOffset(), curr.getFamilyLength(),
232         curr.getBuffer(), curr.getQualifierOffset(), curr.getQualifierLength());
233   }
234 
235   static final int SIZE = 1000;
236 
237   static byte[] _b(String s) {
238     return Bytes.toBytes(s);
239   }
240 
241   List<KeyValue> genSomeKeys() {
242     List<KeyValue> ret = new ArrayList<KeyValue>(SIZE);
243     for (int i = 0; i < SIZE; i++) {
244       KeyValue kv =
245           new KeyValue(
246               _b(String.format("row_%04d", i)),
247               _b("family"),
248               _b("qualifier"),
249               1000, // timestamp
250               _b("value"));
251       ret.add(kv);
252     }
253     return ret;
254   }
255 
256 
257 
258 }
259