View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.mapreduce;
20  
21  import static org.junit.Assert.assertArrayEquals;
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertTrue;
24  import static org.junit.Assert.fail;
25  
26  import java.io.IOException;
27  import java.util.TreeMap;
28  
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.fs.FileSystem;
31  import org.apache.hadoop.fs.Path;
32  import org.apache.hadoop.hbase.HBaseTestingUtility;
33  import org.apache.hadoop.hbase.HColumnDescriptor;
34  import org.apache.hadoop.hbase.HTableDescriptor;
35  import org.apache.hadoop.hbase.LargeTests;
36  import org.apache.hadoop.hbase.TableName;
37  import org.apache.hadoop.hbase.client.HBaseAdmin;
38  import org.apache.hadoop.hbase.client.HTable;
39  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
40  import org.apache.hadoop.hbase.io.hfile.HFile;
41  import org.apache.hadoop.hbase.io.hfile.HFileScanner;
42  import org.apache.hadoop.hbase.regionserver.BloomType;
43  import org.apache.hadoop.hbase.util.Bytes;
44  import org.apache.hadoop.hbase.util.HFileTestUtil;
45  import org.junit.AfterClass;
46  import org.junit.BeforeClass;
47  import org.junit.Test;
48  import org.junit.experimental.categories.Category;
49  
50  /**
51   * Test cases for the "load" half of the HFileOutputFormat bulk load
52   * functionality. These tests run faster than the full MR cluster
53   * tests in TestHFileOutputFormat
54   */
55  @Category(LargeTests.class)
56  public class TestLoadIncrementalHFiles {
57    private static final byte[] QUALIFIER = Bytes.toBytes("myqual");
58    private static final byte[] FAMILY = Bytes.toBytes("myfam");
59    static final String EXPECTED_MSG_FOR_NON_EXISTING_FAMILY = "Unmatched family names found";
60    static final int MAX_FILES_PER_REGION_PER_FAMILY = 4;
61  
62    private static final byte[][] SPLIT_KEYS = new byte[][] {
63      Bytes.toBytes("ddd"),
64      Bytes.toBytes("ppp")
65    };
66  
67    static HBaseTestingUtility util = new HBaseTestingUtility();
68  
69    @BeforeClass
70    public static void setUpBeforeClass() throws Exception {
71      util.getConfiguration().setInt(
72        LoadIncrementalHFiles.MAX_FILES_PER_REGION_PER_FAMILY,
73        MAX_FILES_PER_REGION_PER_FAMILY);
74      util.startMiniCluster();
75    }
76  
77    @AfterClass
78    public static void tearDownAfterClass() throws Exception {
79      util.shutdownMiniCluster();
80    }
81  
82    /**
83     * Test case that creates some regions and loads
84     * HFiles that fit snugly inside those regions
85     */
86    @Test
87    public void testSimpleLoad() throws Exception {
88      runTest("testSimpleLoad", BloomType.NONE,
89          new byte[][][] {
90            new byte[][]{ Bytes.toBytes("aaaa"), Bytes.toBytes("cccc") },
91            new byte[][]{ Bytes.toBytes("ddd"), Bytes.toBytes("ooo") },
92      });
93    }
94  
95    /**
96     * Test case that creates some regions and loads
97     * HFiles that cross the boundaries of those regions
98     */
99    @Test
100   public void testRegionCrossingLoad() throws Exception {
101     runTest("testRegionCrossingLoad", BloomType.NONE,
102         new byte[][][] {
103           new byte[][]{ Bytes.toBytes("aaaa"), Bytes.toBytes("eee") },
104           new byte[][]{ Bytes.toBytes("fff"), Bytes.toBytes("zzz") },
105     });
106   }
107 
108   /**
109    * Test loading into a column family that has a ROW bloom filter.
110    */
111   @Test
112   public void testRegionCrossingRowBloom() throws Exception {
113     runTest("testRegionCrossingLoadRowBloom", BloomType.ROW,
114         new byte[][][] {
115           new byte[][]{ Bytes.toBytes("aaaa"), Bytes.toBytes("eee") },
116           new byte[][]{ Bytes.toBytes("fff"), Bytes.toBytes("zzz") },
117     });
118   }
119   
120   /**
121    * Test loading into a column family that has a ROWCOL bloom filter.
122    */
123   @Test
124   public void testRegionCrossingRowColBloom() throws Exception {
125     runTest("testRegionCrossingLoadRowColBloom", BloomType.ROWCOL,
126         new byte[][][] {
127           new byte[][]{ Bytes.toBytes("aaaa"), Bytes.toBytes("eee") },
128           new byte[][]{ Bytes.toBytes("fff"), Bytes.toBytes("zzz") },
129     });
130   }
131 
132   private void runTest(String testName, BloomType bloomType, 
133           byte[][][] hfileRanges) throws Exception {
134     Path dir = util.getDataTestDirOnTestFS(testName);
135     FileSystem fs = util.getTestFileSystem();
136     dir = dir.makeQualified(fs);
137     Path familyDir = new Path(dir, Bytes.toString(FAMILY));
138 
139     int hfileIdx = 0;
140     for (byte[][] range : hfileRanges) {
141       byte[] from = range[0];
142       byte[] to = range[1];
143       HFileTestUtil.createHFile(util.getConfiguration(), fs, new Path(familyDir, "hfile_"
144           + hfileIdx++), FAMILY, QUALIFIER, from, to, 1000);
145     }
146     int expectedRows = hfileIdx * 1000;
147 
148     final byte[] TABLE = Bytes.toBytes("mytable_"+testName);
149 
150     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
151     HColumnDescriptor familyDesc = new HColumnDescriptor(FAMILY);
152     familyDesc.setBloomFilterType(bloomType);
153     htd.addFamily(familyDesc);
154 
155     LoadIncrementalHFiles loader = new LoadIncrementalHFiles(util.getConfiguration());
156     String [] args= {dir.toString(),"mytable_"+testName};
157     loader.run(args);
158     HTable table = new HTable(util.getConfiguration(), TABLE);
159     
160     assertEquals(expectedRows, util.countRows(table));
161   }
162 
163   /**
164    * Test loading into a column family that does not exist.
165    */
166   @Test
167   public void testNonexistentColumnFamilyLoad() throws Exception {
168     String testName = "testNonexistentColumnFamilyLoad";
169     byte[][][] hFileRanges = new byte[][][] {
170       new byte[][]{ Bytes.toBytes("aaa"), Bytes.toBytes("ccc") },
171       new byte[][]{ Bytes.toBytes("ddd"), Bytes.toBytes("ooo") },
172     }; 
173 
174     Path dir = util.getDataTestDirOnTestFS(testName);
175     FileSystem fs = util.getTestFileSystem();
176     dir = dir.makeQualified(fs);
177     Path familyDir = new Path(dir, Bytes.toString(FAMILY));
178 
179     int hFileIdx = 0;
180     for (byte[][] range : hFileRanges) {
181       byte[] from = range[0];
182       byte[] to = range[1];
183       HFileTestUtil.createHFile(util.getConfiguration(), fs, new Path(familyDir, "hfile_"
184           + hFileIdx++), FAMILY, QUALIFIER, from, to, 1000);
185     }
186 
187     final byte[] TABLE = Bytes.toBytes("mytable_"+testName);
188 
189     HBaseAdmin admin = new HBaseAdmin(util.getConfiguration());
190     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
191     // set real family name to upper case in purpose to simulate the case that
192     // family name in HFiles is invalid
193     HColumnDescriptor family =
194         new HColumnDescriptor(Bytes.toBytes(new String(FAMILY).toUpperCase()));
195     htd.addFamily(family);
196     admin.createTable(htd, SPLIT_KEYS);
197 
198     HTable table = new HTable(util.getConfiguration(), TABLE);
199     util.waitTableEnabled(TABLE);
200     LoadIncrementalHFiles loader = new LoadIncrementalHFiles(util.getConfiguration());
201     try {
202       loader.doBulkLoad(dir, table);
203       assertTrue("Loading into table with non-existent family should have failed", false);
204     } catch (Exception e) {
205       assertTrue("IOException expected", e instanceof IOException);
206       // further check whether the exception message is correct
207       String errMsg = e.getMessage();
208       assertTrue("Incorrect exception message, expected message: ["
209           + EXPECTED_MSG_FOR_NON_EXISTING_FAMILY + "], current message: [" + errMsg + "]",
210           errMsg.contains(EXPECTED_MSG_FOR_NON_EXISTING_FAMILY));
211     }
212     table.close();
213     admin.close();
214   }
215 
216   @Test
217   public void testSplitStoreFile() throws IOException {
218     Path dir = util.getDataTestDirOnTestFS("testSplitHFile");
219     FileSystem fs = util.getTestFileSystem();
220     Path testIn = new Path(dir, "testhfile");
221     HColumnDescriptor familyDesc = new HColumnDescriptor(FAMILY);
222     HFileTestUtil.createHFile(util.getConfiguration(), fs, testIn, FAMILY, QUALIFIER,
223         Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), 1000);
224 
225     Path bottomOut = new Path(dir, "bottom.out");
226     Path topOut = new Path(dir, "top.out");
227 
228     LoadIncrementalHFiles.splitStoreFile(
229         util.getConfiguration(), testIn,
230         familyDesc, Bytes.toBytes("ggg"),
231         bottomOut,
232         topOut);
233 
234     int rowCount = verifyHFile(bottomOut);
235     rowCount += verifyHFile(topOut);
236     assertEquals(1000, rowCount);
237   }
238 
239   private int verifyHFile(Path p) throws IOException {
240     Configuration conf = util.getConfiguration();
241     HFile.Reader reader = HFile.createReader(
242         p.getFileSystem(conf), p, new CacheConfig(conf), conf);
243     reader.loadFileInfo();
244     HFileScanner scanner = reader.getScanner(false, false);
245     scanner.seekTo();
246     int count = 0;
247     do {
248       count++;
249     } while (scanner.next());
250     assertTrue(count > 0);
251     reader.close();
252     return count;
253   }
254 
255   private void addStartEndKeysForTest(TreeMap<byte[], Integer> map, byte[] first, byte[] last) {
256     Integer value = map.containsKey(first)?map.get(first):0;
257     map.put(first, value+1);
258 
259     value = map.containsKey(last)?map.get(last):0;
260     map.put(last, value-1);
261   }
262 
263   @Test 
264   public void testInferBoundaries() {
265     TreeMap<byte[], Integer> map = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
266 
267     /* Toy example
268      *     c---------i            o------p          s---------t     v------x
269      * a------e    g-----k   m-------------q   r----s            u----w
270      *
271      * Should be inferred as:
272      * a-----------------k   m-------------q   r--------------t  u---------x
273      * 
274      * The output should be (m,r,u) 
275      */
276 
277     String first;
278     String last;
279 
280     first = "a"; last = "e";
281     addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
282     
283     first = "r"; last = "s";
284     addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
285 
286     first = "o"; last = "p";
287     addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
288 
289     first = "g"; last = "k";
290     addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
291 
292     first = "v"; last = "x";
293     addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
294 
295     first = "c"; last = "i";
296     addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
297 
298     first = "m"; last = "q";
299     addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
300 
301     first = "s"; last = "t";
302     addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
303     
304     first = "u"; last = "w";
305     addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
306 
307     byte[][] keysArray = LoadIncrementalHFiles.inferBoundaries(map);
308     byte[][] compare = new byte[3][];
309     compare[0] = "m".getBytes();
310     compare[1] = "r".getBytes(); 
311     compare[2] = "u".getBytes();
312 
313     assertEquals(keysArray.length, 3);
314 
315     for (int row = 0; row<keysArray.length; row++){
316       assertArrayEquals(keysArray[row], compare[row]);
317     }
318   }
319 
320   @Test
321   public void testLoadTooMayHFiles() throws Exception {
322     Path dir = util.getDataTestDirOnTestFS("testLoadTooMayHFiles");
323     FileSystem fs = util.getTestFileSystem();
324     dir = dir.makeQualified(fs);
325     Path familyDir = new Path(dir, Bytes.toString(FAMILY));
326 
327     byte[] from = Bytes.toBytes("begin");
328     byte[] to = Bytes.toBytes("end");
329     for (int i = 0; i <= MAX_FILES_PER_REGION_PER_FAMILY; i++) {
330       HFileTestUtil.createHFile(util.getConfiguration(), fs, new Path(familyDir, "hfile_"
331           + i), FAMILY, QUALIFIER, from, to, 1000);
332     }
333 
334     LoadIncrementalHFiles loader = new LoadIncrementalHFiles(util.getConfiguration());
335     String [] args= {dir.toString(), "mytable_testLoadTooMayHFiles"};
336     try {
337       loader.run(args);
338       fail("Bulk loading too many files should fail");
339     } catch (IOException ie) {
340       assertTrue(ie.getMessage().contains("Trying to load more than "
341         + MAX_FILES_PER_REGION_PER_FAMILY + " hfiles"));
342     }
343   }
344 }
345