View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.mapreduce;
20  
21  import java.io.IOException;
22  import java.util.List;
23  
24  import org.apache.hadoop.hbase.HRegionLocation;
25  import org.apache.hadoop.hbase.client.HTable;
26  import org.apache.hadoop.hbase.testclassification.LargeTests;
27  import org.junit.Test;
28  import org.junit.experimental.categories.Category;
29  
30  /**
31   * TestTableInputFormatScan part 1.
32   * @see TestTableInputFormatScanBase
33   */
34  @Category(LargeTests.class)
35  public class TestTableInputFormatScan1 extends TestTableInputFormatScanBase {
36  
37    /**
38     * Tests a MR scan using specific start and stop rows.
39     *
40     * @throws IOException
41     * @throws ClassNotFoundException
42     * @throws InterruptedException
43     */
44    @Test
45    public void testScanEmptyToEmpty()
46    throws IOException, InterruptedException, ClassNotFoundException {
47      testScan(null, null, null);
48    }
49  
50    /**
51     * Tests a MR scan using specific start and stop rows.
52     *
53     * @throws IOException
54     * @throws ClassNotFoundException
55     * @throws InterruptedException
56     */
57    @Test
58    public void testScanEmptyToAPP()
59    throws IOException, InterruptedException, ClassNotFoundException {
60      testScan(null, "app", "apo");
61    }
62  
63    /**
64     * Tests a MR scan using specific start and stop rows.
65     *
66     * @throws IOException
67     * @throws ClassNotFoundException
68     * @throws InterruptedException
69     */
70    @Test
71    public void testScanEmptyToBBA()
72    throws IOException, InterruptedException, ClassNotFoundException {
73      testScan(null, "bba", "baz");
74    }
75  
76    /**
77     * Tests a MR scan using specific start and stop rows.
78     *
79     * @throws IOException
80     * @throws ClassNotFoundException
81     * @throws InterruptedException
82     */
83    @Test
84    public void testScanEmptyToBBB()
85    throws IOException, InterruptedException, ClassNotFoundException {
86      testScan(null, "bbb", "bba");
87    }
88  
89    /**
90     * Tests a MR scan using specific start and stop rows.
91     *
92     * @throws IOException
93     * @throws ClassNotFoundException
94     * @throws InterruptedException
95     */
96    @Test
97    public void testScanEmptyToOPP()
98    throws IOException, InterruptedException, ClassNotFoundException {
99      testScan(null, "opp", "opo");
100   }
101 
102   /**
103    * Tests a MR scan using specific number of mappers. The test table has 25 regions,
104    * and all region sizes are set as 0 as default. The average region size is 1 (the smallest
105    * positive). When we set hbase.mapreduce.input.ratio as -1, all regions will be cut into two
106    * MapRedcue input splits, the number of MR input splits should be 50; when we set hbase
107    * .mapreduce.input.ratio as 100, the sum of all region sizes is less then the average region
108    * size, all regions will be combined into 1 MapRedcue input split.
109    *
110    * @throws IOException
111    * @throws ClassNotFoundException
112    * @throws InterruptedException
113    */
114   @Test
115   public void testGetSplits() throws IOException, InterruptedException, ClassNotFoundException {
116     HTable table = new HTable(TEST_UTIL.getConfiguration(), TABLE_NAME);
117     List<HRegionLocation> locs = table.getRegionLocator().getAllRegionLocations();
118 
119     testNumOfSplits("-1", locs.size()*2);
120     table.close();
121     testNumOfSplits("100", 1);
122   }
123 
124   /**
125    * Tests the getSplitKey() method in TableInputFormatBase.java
126    *
127    * @throws IOException
128    * @throws ClassNotFoundException
129    * @throws InterruptedException
130    */
131   @Test
132   public void testGetSplitsPoint() throws IOException, InterruptedException,
133   ClassNotFoundException {
134     // Test Case 1: "aaabcdef" and "aaaff", split point is "aaad".
135     byte[] start1 = { 'a', 'a', 'a', 'b', 'c', 'd', 'e', 'f' };
136     byte[] end1 = { 'a', 'a', 'a', 'f', 'f' };
137     byte[] splitPoint1 = { 'a', 'a', 'a', 'd' };
138     testGetSplitKey(start1, end1, splitPoint1, true);
139 
140     // Test Case 2: "111000" and "1125790", split point is "111b".
141     byte[] start2 = { '1', '1', '1', '0', '0', '0' };
142     byte[] end2 = { '1', '1', '2', '5', '7', '9', '0' };
143     byte[] splitPoint2 = { '1', '1', '1', 'b' };
144     testGetSplitKey(start2, end2, splitPoint2, true);
145 
146     // Test Case 3: "aaaaaa" and "aab", split point is "aaap".
147     byte[] start3 = { 'a', 'a', 'a', 'a', 'a', 'a' };
148     byte[] end3 = { 'a', 'a', 'b' };
149     byte[] splitPoint3 = { 'a', 'a', 'a', 'p' };
150     testGetSplitKey(start3, end3, splitPoint3, true);
151 
152     // Test Case 4: "aaa" and "aaaz", split point is "aaaM".
153     byte[] start4 = { 'a', 'a', 'a' };
154     byte[] end4 = { 'a', 'a', 'a', 'z' };
155     byte[] splitPoint4 = { 'a', 'a', 'a', 'M' };
156     testGetSplitKey(start4, end4, splitPoint4, true);
157 
158     // Test Case 5: "aaa" and "aaba", split point is "aaap".
159     byte[] start5 = { 'a', 'a', 'a' };
160     byte[] end5 = { 'a', 'a', 'b', 'a' };
161     byte[] splitPoint5 = { 'a', 'a', 'a', 'p' };
162     testGetSplitKey(start5, end5, splitPoint5, true);
163 
164     // Test Case 6: empty key and "hhhqqqwww", split point is "h"
165     byte[] start6 = {};
166     byte[] end6 = { 'h', 'h', 'h', 'q', 'q', 'q', 'w', 'w' };
167     byte[] splitPoint6 = { 'h' };
168     testGetSplitKey(start6, end6, splitPoint6, true);
169 
170     // Test Case 7: "ffffaaa" and empty key, split point depends on the mode we choose(text key or
171     // binary key).
172     byte[] start7 = { 'f', 'f', 'f', 'f', 'a', 'a', 'a' };
173     byte[] end7 = {};
174     byte[] splitPointText7 = { 'f', '~', '~', '~', '~', '~', '~'  };
175     byte[] splitPointBinary7 = { 'f', 127, 127, 127, 127, 127, 127  };
176     testGetSplitKey(start7, end7, splitPointText7, true);
177     testGetSplitKey(start7, end7, splitPointBinary7, false);
178 
179     // Test Case 8: both start key and end key are empty. Split point depends on the mode we
180     // choose (text key or binary key).
181     byte[] start8 = {};
182     byte[] end8 = {};
183     byte[] splitPointText8 = { 'O' };
184     byte[] splitPointBinary8 = { 0 };
185     testGetSplitKey(start8, end8, splitPointText8, true);
186     testGetSplitKey(start8, end8, splitPointBinary8, false);
187 
188     // Test Case 9: Binary Key example
189     byte[] start9 = { 13, -19, 126, 127 };
190     byte[] end9 = { 13, -19, 127, 0 };
191     byte[] splitPoint9 = { 13, -19, 127, -64 };
192     testGetSplitKey(start9, end9, splitPoint9, false);
193   }
194 }