View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.util;
20  
21  import static org.junit.Assert.assertArrayEquals;
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertFalse;
24  import static org.junit.Assert.assertNotSame;
25  import static org.junit.Assert.assertTrue;
26  
27  import java.util.ArrayList;
28  import java.util.List;
29  import java.util.Map;
30  
31  import org.apache.commons.lang.ArrayUtils;
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.hbase.HBaseTestingUtility;
36  import org.apache.hadoop.hbase.HRegionInfo;
37  import org.apache.hadoop.hbase.MediumTests;
38  import org.apache.hadoop.hbase.ServerName;
39  import org.apache.hadoop.hbase.client.HTable;
40  import org.apache.hadoop.hbase.util.RegionSplitter.HexStringSplit;
41  import org.apache.hadoop.hbase.util.RegionSplitter.SplitAlgorithm;
42  import org.apache.hadoop.hbase.util.RegionSplitter.UniformSplit;
43  import org.junit.AfterClass;
44  import org.junit.BeforeClass;
45  import org.junit.Test;
46  import org.junit.experimental.categories.Category;
47  
48  /**
49   * Tests for {@link RegionSplitter}, which can create a pre-split table or do a
50   * rolling split of an existing table.
51   */
52  @Category(MediumTests.class)
53  public class TestRegionSplitter {
54      private final static Log LOG = LogFactory.getLog(TestRegionSplitter.class);
55      private final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
56      private final static String CF_NAME = "SPLIT_TEST_CF";
57      private final static byte xFF = (byte) 0xff;
58  
59      @BeforeClass
60      public static void setup() throws Exception {
61          UTIL.startMiniCluster();
62      }
63  
64      @AfterClass
65      public static void teardown() throws Exception {
66          UTIL.shutdownMiniCluster();
67      }
68  
69      /**
70       * Test creating a pre-split table using the HexStringSplit algorithm.
71       */
72      @Test
73      public void testCreatePresplitTableHex() throws Exception {
74        final List<byte[]> expectedBounds = new ArrayList<byte[]>();
75        expectedBounds.add(ArrayUtils.EMPTY_BYTE_ARRAY);
76        expectedBounds.add("10000000".getBytes());
77        expectedBounds.add("20000000".getBytes());
78        expectedBounds.add("30000000".getBytes());
79        expectedBounds.add("40000000".getBytes());
80        expectedBounds.add("50000000".getBytes());
81        expectedBounds.add("60000000".getBytes());
82        expectedBounds.add("70000000".getBytes());
83        expectedBounds.add("80000000".getBytes());
84        expectedBounds.add("90000000".getBytes());
85        expectedBounds.add("a0000000".getBytes());
86        expectedBounds.add("b0000000".getBytes());
87        expectedBounds.add("c0000000".getBytes());
88        expectedBounds.add("d0000000".getBytes());
89        expectedBounds.add("e0000000".getBytes());
90        expectedBounds.add("f0000000".getBytes());
91            expectedBounds.add(ArrayUtils.EMPTY_BYTE_ARRAY);
92  
93            // Do table creation/pre-splitting and verification of region boundaries
94        preSplitTableAndVerify(expectedBounds,
95            HexStringSplit.class.getSimpleName(), "NewHexPresplitTable");
96      }
97  
98      /**
99       * Test creating a pre-split table using the UniformSplit algorithm.
100      */
101     @Test
102     public void testCreatePresplitTableUniform() throws Exception {
103       List<byte[]> expectedBounds = new ArrayList<byte[]>();
104       expectedBounds.add(ArrayUtils.EMPTY_BYTE_ARRAY);
105       expectedBounds.add(new byte[] {      0x10, 0, 0, 0, 0, 0, 0, 0});
106       expectedBounds.add(new byte[] {      0x20, 0, 0, 0, 0, 0, 0, 0});
107       expectedBounds.add(new byte[] {      0x30, 0, 0, 0, 0, 0, 0, 0});
108       expectedBounds.add(new byte[] {      0x40, 0, 0, 0, 0, 0, 0, 0});
109       expectedBounds.add(new byte[] {      0x50, 0, 0, 0, 0, 0, 0, 0});
110       expectedBounds.add(new byte[] {      0x60, 0, 0, 0, 0, 0, 0, 0});
111       expectedBounds.add(new byte[] {      0x70, 0, 0, 0, 0, 0, 0, 0});
112       expectedBounds.add(new byte[] {(byte)0x80, 0, 0, 0, 0, 0, 0, 0});
113       expectedBounds.add(new byte[] {(byte)0x90, 0, 0, 0, 0, 0, 0, 0});
114       expectedBounds.add(new byte[] {(byte)0xa0, 0, 0, 0, 0, 0, 0, 0});
115       expectedBounds.add(new byte[] {(byte)0xb0, 0, 0, 0, 0, 0, 0, 0});
116       expectedBounds.add(new byte[] {(byte)0xc0, 0, 0, 0, 0, 0, 0, 0});
117       expectedBounds.add(new byte[] {(byte)0xd0, 0, 0, 0, 0, 0, 0, 0});
118       expectedBounds.add(new byte[] {(byte)0xe0, 0, 0, 0, 0, 0, 0, 0});
119       expectedBounds.add(new byte[] {(byte)0xf0, 0, 0, 0, 0, 0, 0, 0});
120       expectedBounds.add(ArrayUtils.EMPTY_BYTE_ARRAY);
121 
122       // Do table creation/pre-splitting and verification of region boundaries
123       preSplitTableAndVerify(expectedBounds, UniformSplit.class.getSimpleName(),
124         "NewUniformPresplitTable");
125     }
126 
127     /**
128      * Unit tests for the HexStringSplit algorithm. Makes sure it divides up the
129      * space of keys in the way that we expect.
130      */
131     @Test
132     public void unitTestHexStringSplit() {
133         HexStringSplit splitter = new HexStringSplit();
134         // Check splitting while starting from scratch
135 
136         byte[][] twoRegionsSplits = splitter.split(2);
137         assertEquals(1, twoRegionsSplits.length);
138     assertArrayEquals(twoRegionsSplits[0], "80000000".getBytes());
139 
140         byte[][] threeRegionsSplits = splitter.split(3);
141         assertEquals(2, threeRegionsSplits.length);
142         byte[] expectedSplit0 = "55555555".getBytes();
143         assertArrayEquals(expectedSplit0, threeRegionsSplits[0]);
144         byte[] expectedSplit1 = "aaaaaaaa".getBytes();
145         assertArrayEquals(expectedSplit1, threeRegionsSplits[1]);
146 
147         // Check splitting existing regions that have start and end points
148         byte[] splitPoint = splitter.split("10000000".getBytes(), "30000000".getBytes());
149         assertArrayEquals("20000000".getBytes(), splitPoint);
150 
151         byte[] lastRow = "ffffffff".getBytes();
152         assertArrayEquals(lastRow, splitter.lastRow());
153         byte[] firstRow = "00000000".getBytes();
154         assertArrayEquals(firstRow, splitter.firstRow());
155 
156         // Halfway between 00... and 20... should be 10...
157         splitPoint = splitter.split(firstRow, "20000000".getBytes());
158         assertArrayEquals(splitPoint, "10000000".getBytes());
159 
160         // Halfway between df... and ff... should be ef....
161         splitPoint = splitter.split("dfffffff".getBytes(), lastRow);
162         assertArrayEquals(splitPoint,"efffffff".getBytes());
163     }
164 
165     /**
166      * Unit tests for the UniformSplit algorithm. Makes sure it divides up the space of
167      * keys in the way that we expect.
168      */
169     @Test
170     public void unitTestUniformSplit() {
171         UniformSplit splitter = new UniformSplit();
172 
173         // Check splitting while starting from scratch
174         try {
175             splitter.split(1);
176             throw new AssertionError("Splitting into <2 regions should have thrown exception");
177         } catch (IllegalArgumentException e) { }
178 
179         byte[][] twoRegionsSplits = splitter.split(2);
180         assertEquals(1, twoRegionsSplits.length);
181         assertArrayEquals(twoRegionsSplits[0],
182             new byte[] { (byte) 0x80, 0, 0, 0, 0, 0, 0, 0 });
183 
184         byte[][] threeRegionsSplits = splitter.split(3);
185         assertEquals(2, threeRegionsSplits.length);
186         byte[] expectedSplit0 = new byte[] {0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55};
187         assertArrayEquals(expectedSplit0, threeRegionsSplits[0]);
188         byte[] expectedSplit1 = new byte[] {(byte)0xAA, (byte)0xAA, (byte)0xAA, (byte)0xAA,
189                 (byte)0xAA, (byte)0xAA, (byte)0xAA, (byte)0xAA};
190         assertArrayEquals(expectedSplit1, threeRegionsSplits[1]);
191 
192         // Check splitting existing regions that have start and end points
193         byte[] splitPoint = splitter.split(new byte[] {0x10}, new byte[] {0x30});
194         assertArrayEquals(new byte[] {0x20}, splitPoint);
195 
196         byte[] lastRow = new byte[] {xFF, xFF, xFF, xFF, xFF, xFF, xFF, xFF};
197         assertArrayEquals(lastRow, splitter.lastRow());
198         byte[] firstRow = ArrayUtils.EMPTY_BYTE_ARRAY;
199         assertArrayEquals(firstRow, splitter.firstRow());
200 
201         splitPoint = splitter.split(firstRow, new byte[] {0x20});
202         assertArrayEquals(splitPoint, new byte[] {0x10});
203 
204         splitPoint = splitter.split(new byte[] {(byte)0xdf, xFF, xFF, xFF, xFF,
205                 xFF, xFF, xFF}, lastRow);
206         assertArrayEquals(splitPoint,
207                 new byte[] {(byte)0xef, xFF, xFF, xFF, xFF, xFF, xFF, xFF});
208     }
209 
210   @Test
211   public void testUserInput() {
212     SplitAlgorithm algo = new HexStringSplit();
213     assertFalse(splitFailsPrecondition(algo)); // default settings are fine
214     assertFalse(splitFailsPrecondition(algo, "00", "AA")); // custom is fine
215     assertTrue(splitFailsPrecondition(algo, "AA", "00")); // range error
216     assertTrue(splitFailsPrecondition(algo, "AA", "AA")); // range error
217     assertFalse(splitFailsPrecondition(algo, "0", "2", 3)); // should be fine
218     assertFalse(splitFailsPrecondition(algo, "0", "A", 11)); // should be fine
219     assertTrue(splitFailsPrecondition(algo, "0", "A", 12)); // too granular
220 
221     algo = new UniformSplit();
222     assertFalse(splitFailsPrecondition(algo)); // default settings are fine
223     assertFalse(splitFailsPrecondition(algo, "\\x00", "\\xAA")); // custom is fine
224     assertTrue(splitFailsPrecondition(algo, "\\xAA", "\\x00")); // range error
225     assertTrue(splitFailsPrecondition(algo, "\\xAA", "\\xAA")); // range error
226     assertFalse(splitFailsPrecondition(algo, "\\x00", "\\x02", 3)); // should be fine
227     assertFalse(splitFailsPrecondition(algo, "\\x00", "\\x0A", 11)); // should be fine
228     assertTrue(splitFailsPrecondition(algo, "\\x00", "\\x0A", 12)); // too granular
229   }
230 
231   private boolean splitFailsPrecondition(SplitAlgorithm algo) {
232     return splitFailsPrecondition(algo, 100);
233   }
234 
235   private boolean splitFailsPrecondition(SplitAlgorithm algo, String firstRow,
236       String lastRow) {
237     return splitFailsPrecondition(algo, firstRow, lastRow, 100);
238   }
239 
240   private boolean splitFailsPrecondition(SplitAlgorithm algo, String firstRow,
241       String lastRow, int numRegions) {
242     algo.setFirstRow(firstRow);
243     algo.setLastRow(lastRow);
244     return splitFailsPrecondition(algo, numRegions);
245   }
246 
247   private boolean splitFailsPrecondition(SplitAlgorithm algo, int numRegions) {
248     try {
249       byte[][] s = algo.split(numRegions);
250       LOG.debug("split algo = " + algo);
251       if (s != null) {
252         StringBuilder sb = new StringBuilder();
253         for (byte[] b : s) {
254           sb.append(Bytes.toStringBinary(b) + "  ");
255         }
256         LOG.debug(sb.toString());
257       }
258       return false;
259     } catch (IllegalArgumentException e) {
260       return true;
261     } catch (IllegalStateException e) {
262       return true;
263     } catch (IndexOutOfBoundsException e) {
264       return true;
265     }
266   }
267 
268     /**
269      * Creates a pre-split table with expectedBounds.size()+1 regions, then
270      * verifies that the region boundaries are the same as the expected
271      * region boundaries in expectedBounds.
272      * @throws Various junit assertions
273      */
274     private void preSplitTableAndVerify(List<byte[]> expectedBounds,
275             String splitClass, String tableName) throws Exception {
276         final int numRegions = expectedBounds.size()-1;
277         final Configuration conf = UTIL.getConfiguration();
278         conf.setInt("split.count", numRegions);
279         SplitAlgorithm splitAlgo = RegionSplitter.newSplitAlgoInstance(conf, splitClass);
280         RegionSplitter.createPresplitTable(tableName, splitAlgo,
281                 new String[] {CF_NAME}, conf);
282         verifyBounds(expectedBounds, tableName);
283     }
284 
285     private void rollingSplitAndVerify(String tableName, String splitClass,
286             List<byte[]> expectedBounds)  throws Exception {
287         final Configuration conf = UTIL.getConfiguration();
288 
289         // Set this larger than the number of splits so RegionSplitter won't block
290         conf.setInt("split.outstanding", 5);
291         SplitAlgorithm splitAlgo = RegionSplitter.newSplitAlgoInstance(conf, splitClass);
292         RegionSplitter.rollingSplit(tableName, splitAlgo, conf);
293         verifyBounds(expectedBounds, tableName);
294     }
295 
296     private void verifyBounds(List<byte[]> expectedBounds, String tableName)
297             throws Exception {
298         // Get region boundaries from the cluster and verify their endpoints
299         final Configuration conf = UTIL.getConfiguration();
300         final int numRegions = expectedBounds.size()-1;
301         final HTable hTable = new HTable(conf, tableName.getBytes());
302         final Map<HRegionInfo, ServerName> regionInfoMap = hTable.getRegionLocations();
303         assertEquals(numRegions, regionInfoMap.size());
304         for (Map.Entry<HRegionInfo, ServerName> entry: regionInfoMap.entrySet()) {
305             final HRegionInfo regionInfo = entry.getKey();
306             byte[] regionStart = regionInfo.getStartKey();
307             byte[] regionEnd = regionInfo.getEndKey();
308 
309             // This region's start key should be one of the region boundaries
310             int startBoundaryIndex = indexOfBytes(expectedBounds, regionStart);
311             assertNotSame(-1, startBoundaryIndex);
312 
313             // This region's end key should be the region boundary that comes
314             // after the starting boundary.
315             byte[] expectedRegionEnd = expectedBounds.get(
316                     startBoundaryIndex+1);
317             assertEquals(0, Bytes.compareTo(regionEnd, expectedRegionEnd));
318         }
319     }
320 
321     /**
322      * List.indexOf() doesn't really work for a List<byte[]>, because byte[]
323      * doesn't override equals(). This method checks whether a list contains
324      * a given element by checking each element using the byte array
325      * comparator.
326      * @return the index of the first element that equals compareTo, or -1
327      * if no elements are equal.
328      */
329     static private int indexOfBytes(List<byte[]> list,  byte[] compareTo) {
330         int listIndex = 0;
331         for(byte[] elem: list) {
332             if(Bytes.BYTES_COMPARATOR.compare(elem, compareTo) == 0) {
333                 return listIndex;
334             }
335             listIndex++;
336         }
337         return -1;
338     }
339 
340 }
341