View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertNull;
23  import static org.junit.Assert.assertTrue;
24  
25  import java.io.IOException;
26  import java.util.ArrayList;
27  import java.util.List;
28  
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.hbase.TableName;
31  import org.apache.hadoop.hbase.HBaseConfiguration;
32  import org.apache.hadoop.hbase.HConstants;
33  import org.apache.hadoop.hbase.HRegionInfo;
34  import org.apache.hadoop.hbase.HTableDescriptor;
35  import org.apache.hadoop.hbase.testclassification.SmallTests;
36  import org.apache.hadoop.hbase.util.Bytes;
37  import org.junit.Before;
38  import org.junit.Test;
39  import org.junit.experimental.categories.Category;
40  import org.mockito.Mockito;
41  
42  @Category(SmallTests.class)
43  public class TestRegionSplitPolicy {
44  
45    private Configuration conf;
46    private HTableDescriptor htd;
47    private HRegion mockRegion;
48    private List<Store> stores;
49    private static final TableName TABLENAME = TableName.valueOf("t");
50  
51    @Before
52    public void setupMocks() {
53      conf = HBaseConfiguration.create();
54      HRegionInfo hri = new HRegionInfo(TABLENAME);
55      htd = new HTableDescriptor(TABLENAME);
56      mockRegion = Mockito.mock(HRegion.class);
57      Mockito.doReturn(htd).when(mockRegion).getTableDesc();
58      Mockito.doReturn(hri).when(mockRegion).getRegionInfo();
59      stores = new ArrayList<Store>();
60      Mockito.doReturn(stores).when(mockRegion).getStores();
61    }
62  
63    @Test
64    public void testForceSplitRegionWithReference() throws IOException {
65      htd.setMaxFileSize(1024L);
66      // Add a store above the requisite size. Should split.
67      HStore mockStore = Mockito.mock(HStore.class);
68      Mockito.doReturn(2000L).when(mockStore).getSize();
69      // Act as if there's a reference file or some other reason it can't split.
70      // This should prevent splitting even though it's big enough.
71      Mockito.doReturn(false).when(mockStore).canSplit();
72      stores.add(mockStore);
73  
74      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
75        ConstantSizeRegionSplitPolicy.class.getName());
76      ConstantSizeRegionSplitPolicy policy =
77          (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf);
78      assertFalse(policy.shouldSplit());
79      Mockito.doReturn(true).when(mockRegion).shouldForceSplit();
80      assertFalse(policy.shouldSplit());
81  
82      Mockito.doReturn(false).when(mockRegion).shouldForceSplit();
83      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
84        IncreasingToUpperBoundRegionSplitPolicy.class.getName());
85      policy = (IncreasingToUpperBoundRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
86      assertFalse(policy.shouldSplit());
87      Mockito.doReturn(true).when(mockRegion).shouldForceSplit();
88      assertFalse(policy.shouldSplit());
89    }
90  
91    @Test
92    public void testIncreasingToUpperBoundRegionSplitPolicy() throws IOException {
93      // Configure IncreasingToUpperBoundRegionSplitPolicy as our split policy
94      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
95        IncreasingToUpperBoundRegionSplitPolicy.class.getName());
96      // Now make it so the mock region has a RegionServerService that will
97      // return 'online regions'.
98      RegionServerServices rss = Mockito.mock(RegionServerServices.class);
99      final List<Region> regions = new ArrayList<Region>();
100     Mockito.when(rss.getOnlineRegions(TABLENAME)).thenReturn(regions);
101     Mockito.when(mockRegion.getRegionServerServices()).thenReturn(rss);
102     // Set max size for this 'table'.
103     long maxSplitSize = 1024L;
104     htd.setMaxFileSize(maxSplitSize);
105     // Set flush size to 1/8.  IncreasingToUpperBoundRegionSplitPolicy
106     // grows by the cube of the number of regions times flushsize each time.
107     long flushSize = maxSplitSize/8;
108     conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, flushSize);
109     htd.setMemStoreFlushSize(flushSize);
110     // If RegionServerService with no regions in it -- 'online regions' == 0 --
111     // then IncreasingToUpperBoundRegionSplitPolicy should act like a
112     // ConstantSizePolicy
113     IncreasingToUpperBoundRegionSplitPolicy policy =
114       (IncreasingToUpperBoundRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf);
115     doConstantSizePolicyTests(policy);
116 
117     // Add a store in excess of split size.  Because there are "no regions"
118     // on this server -- rss.getOnlineRegions is 0 -- then we should split
119     // like a constantsizeregionsplitpolicy would
120     HStore mockStore = Mockito.mock(HStore.class);
121     Mockito.doReturn(2000L).when(mockStore).getSize();
122     Mockito.doReturn(true).when(mockStore).canSplit();
123     stores.add(mockStore);
124     // It should split
125     assertTrue(policy.shouldSplit());
126 
127     // Now test that we increase our split size as online regions for a table
128     // grows. With one region, split size should be flushsize.
129     regions.add(mockRegion);
130     Mockito.doReturn(flushSize).when(mockStore).getSize();
131     // Should not split since store is flush size.
132     assertFalse(policy.shouldSplit());
133     // Set size of store to be > 2*flush size and we should split
134     Mockito.doReturn(flushSize*2 + 1).when(mockStore).getSize();
135     assertTrue(policy.shouldSplit());
136     // Add another region to the 'online regions' on this server and we should
137     // now be no longer be splittable since split size has gone up.
138     regions.add(mockRegion);
139     assertFalse(policy.shouldSplit());
140     // make sure its just over; verify it'll split
141     Mockito.doReturn((long)(maxSplitSize * 1.25 + 1)).when(mockStore).getSize();
142     assertTrue(policy.shouldSplit());
143 
144     // Finally assert that even if loads of regions, we'll split at max size
145     assertWithinJitter(maxSplitSize, policy.getSizeToCheck(1000));
146     // Assert same is true if count of regions is zero.
147     assertWithinJitter(maxSplitSize, policy.getSizeToCheck(0));
148   }
149 
150   private void assertWithinJitter(long maxSplitSize, long sizeToCheck) {
151     assertTrue("Size greater than lower bound of jitter",
152         (long)(maxSplitSize * 0.75) <= sizeToCheck);
153     assertTrue("Size less than upper bound of jitter",
154         (long)(maxSplitSize * 1.25) >= sizeToCheck);
155   }
156 
157   @Test
158   public void testCreateDefault() throws IOException {
159     conf.setLong(HConstants.HREGION_MAX_FILESIZE, 1234L);
160 
161     // Using a default HTD, should pick up the file size from
162     // configuration.
163     ConstantSizeRegionSplitPolicy policy =
164         (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(
165             mockRegion, conf);
166     assertWithinJitter(1234L, policy.getDesiredMaxFileSize());
167 
168     // If specified in HTD, should use that
169     htd.setMaxFileSize(9999L);
170     policy = (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(
171         mockRegion, conf);
172     assertWithinJitter(9999L, policy.getDesiredMaxFileSize());
173   }
174 
175   /**
176    * Test setting up a customized split policy
177    */
178   @Test
179   public void testCustomPolicy() throws IOException {
180     HTableDescriptor myHtd = new HTableDescriptor();
181     myHtd.setValue(HTableDescriptor.SPLIT_POLICY,
182         KeyPrefixRegionSplitPolicy.class.getName());
183     myHtd.setValue(KeyPrefixRegionSplitPolicy.PREFIX_LENGTH_KEY, String.valueOf(2));
184 
185     HRegion myMockRegion = Mockito.mock(HRegion.class);
186     Mockito.doReturn(myHtd).when(myMockRegion).getTableDesc();
187     Mockito.doReturn(stores).when(myMockRegion).getStores();
188 
189     HStore mockStore = Mockito.mock(HStore.class);
190     Mockito.doReturn(2000L).when(mockStore).getSize();
191     Mockito.doReturn(true).when(mockStore).canSplit();
192     Mockito.doReturn(Bytes.toBytes("abcd")).when(mockStore).getSplitPoint();
193     stores.add(mockStore);
194 
195     KeyPrefixRegionSplitPolicy policy = (KeyPrefixRegionSplitPolicy) RegionSplitPolicy
196         .create(myMockRegion, conf);
197 
198     assertEquals("ab", Bytes.toString(policy.getSplitPoint()));
199 
200     Mockito.doReturn(true).when(myMockRegion).shouldForceSplit();
201     Mockito.doReturn(Bytes.toBytes("efgh")).when(myMockRegion)
202         .getExplicitSplitPoint();
203 
204     policy = (KeyPrefixRegionSplitPolicy) RegionSplitPolicy
205         .create(myMockRegion, conf);
206 
207     assertEquals("ef", Bytes.toString(policy.getSplitPoint()));
208   }
209 
210   @Test
211   public void testConstantSizePolicy() throws IOException {
212     htd.setMaxFileSize(1024L);
213     ConstantSizeRegionSplitPolicy policy =
214       (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf);
215     doConstantSizePolicyTests(policy);
216   }
217 
218   /**
219    * Run through tests for a ConstantSizeRegionSplitPolicy
220    * @param policy
221    */
222   private void doConstantSizePolicyTests(final ConstantSizeRegionSplitPolicy policy) {
223     // For no stores, should not split
224     assertFalse(policy.shouldSplit());
225 
226     // Add a store above the requisite size. Should split.
227     HStore mockStore = Mockito.mock(HStore.class);
228     Mockito.doReturn(2000L).when(mockStore).getSize();
229     Mockito.doReturn(true).when(mockStore).canSplit();
230     stores.add(mockStore);
231 
232     assertTrue(policy.shouldSplit());
233 
234     // Act as if there's a reference file or some other reason it can't split.
235     // This should prevent splitting even though it's big enough.
236     Mockito.doReturn(false).when(mockStore).canSplit();
237     assertFalse(policy.shouldSplit());
238 
239     // Reset splittability after above
240     Mockito.doReturn(true).when(mockStore).canSplit();
241 
242     // Set to a small size but turn on forceSplit. Should result in a split.
243     Mockito.doReturn(true).when(mockRegion).shouldForceSplit();
244     Mockito.doReturn(100L).when(mockStore).getSize();
245     assertTrue(policy.shouldSplit());
246 
247     // Turn off forceSplit, should not split
248     Mockito.doReturn(false).when(mockRegion).shouldForceSplit();
249     assertFalse(policy.shouldSplit());
250 
251     // Clear families we added above
252     stores.clear();
253   }
254 
255   @Test
256   public void testGetSplitPoint() throws IOException {
257     ConstantSizeRegionSplitPolicy policy =
258       (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf);
259 
260     // For no stores, should not split
261     assertFalse(policy.shouldSplit());
262     assertNull(policy.getSplitPoint());
263 
264     // Add a store above the requisite size. Should split.
265     HStore mockStore = Mockito.mock(HStore.class);
266     Mockito.doReturn(2000L).when(mockStore).getSize();
267     Mockito.doReturn(true).when(mockStore).canSplit();
268     Mockito.doReturn(Bytes.toBytes("store 1 split"))
269       .when(mockStore).getSplitPoint();
270     stores.add(mockStore);
271 
272     assertEquals("store 1 split",
273         Bytes.toString(policy.getSplitPoint()));
274 
275     // Add a bigger store. The split point should come from that one
276     HStore mockStore2 = Mockito.mock(HStore.class);
277     Mockito.doReturn(4000L).when(mockStore2).getSize();
278     Mockito.doReturn(true).when(mockStore2).canSplit();
279     Mockito.doReturn(Bytes.toBytes("store 2 split"))
280       .when(mockStore2).getSplitPoint();
281     stores.add(mockStore2);
282 
283     assertEquals("store 2 split",
284         Bytes.toString(policy.getSplitPoint()));
285   }
286 
287   @Test
288   public void testDelimitedKeyPrefixRegionSplitPolicy() throws IOException {
289     HTableDescriptor myHtd = new HTableDescriptor();
290     myHtd.setValue(HTableDescriptor.SPLIT_POLICY,
291         DelimitedKeyPrefixRegionSplitPolicy.class.getName());
292     myHtd.setValue(DelimitedKeyPrefixRegionSplitPolicy.DELIMITER_KEY, ",");
293 
294     HRegion myMockRegion = Mockito.mock(HRegion.class);
295     Mockito.doReturn(myHtd).when(myMockRegion).getTableDesc();
296     Mockito.doReturn(stores).when(myMockRegion).getStores();
297 
298     HStore mockStore = Mockito.mock(HStore.class);
299     Mockito.doReturn(2000L).when(mockStore).getSize();
300     Mockito.doReturn(true).when(mockStore).canSplit();
301     Mockito.doReturn(Bytes.toBytes("ab,cd")).when(mockStore).getSplitPoint();
302     stores.add(mockStore);
303 
304     DelimitedKeyPrefixRegionSplitPolicy policy = (DelimitedKeyPrefixRegionSplitPolicy) RegionSplitPolicy
305         .create(myMockRegion, conf);
306 
307     assertEquals("ab", Bytes.toString(policy.getSplitPoint()));
308 
309     Mockito.doReturn(true).when(myMockRegion).shouldForceSplit();
310     Mockito.doReturn(Bytes.toBytes("efg,h")).when(myMockRegion)
311         .getExplicitSplitPoint();
312 
313     policy = (DelimitedKeyPrefixRegionSplitPolicy) RegionSplitPolicy
314         .create(myMockRegion, conf);
315 
316     assertEquals("efg", Bytes.toString(policy.getSplitPoint()));
317 
318     Mockito.doReturn(Bytes.toBytes("ijk")).when(myMockRegion)
319     .getExplicitSplitPoint();
320     assertEquals("ijk", Bytes.toString(policy.getSplitPoint()));
321   }
322 
323 }