View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master.normalizer;
20  
21  import org.apache.commons.logging.Log;
22  import org.apache.commons.logging.LogFactory;
23  import org.apache.hadoop.hbase.HBaseTestingUtility;
24  import org.apache.hadoop.hbase.HConstants;
25  import org.apache.hadoop.hbase.HTableDescriptor;
26  import org.apache.hadoop.hbase.MetaTableAccessor;
27  import org.apache.hadoop.hbase.MiniHBaseCluster;
28  import org.apache.hadoop.hbase.TableName;
29  import org.apache.hadoop.hbase.client.Admin;
30  import org.apache.hadoop.hbase.client.HTable;
31  import org.apache.hadoop.hbase.client.Put;
32  import org.apache.hadoop.hbase.master.HMaster;
33  import org.apache.hadoop.hbase.regionserver.HRegion;
34  import org.apache.hadoop.hbase.regionserver.Region;
35  import org.apache.hadoop.hbase.testclassification.MediumTests;
36  import org.apache.hadoop.hbase.util.Bytes;
37  import org.apache.hadoop.hbase.util.test.LoadTestKVGenerator;
38  import org.junit.AfterClass;
39  import org.junit.BeforeClass;
40  import org.junit.Test;
41  import org.junit.experimental.categories.Category;
42  
43  import java.io.IOException;
44  import java.util.Collections;
45  import java.util.Comparator;
46  import java.util.List;
47  
48  import static org.junit.Assert.assertEquals;
49  
50  /**
51   * Testing {@link SimpleRegionNormalizer} on minicluster.
52   */
53  @Category(MediumTests.class)
54  public class TestSimpleRegionNormalizerOnCluster {
55    private static final Log LOG = LogFactory.getLog(TestSimpleRegionNormalizerOnCluster.class);
56    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
57    private static final byte[] FAMILYNAME = Bytes.toBytes("fam");
58    private static Admin admin;
59  
60    @BeforeClass
61    public static void beforeAllTests() throws Exception {
62      // we will retry operations when PleaseHoldException is thrown
63      TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3);
64  
65      // Start a cluster of two regionservers.
66      TEST_UTIL.startMiniCluster(1);
67      admin = TEST_UTIL.getHBaseAdmin();
68    }
69  
70    @AfterClass
71    public static void afterAllTests() throws Exception {
72      TEST_UTIL.shutdownMiniCluster();
73    }
74  
75    @Test(timeout = 60000)
76    @SuppressWarnings("deprecation")
77    public void testRegionNormalizationSplitOnCluster() throws Exception {
78      final TableName TABLENAME =
79        TableName.valueOf("testRegionNormalizationSplitOnCluster");
80      MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
81      HMaster m = cluster.getMaster();
82  
83      try (HTable ht = TEST_UTIL.createMultiRegionTable(TABLENAME, FAMILYNAME, 5)) {
84        // Need to get sorted list of regions here
85        List<HRegion> generatedRegions = TEST_UTIL.getHBaseCluster().getRegions(TABLENAME);
86        Collections.sort(generatedRegions, new Comparator<HRegion>() {
87          @Override
88          public int compare(HRegion o1, HRegion o2) {
89            return o1.getRegionInfo().compareTo(o2.getRegionInfo());
90          }
91        });
92  
93        HRegion region = generatedRegions.get(0);
94        generateTestData(region, 1);
95        region.flush(true);
96  
97        region = generatedRegions.get(1);
98        generateTestData(region, 1);
99        region.flush(true);
100 
101       region = generatedRegions.get(2);
102       generateTestData(region, 2);
103       region.flush(true);
104 
105       region = generatedRegions.get(3);
106       generateTestData(region, 2);
107       region.flush(true);
108 
109       region = generatedRegions.get(4);
110       generateTestData(region, 5);
111       region.flush(true);
112     }
113 
114     HTableDescriptor htd = admin.getTableDescriptor(TABLENAME);
115     htd.setNormalizationEnabled(true);
116     admin.modifyTable(TABLENAME, htd);
117 
118     admin.flush(TABLENAME);
119 
120     System.out.println(admin.getTableDescriptor(TABLENAME));
121 
122     assertEquals(5, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME));
123 
124     // Now trigger a split and stop when the split is in progress
125     Thread.sleep(5000); // to let region load to update
126     m.normalizeRegions();
127 
128     while (true) {
129       List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(TABLENAME);
130       int cnt = 0;
131       for (HRegion region : regions) {
132         String regionName = region.getRegionInfo().getRegionNameAsString();
133         if (regionName.startsWith("testRegionNormalizationSplitOnCluster,zzzzz")) {
134           cnt++;
135         }
136       }
137       if (cnt >= 2) {
138         break;
139       }
140     }
141     admin.disableTable(TABLENAME);
142     admin.deleteTable(TABLENAME);
143   }
144 
145   @Test(timeout = 60000)
146   @SuppressWarnings("deprecation")
147   public void testRegionNormalizationMergeOnCluster() throws Exception {
148     final TableName TABLENAME =
149       TableName.valueOf("testRegionNormalizationMergeOnCluster");
150     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
151     HMaster m = cluster.getMaster();
152 
153     // create 5 regions with sizes to trigger merge of small regions
154     try (HTable ht = TEST_UTIL.createMultiRegionTable(TABLENAME, FAMILYNAME, 5)) {
155       // Need to get sorted list of regions here
156       List<HRegion> generatedRegions = TEST_UTIL.getHBaseCluster().getRegions(TABLENAME);
157       Collections.sort(generatedRegions, new Comparator<HRegion>() {
158         @Override
159         public int compare(HRegion o1, HRegion o2) {
160           return o1.getRegionInfo().compareTo(o2.getRegionInfo());
161         }
162       });
163 
164       HRegion region = generatedRegions.get(0);
165       generateTestData(region, 1);
166       region.flush(true);
167 
168       region = generatedRegions.get(1);
169       generateTestData(region, 1);
170       region.flush(true);
171 
172       region = generatedRegions.get(2);
173       generateTestData(region, 3);
174       region.flush(true);
175 
176       region = generatedRegions.get(3);
177       generateTestData(region, 3);
178       region.flush(true);
179 
180       region = generatedRegions.get(4);
181       generateTestData(region, 5);
182       region.flush(true);
183     }
184 
185     HTableDescriptor htd = admin.getTableDescriptor(TABLENAME);
186     htd.setNormalizationEnabled(true);
187     admin.modifyTable(TABLENAME, htd);
188 
189     admin.flush(TABLENAME);
190 
191     assertEquals(5, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME));
192 
193     // Now trigger a merge and stop when the merge is in progress
194     Thread.sleep(5000); // to let region load to update
195     m.normalizeRegions();
196 
197     while (MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME) > 4) {
198       LOG.info("Waiting for normalization merge to complete");
199       Thread.sleep(100);
200     }
201 
202     assertEquals(4, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME));
203 
204     admin.disableTable(TABLENAME);
205     admin.deleteTable(TABLENAME);
206   }
207 
208   private void generateTestData(Region region, int numRows) throws IOException {
209     // generating 1Mb values
210     LoadTestKVGenerator dataGenerator = new LoadTestKVGenerator(1024 * 1024, 1024 * 1024);
211     for (int i = 0; i < numRows; ++i) {
212       byte[] key = Bytes.add(region.getRegionInfo().getStartKey(), Bytes.toBytes(i));
213       for (int j = 0; j < 1; ++j) {
214         Put put = new Put(key);
215         byte[] col = Bytes.toBytes(String.valueOf(j));
216         byte[] value = dataGenerator.generateRandomSizeValue(key, col);
217         put.add(FAMILYNAME, col, value);
218         region.put(put);
219       }
220     }
221   }
222 }