1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master.normalizer;
20
21 import org.apache.commons.logging.Log;
22 import org.apache.commons.logging.LogFactory;
23 import org.apache.hadoop.hbase.HBaseTestingUtility;
24 import org.apache.hadoop.hbase.HConstants;
25 import org.apache.hadoop.hbase.HTableDescriptor;
26 import org.apache.hadoop.hbase.MetaTableAccessor;
27 import org.apache.hadoop.hbase.MiniHBaseCluster;
28 import org.apache.hadoop.hbase.TableName;
29 import org.apache.hadoop.hbase.client.Admin;
30 import org.apache.hadoop.hbase.client.HTable;
31 import org.apache.hadoop.hbase.client.Put;
32 import org.apache.hadoop.hbase.master.HMaster;
33 import org.apache.hadoop.hbase.regionserver.HRegion;
34 import org.apache.hadoop.hbase.regionserver.Region;
35 import org.apache.hadoop.hbase.testclassification.MediumTests;
36 import org.apache.hadoop.hbase.util.Bytes;
37 import org.apache.hadoop.hbase.util.test.LoadTestKVGenerator;
38 import org.junit.AfterClass;
39 import org.junit.BeforeClass;
40 import org.junit.Test;
41 import org.junit.experimental.categories.Category;
42
43 import java.io.IOException;
44 import java.util.Collections;
45 import java.util.Comparator;
46 import java.util.List;
47
48 import static org.junit.Assert.assertEquals;
49
50
51
52
53 @Category(MediumTests.class)
54 public class TestSimpleRegionNormalizerOnCluster {
55 private static final Log LOG = LogFactory.getLog(TestSimpleRegionNormalizerOnCluster.class);
56 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
57 private static final byte[] FAMILYNAME = Bytes.toBytes("fam");
58 private static Admin admin;
59
60 @BeforeClass
61 public static void beforeAllTests() throws Exception {
62
63 TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3);
64
65
66 TEST_UTIL.startMiniCluster(1);
67 admin = TEST_UTIL.getHBaseAdmin();
68 }
69
70 @AfterClass
71 public static void afterAllTests() throws Exception {
72 TEST_UTIL.shutdownMiniCluster();
73 }
74
75 @Test(timeout = 60000)
76 @SuppressWarnings("deprecation")
77 public void testRegionNormalizationSplitOnCluster() throws Exception {
78 final TableName TABLENAME =
79 TableName.valueOf("testRegionNormalizationSplitOnCluster");
80 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
81 HMaster m = cluster.getMaster();
82
83 try (HTable ht = TEST_UTIL.createMultiRegionTable(TABLENAME, FAMILYNAME, 5)) {
84
85 List<HRegion> generatedRegions = TEST_UTIL.getHBaseCluster().getRegions(TABLENAME);
86 Collections.sort(generatedRegions, new Comparator<HRegion>() {
87 @Override
88 public int compare(HRegion o1, HRegion o2) {
89 return o1.getRegionInfo().compareTo(o2.getRegionInfo());
90 }
91 });
92
93 HRegion region = generatedRegions.get(0);
94 generateTestData(region, 1);
95 region.flush(true);
96
97 region = generatedRegions.get(1);
98 generateTestData(region, 1);
99 region.flush(true);
100
101 region = generatedRegions.get(2);
102 generateTestData(region, 2);
103 region.flush(true);
104
105 region = generatedRegions.get(3);
106 generateTestData(region, 2);
107 region.flush(true);
108
109 region = generatedRegions.get(4);
110 generateTestData(region, 5);
111 region.flush(true);
112 }
113
114 HTableDescriptor htd = admin.getTableDescriptor(TABLENAME);
115 htd.setNormalizationEnabled(true);
116 admin.modifyTable(TABLENAME, htd);
117
118 admin.flush(TABLENAME);
119
120 System.out.println(admin.getTableDescriptor(TABLENAME));
121
122 assertEquals(5, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME));
123
124
125 Thread.sleep(5000);
126 m.normalizeRegions();
127
128 while (true) {
129 List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(TABLENAME);
130 int cnt = 0;
131 for (HRegion region : regions) {
132 String regionName = region.getRegionInfo().getRegionNameAsString();
133 if (regionName.startsWith("testRegionNormalizationSplitOnCluster,zzzzz")) {
134 cnt++;
135 }
136 }
137 if (cnt >= 2) {
138 break;
139 }
140 }
141 admin.disableTable(TABLENAME);
142 admin.deleteTable(TABLENAME);
143 }
144
145 @Test(timeout = 60000)
146 @SuppressWarnings("deprecation")
147 public void testRegionNormalizationMergeOnCluster() throws Exception {
148 final TableName TABLENAME =
149 TableName.valueOf("testRegionNormalizationMergeOnCluster");
150 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
151 HMaster m = cluster.getMaster();
152
153
154 try (HTable ht = TEST_UTIL.createMultiRegionTable(TABLENAME, FAMILYNAME, 5)) {
155
156 List<HRegion> generatedRegions = TEST_UTIL.getHBaseCluster().getRegions(TABLENAME);
157 Collections.sort(generatedRegions, new Comparator<HRegion>() {
158 @Override
159 public int compare(HRegion o1, HRegion o2) {
160 return o1.getRegionInfo().compareTo(o2.getRegionInfo());
161 }
162 });
163
164 HRegion region = generatedRegions.get(0);
165 generateTestData(region, 1);
166 region.flush(true);
167
168 region = generatedRegions.get(1);
169 generateTestData(region, 1);
170 region.flush(true);
171
172 region = generatedRegions.get(2);
173 generateTestData(region, 3);
174 region.flush(true);
175
176 region = generatedRegions.get(3);
177 generateTestData(region, 3);
178 region.flush(true);
179
180 region = generatedRegions.get(4);
181 generateTestData(region, 5);
182 region.flush(true);
183 }
184
185 HTableDescriptor htd = admin.getTableDescriptor(TABLENAME);
186 htd.setNormalizationEnabled(true);
187 admin.modifyTable(TABLENAME, htd);
188
189 admin.flush(TABLENAME);
190
191 assertEquals(5, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME));
192
193
194 Thread.sleep(5000);
195 m.normalizeRegions();
196
197 while (MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME) > 4) {
198 LOG.info("Waiting for normalization merge to complete");
199 Thread.sleep(100);
200 }
201
202 assertEquals(4, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME));
203
204 admin.disableTable(TABLENAME);
205 admin.deleteTable(TABLENAME);
206 }
207
208 private void generateTestData(Region region, int numRows) throws IOException {
209
210 LoadTestKVGenerator dataGenerator = new LoadTestKVGenerator(1024 * 1024, 1024 * 1024);
211 for (int i = 0; i < numRows; ++i) {
212 byte[] key = Bytes.add(region.getRegionInfo().getStartKey(), Bytes.toBytes(i));
213 for (int j = 0; j < 1; ++j) {
214 Put put = new Put(key);
215 byte[] col = Bytes.toBytes(String.valueOf(j));
216 byte[] value = dataGenerator.generateRandomSizeValue(key, col);
217 put.add(FAMILYNAME, col, value);
218 region.put(put);
219 }
220 }
221 }
222 }