1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertTrue;
24 import static org.mockito.Mockito.doReturn;
25 import static org.mockito.Mockito.when;
26
27 import java.io.IOException;
28 import java.util.ArrayList;
29 import java.util.List;
30
31 import org.apache.hadoop.fs.FileSystem;
32 import org.apache.hadoop.fs.Path;
33 import org.apache.hadoop.hbase.Cell;
34 import org.apache.hadoop.hbase.HBaseTestingUtility;
35 import org.apache.hadoop.hbase.HColumnDescriptor;
36 import org.apache.hadoop.hbase.HConstants;
37 import org.apache.hadoop.hbase.HRegionInfo;
38 import org.apache.hadoop.hbase.HTableDescriptor;
39 import org.apache.hadoop.hbase.Server;
40 import org.apache.hadoop.hbase.SmallTests;
41 import org.apache.hadoop.hbase.TableName;
42 import org.apache.hadoop.hbase.client.Durability;
43 import org.apache.hadoop.hbase.client.Put;
44 import org.apache.hadoop.hbase.client.Scan;
45 import org.apache.hadoop.hbase.regionserver.wal.HLog;
46 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
47 import org.apache.hadoop.hbase.util.Bytes;
48 import org.apache.zookeeper.KeeperException;
49 import org.junit.After;
50 import org.junit.Before;
51 import org.junit.Test;
52 import org.junit.experimental.categories.Category;
53 import org.mockito.Mockito;
54
55 import com.google.common.collect.ImmutableList;
56
57
58
59
60
61 @Category(SmallTests.class)
62 public class TestRegionMergeTransaction {
63 private final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
64 private final Path testdir = TEST_UTIL.getDataTestDir(this.getClass()
65 .getName());
66 private HRegion region_a;
67 private HRegion region_b;
68 private HRegion region_c;
69 private HLog wal;
70 private FileSystem fs;
71
72 private static final byte[] STARTROW_A = new byte[] { 'a', 'a', 'a' };
73 private static final byte[] STARTROW_B = new byte[] { 'g', 'g', 'g' };
74 private static final byte[] STARTROW_C = new byte[] { 'w', 'w', 'w' };
75 private static final byte[] ENDROW = new byte[] { '{', '{', '{' };
76 private static final byte[] CF = HConstants.CATALOG_FAMILY;
77
78 @Before
79 public void setup() throws IOException {
80 this.fs = FileSystem.get(TEST_UTIL.getConfiguration());
81 this.fs.delete(this.testdir, true);
82 this.wal = HLogFactory.createHLog(fs, this.testdir, "logs",
83 TEST_UTIL.getConfiguration());
84 this.region_a = createRegion(this.testdir, this.wal, STARTROW_A, STARTROW_B);
85 this.region_b = createRegion(this.testdir, this.wal, STARTROW_B, STARTROW_C);
86 this.region_c = createRegion(this.testdir, this.wal, STARTROW_C, ENDROW);
87 assert region_a != null && region_b != null && region_c != null;
88 TEST_UTIL.getConfiguration().setBoolean("hbase.testing.nocluster", true);
89 }
90
91 @After
92 public void teardown() throws IOException {
93 for (HRegion region : new HRegion[] { region_a, region_b, region_c }) {
94 if (region != null && !region.isClosed()) region.close();
95 if (this.fs.exists(region.getRegionFileSystem().getRegionDir())
96 && !this.fs.delete(region.getRegionFileSystem().getRegionDir(), true)) {
97 throw new IOException("Failed deleting of "
98 + region.getRegionFileSystem().getRegionDir());
99 }
100 }
101 if (this.wal != null)
102 this.wal.closeAndDelete();
103 this.fs.delete(this.testdir, true);
104 }
105
106
107
108
109
110
111 @Test
112 public void testPrepare() throws IOException {
113 prepareOnGoodRegions();
114 }
115
116 private RegionMergeTransaction prepareOnGoodRegions() throws IOException {
117 RegionMergeTransaction mt = new RegionMergeTransaction(region_a, region_b,
118 false);
119 RegionMergeTransaction spyMT = Mockito.spy(mt);
120 doReturn(false).when(spyMT).hasMergeQualifierInMeta(null,
121 region_a.getRegionName());
122 doReturn(false).when(spyMT).hasMergeQualifierInMeta(null,
123 region_b.getRegionName());
124 assertTrue(spyMT.prepare(null));
125 return spyMT;
126 }
127
128
129
130
131 @Test
132 public void testPrepareWithSameRegion() throws IOException {
133 RegionMergeTransaction mt = new RegionMergeTransaction(this.region_a,
134 this.region_a, true);
135 assertFalse("should not merge the same region even if it is forcible ",
136 mt.prepare(null));
137 }
138
139
140
141
142 @Test
143 public void testPrepareWithRegionsNotAdjacent() throws IOException {
144 RegionMergeTransaction mt = new RegionMergeTransaction(this.region_a,
145 this.region_c, false);
146 assertFalse("should not merge two regions if they are adjacent except it is forcible",
147 mt.prepare(null));
148 }
149
150
151
152
153 @Test
154 public void testPrepareWithRegionsNotAdjacentUnderCompulsory()
155 throws IOException {
156 RegionMergeTransaction mt = new RegionMergeTransaction(region_a, region_c,
157 true);
158 RegionMergeTransaction spyMT = Mockito.spy(mt);
159 doReturn(false).when(spyMT).hasMergeQualifierInMeta(null,
160 region_a.getRegionName());
161 doReturn(false).when(spyMT).hasMergeQualifierInMeta(null,
162 region_c.getRegionName());
163 assertTrue("Since focible is true, should merge two regions even if they are not adjacent",
164 spyMT.prepare(null));
165 }
166
167
168
169
170 @Test
171 public void testPrepareWithRegionsWithReference() throws IOException {
172 HStore storeMock = Mockito.mock(HStore.class);
173 when(storeMock.hasReferences()).thenReturn(true);
174 when(storeMock.getFamily()).thenReturn(new HColumnDescriptor("cf"));
175 when(storeMock.close()).thenReturn(ImmutableList.<StoreFile>of());
176 this.region_a.stores.put(Bytes.toBytes(""), storeMock);
177 RegionMergeTransaction mt = new RegionMergeTransaction(this.region_a,
178 this.region_b, false);
179 assertFalse(
180 "a region should not be mergeable if it has instances of store file references",
181 mt.prepare(null));
182 }
183
184 @Test
185 public void testPrepareWithClosedRegion() throws IOException {
186 this.region_a.close();
187 RegionMergeTransaction mt = new RegionMergeTransaction(this.region_a,
188 this.region_b, false);
189 assertFalse(mt.prepare(null));
190 }
191
192
193
194
195
196 @Test
197 public void testPrepareWithRegionsWithMergeReference() throws IOException {
198 RegionMergeTransaction mt = new RegionMergeTransaction(region_a, region_b,
199 false);
200 RegionMergeTransaction spyMT = Mockito.spy(mt);
201 doReturn(true).when(spyMT).hasMergeQualifierInMeta(null,
202 region_a.getRegionName());
203 doReturn(true).when(spyMT).hasMergeQualifierInMeta(null,
204 region_b.getRegionName());
205 assertFalse(spyMT.prepare(null));
206 }
207
208 @Test
209 public void testWholesomeMerge() throws IOException, InterruptedException {
210 final int rowCountOfRegionA = loadRegion(this.region_a, CF, true);
211 final int rowCountOfRegionB = loadRegion(this.region_b, CF, true);
212 assertTrue(rowCountOfRegionA > 0 && rowCountOfRegionB > 0);
213 assertEquals(rowCountOfRegionA, countRows(this.region_a));
214 assertEquals(rowCountOfRegionB, countRows(this.region_b));
215
216
217 RegionMergeTransaction mt = prepareOnGoodRegions();
218
219
220 TEST_UTIL.getConfiguration().setInt(HConstants.REGIONSERVER_PORT, 0);
221 Server mockServer = new HRegionServer(TEST_UTIL.getConfiguration());
222 HRegion mergedRegion = mt.execute(mockServer, null);
223
224 assertTrue(this.fs.exists(mt.getMergesDir()));
225
226 assertTrue(region_a.isClosed());
227 assertTrue(region_b.isClosed());
228
229
230
231 assertEquals(0, this.fs.listStatus(mt.getMergesDir()).length);
232
233 assertTrue(Bytes.equals(this.region_a.getStartKey(),
234 mergedRegion.getStartKey()));
235 assertTrue(Bytes.equals(this.region_b.getEndKey(),
236 mergedRegion.getEndKey()));
237
238 try {
239 int mergedRegionRowCount = countRows(mergedRegion);
240 assertEquals((rowCountOfRegionA + rowCountOfRegionB),
241 mergedRegionRowCount);
242 } finally {
243 HRegion.closeHRegion(mergedRegion);
244 }
245
246 assertTrue(!this.region_a.lock.writeLock().isHeldByCurrentThread());
247 assertTrue(!this.region_b.lock.writeLock().isHeldByCurrentThread());
248 }
249
250 @Test
251 public void testRollback() throws IOException, InterruptedException {
252 final int rowCountOfRegionA = loadRegion(this.region_a, CF, true);
253 final int rowCountOfRegionB = loadRegion(this.region_b, CF, true);
254 assertTrue(rowCountOfRegionA > 0 && rowCountOfRegionB > 0);
255 assertEquals(rowCountOfRegionA, countRows(this.region_a));
256 assertEquals(rowCountOfRegionB, countRows(this.region_b));
257
258
259 RegionMergeTransaction mt = prepareOnGoodRegions();
260
261 when(mt.createMergedRegionFromMerges(region_a, region_b,
262 mt.getMergedRegionInfo())).thenThrow(
263 new MockedFailedMergedRegionCreation());
264
265
266 boolean expectedException = false;
267 TEST_UTIL.getConfiguration().setInt(HConstants.REGIONSERVER_PORT, 0);
268 Server mockServer = new HRegionServer(TEST_UTIL.getConfiguration());
269 try {
270 mt.execute(mockServer, null);
271 } catch (MockedFailedMergedRegionCreation e) {
272 expectedException = true;
273 }
274 assertTrue(expectedException);
275
276 assertTrue(mt.rollback(null, null));
277
278
279 int rowCountOfRegionA2 = countRows(this.region_a);
280 assertEquals(rowCountOfRegionA, rowCountOfRegionA2);
281 int rowCountOfRegionB2 = countRows(this.region_b);
282 assertEquals(rowCountOfRegionB, rowCountOfRegionB2);
283
284
285 assertTrue(!this.fs.exists(HRegion.getRegionDir(this.testdir,
286 mt.getMergedRegionInfo())));
287
288 assertTrue(!this.region_a.lock.writeLock().isHeldByCurrentThread());
289 assertTrue(!this.region_b.lock.writeLock().isHeldByCurrentThread());
290
291
292 assertTrue(mt.prepare(null));
293 HRegion mergedRegion = mt.execute(mockServer, null);
294
295
296 try {
297 int mergedRegionRowCount = countRows(mergedRegion);
298 assertEquals((rowCountOfRegionA + rowCountOfRegionB),
299 mergedRegionRowCount);
300 } finally {
301 HRegion.closeHRegion(mergedRegion);
302 }
303
304 assertTrue(!this.region_a.lock.writeLock().isHeldByCurrentThread());
305 assertTrue(!this.region_b.lock.writeLock().isHeldByCurrentThread());
306 }
307
308 @Test
309 public void testFailAfterPONR() throws IOException, KeeperException, InterruptedException {
310 final int rowCountOfRegionA = loadRegion(this.region_a, CF, true);
311 final int rowCountOfRegionB = loadRegion(this.region_b, CF, true);
312 assertTrue(rowCountOfRegionA > 0 && rowCountOfRegionB > 0);
313 assertEquals(rowCountOfRegionA, countRows(this.region_a));
314 assertEquals(rowCountOfRegionB, countRows(this.region_b));
315
316
317 RegionMergeTransaction mt = prepareOnGoodRegions();
318 Mockito.doThrow(new MockedFailedMergedRegionOpen())
319 .when(mt)
320 .openMergedRegion((Server) Mockito.anyObject(),
321 (RegionServerServices) Mockito.anyObject(),
322 (HRegion) Mockito.anyObject());
323
324
325 boolean expectedException = false;
326 TEST_UTIL.getConfiguration().setInt(HConstants.REGIONSERVER_PORT, 0);
327 Server mockServer = new HRegionServer(TEST_UTIL.getConfiguration());
328 try {
329 mt.execute(mockServer, null);
330 } catch (MockedFailedMergedRegionOpen e) {
331 expectedException = true;
332 }
333 assertTrue(expectedException);
334
335 assertFalse(mt.rollback(null, null));
336
337
338
339 Path tableDir = this.region_a.getRegionFileSystem().getRegionDir()
340 .getParent();
341 Path mergedRegionDir = new Path(tableDir, mt.getMergedRegionInfo()
342 .getEncodedName());
343 assertTrue(TEST_UTIL.getTestFileSystem().exists(mergedRegionDir));
344 }
345
346 @Test
347 public void testMeregedRegionBoundary() {
348 TableName tableName =
349 TableName.valueOf("testMeregedRegionBoundary");
350 byte[] a = Bytes.toBytes("a");
351 byte[] b = Bytes.toBytes("b");
352 byte[] z = Bytes.toBytes("z");
353 HRegionInfo r1 = new HRegionInfo(tableName);
354 HRegionInfo r2 = new HRegionInfo(tableName, a, z);
355 HRegionInfo m = RegionMergeTransaction.getMergedRegionInfo(r1, r2);
356 assertTrue(Bytes.equals(m.getStartKey(), r1.getStartKey())
357 && Bytes.equals(m.getEndKey(), r1.getEndKey()));
358
359 r1 = new HRegionInfo(tableName, null, a);
360 r2 = new HRegionInfo(tableName, a, z);
361 m = RegionMergeTransaction.getMergedRegionInfo(r1, r2);
362 assertTrue(Bytes.equals(m.getStartKey(), r1.getStartKey())
363 && Bytes.equals(m.getEndKey(), r2.getEndKey()));
364
365 r1 = new HRegionInfo(tableName, null, a);
366 r2 = new HRegionInfo(tableName, z, null);
367 m = RegionMergeTransaction.getMergedRegionInfo(r1, r2);
368 assertTrue(Bytes.equals(m.getStartKey(), r1.getStartKey())
369 && Bytes.equals(m.getEndKey(), r2.getEndKey()));
370
371 r1 = new HRegionInfo(tableName, a, z);
372 r2 = new HRegionInfo(tableName, z, null);
373 m = RegionMergeTransaction.getMergedRegionInfo(r1, r2);
374 assertTrue(Bytes.equals(m.getStartKey(), r1.getStartKey())
375 && Bytes.equals(m.getEndKey(), r2.getEndKey()));
376
377 r1 = new HRegionInfo(tableName, a, b);
378 r2 = new HRegionInfo(tableName, b, z);
379 m = RegionMergeTransaction.getMergedRegionInfo(r1, r2);
380 assertTrue(Bytes.equals(m.getStartKey(), r1.getStartKey())
381 && Bytes.equals(m.getEndKey(), r2.getEndKey()));
382 }
383
384
385
386
387 @SuppressWarnings("serial")
388 private class MockedFailedMergedRegionCreation extends IOException {
389 }
390
391 @SuppressWarnings("serial")
392 private class MockedFailedMergedRegionOpen extends IOException {
393 }
394
395 private HRegion createRegion(final Path testdir, final HLog wal,
396 final byte[] startrow, final byte[] endrow)
397 throws IOException {
398
399 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("table"));
400 HColumnDescriptor hcd = new HColumnDescriptor(CF);
401 htd.addFamily(hcd);
402 HRegionInfo hri = new HRegionInfo(htd.getTableName(), startrow, endrow);
403 HRegion a = HRegion.createHRegion(hri, testdir,
404 TEST_UTIL.getConfiguration(), htd);
405 HRegion.closeHRegion(a);
406 return HRegion.openHRegion(testdir, hri, htd, wal,
407 TEST_UTIL.getConfiguration());
408 }
409
410 private int countRows(final HRegion r) throws IOException {
411 int rowcount = 0;
412 InternalScanner scanner = r.getScanner(new Scan());
413 try {
414 List<Cell> kvs = new ArrayList<Cell>();
415 boolean hasNext = true;
416 while (hasNext) {
417 hasNext = scanner.next(kvs);
418 if (!kvs.isEmpty())
419 rowcount++;
420 }
421 } finally {
422 scanner.close();
423 }
424 return rowcount;
425 }
426
427
428
429
430
431
432
433
434
435
436 private int loadRegion(final HRegion r, final byte[] f, final boolean flush)
437 throws IOException {
438 byte[] k = new byte[3];
439 int rowCount = 0;
440 for (byte b1 = 'a'; b1 <= 'z'; b1++) {
441 for (byte b2 = 'a'; b2 <= 'z'; b2++) {
442 for (byte b3 = 'a'; b3 <= 'z'; b3++) {
443 k[0] = b1;
444 k[1] = b2;
445 k[2] = b3;
446 if (!HRegion.rowIsInRange(r.getRegionInfo(), k)) {
447 continue;
448 }
449 Put put = new Put(k);
450 put.add(f, null, k);
451 if (r.getLog() == null)
452 put.setDurability(Durability.SKIP_WAL);
453 r.put(put);
454 rowCount++;
455 }
456 }
457 if (flush) {
458 r.flushcache();
459 }
460 }
461 return rowCount;
462 }
463
464 }