1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master;
20
21 import static org.apache.hadoop.hbase.util.HFileArchiveTestingUtil.assertArchiveEqualToOriginal;
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertFalse;
24 import static org.junit.Assert.assertTrue;
25 import static org.mockito.Mockito.doReturn;
26 import static org.mockito.Mockito.spy;
27
28 import java.io.IOException;
29 import java.util.List;
30 import java.util.Map;
31 import java.util.SortedMap;
32 import java.util.TreeMap;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FSDataOutputStream;
38 import org.apache.hadoop.fs.FileStatus;
39 import org.apache.hadoop.fs.FileSystem;
40 import org.apache.hadoop.fs.Path;
41 import org.apache.hadoop.hbase.TableName;
42 import org.apache.hadoop.hbase.HBaseTestingUtility;
43 import org.apache.hadoop.hbase.HColumnDescriptor;
44 import org.apache.hadoop.hbase.HConstants;
45 import org.apache.hadoop.hbase.HRegionInfo;
46 import org.apache.hadoop.hbase.HTableDescriptor;
47 import org.apache.hadoop.hbase.NamespaceDescriptor;
48 import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
49 import org.apache.hadoop.hbase.Server;
50 import org.apache.hadoop.hbase.ServerName;
51 import org.apache.hadoop.hbase.SmallTests;
52 import org.apache.hadoop.hbase.TableDescriptors;
53 import org.apache.hadoop.hbase.catalog.CatalogTracker;
54 import org.apache.hadoop.hbase.catalog.MetaMockingUtil;
55 import org.apache.hadoop.hbase.client.HConnection;
56 import org.apache.hadoop.hbase.client.HConnectionManager;
57 import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
58 import org.apache.hadoop.hbase.client.Result;
59 import org.apache.hadoop.hbase.executor.ExecutorService;
60 import org.apache.hadoop.hbase.io.Reference;
61 import org.apache.hadoop.hbase.master.CatalogJanitor.SplitParentFirstComparator;
62 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
63 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
64 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
65 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutateRequest;
66 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutateResponse;
67 import org.apache.hadoop.hbase.regionserver.HStore;
68 import org.apache.hadoop.hbase.util.Bytes;
69 import org.apache.hadoop.hbase.util.FSUtils;
70 import org.apache.hadoop.hbase.util.HFileArchiveUtil;
71 import org.apache.hadoop.hbase.util.Triple;
72 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
73 import org.junit.Test;
74 import org.junit.experimental.categories.Category;
75 import org.mockito.Mockito;
76
77 import com.google.protobuf.RpcController;
78 import com.google.protobuf.Service;
79 import com.google.protobuf.ServiceException;
80
81 @Category(SmallTests.class)
82 public class TestCatalogJanitor {
83 private static final Log LOG = LogFactory.getLog(TestCatalogJanitor.class);
84
85
86
87
88
89 class MockServer implements Server {
90 private final HConnection connection;
91 private final Configuration c;
92 private final CatalogTracker ct;
93
94 MockServer(final HBaseTestingUtility htu)
95 throws NotAllMetaRegionsOnlineException, IOException, InterruptedException {
96 this.c = htu.getConfiguration();
97 ClientProtos.ClientService.BlockingInterface ri =
98 Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
99 MutateResponse.Builder builder = MutateResponse.newBuilder();
100 builder.setProcessed(true);
101 try {
102 Mockito.when(ri.mutate(
103 (RpcController)Mockito.any(), (MutateRequest)Mockito.any())).
104 thenReturn(builder.build());
105 } catch (ServiceException se) {
106 throw ProtobufUtil.getRemoteException(se);
107 }
108
109
110
111 this.connection =
112 HConnectionTestingUtility.getMockedConnectionAndDecorate(this.c,
113 Mockito.mock(AdminProtos.AdminService.BlockingInterface.class), ri,
114 ServerName.valueOf("example.org,12345,6789"),
115 HRegionInfo.FIRST_META_REGIONINFO);
116
117 FileSystem fs = FileSystem.get(this.c);
118 Path rootdir = FSUtils.getRootDir(this.c);
119 FSUtils.setRootDir(this.c, rootdir);
120 this.ct = Mockito.mock(CatalogTracker.class);
121 AdminProtos.AdminService.BlockingInterface hri =
122 Mockito.mock(AdminProtos.AdminService.BlockingInterface.class);
123 Mockito.when(this.ct.getConnection()).thenReturn(this.connection);
124 Mockito.when(ct.waitForMetaServerConnection(Mockito.anyLong())).thenReturn(hri);
125 }
126
127 @Override
128 public CatalogTracker getCatalogTracker() {
129 return this.ct;
130 }
131
132 @Override
133 public Configuration getConfiguration() {
134 return this.c;
135 }
136
137 @Override
138 public ServerName getServerName() {
139 return ServerName.valueOf("mockserver.example.org", 1234, -1L);
140 }
141
142 @Override
143 public ZooKeeperWatcher getZooKeeper() {
144 return null;
145 }
146
147 @Override
148 public void abort(String why, Throwable e) {
149
150 }
151
152 @Override
153 public boolean isAborted() {
154 return false;
155 }
156
157 @Override
158 public boolean isStopped() {
159 return false;
160 }
161
162 @Override
163 public void stop(String why) {
164 if (this.ct != null) {
165 this.ct.stop();
166 }
167 if (this.connection != null) {
168 HConnectionManager.deleteConnection(this.connection.getConfiguration());
169 }
170 }
171 }
172
173
174
175
176 class MockMasterServices implements MasterServices {
177 private final MasterFileSystem mfs;
178 private final AssignmentManager asm;
179
180 MockMasterServices(final Server server) throws IOException {
181 this.mfs = new MasterFileSystem(server, this, false);
182 this.asm = Mockito.mock(AssignmentManager.class);
183 }
184
185 @Override
186 public void checkTableModifiable(TableName tableName) throws IOException {
187
188 }
189
190 @Override
191 public void createTable(HTableDescriptor desc, byte[][] splitKeys)
192 throws IOException {
193
194 }
195
196 @Override
197 public AssignmentManager getAssignmentManager() {
198 return this.asm;
199 }
200
201 @Override
202 public ExecutorService getExecutorService() {
203 return null;
204 }
205
206 @Override
207 public MasterFileSystem getMasterFileSystem() {
208 return this.mfs;
209 }
210
211 @Override
212 public MasterCoprocessorHost getCoprocessorHost() {
213 return null;
214 }
215
216 @Override
217 public ServerManager getServerManager() {
218 return null;
219 }
220
221 @Override
222 public ZooKeeperWatcher getZooKeeper() {
223 return null;
224 }
225
226 @Override
227 public CatalogTracker getCatalogTracker() {
228 return null;
229 }
230
231 @Override
232 public Configuration getConfiguration() {
233 return mfs.conf;
234 }
235
236 @Override
237 public ServerName getServerName() {
238 return null;
239 }
240
241 @Override
242 public void abort(String why, Throwable e) {
243
244 }
245
246 @Override
247 public boolean isAborted() {
248 return false;
249 }
250
251 private boolean stopped = false;
252
253 @Override
254 public void stop(String why) {
255 stopped = true;
256 }
257
258 @Override
259 public boolean isStopped() {
260 return stopped;
261 }
262
263 @Override
264 public TableDescriptors getTableDescriptors() {
265 return new TableDescriptors() {
266 @Override
267 public HTableDescriptor remove(TableName tablename) throws IOException {
268
269 return null;
270 }
271
272 @Override
273 public Map<String, HTableDescriptor> getAll() throws IOException {
274
275 return null;
276 }
277
278 @Override
279 public HTableDescriptor get(TableName tablename)
280 throws IOException {
281 return createHTableDescriptor();
282 }
283
284 @Override
285 public Map<String, HTableDescriptor> getByNamespace(String name) throws IOException {
286 return null;
287 }
288
289 @Override
290 public void add(HTableDescriptor htd) throws IOException {
291
292
293 }
294 };
295 }
296
297 @Override
298 public boolean isServerShutdownHandlerEnabled() {
299 return true;
300 }
301
302 @Override
303 public boolean registerService(Service instance) {
304 return false;
305 }
306
307 @Override
308 public void createNamespace(NamespaceDescriptor descriptor) throws IOException {
309
310 }
311
312 @Override
313 public void modifyNamespace(NamespaceDescriptor descriptor) throws IOException {
314
315 }
316
317 @Override
318 public void deleteNamespace(String name) throws IOException {
319
320 }
321
322 @Override
323 public NamespaceDescriptor getNamespaceDescriptor(String name) throws IOException {
324 return null;
325 }
326
327 @Override
328 public List<NamespaceDescriptor> listNamespaceDescriptors() throws IOException {
329 return null;
330 }
331
332 @Override
333 public List<HTableDescriptor> listTableDescriptorsByNamespace(String name) throws IOException {
334 return null;
335 }
336
337 @Override
338 public List<TableName> listTableNamesByNamespace(String name) throws IOException {
339 return null;
340 }
341
342 @Override
343 public void deleteTable(TableName tableName) throws IOException { }
344
345 @Override
346 public void modifyTable(TableName tableName, HTableDescriptor descriptor)
347 throws IOException { }
348
349 @Override
350 public void enableTable(TableName tableName) throws IOException { }
351
352 @Override
353 public void disableTable(TableName tableName) throws IOException { }
354
355 @Override
356 public void addColumn(TableName tableName, HColumnDescriptor column)
357 throws IOException { }
358
359 @Override
360 public void modifyColumn(TableName tableName, HColumnDescriptor descriptor)
361 throws IOException { }
362
363 @Override
364 public void deleteColumn(TableName tableName, byte[] columnName)
365 throws IOException { }
366
367 @Override
368 public TableLockManager getTableLockManager() {
369 return null;
370 }
371
372 @Override
373 public void dispatchMergingRegions(HRegionInfo region_a, HRegionInfo region_b,
374 boolean forcible) throws IOException {
375 }
376
377 @Override
378 public boolean isInitialized() {
379
380 return false;
381 }
382 }
383
384 @Test
385 public void testCleanParent() throws IOException, InterruptedException {
386 HBaseTestingUtility htu = new HBaseTestingUtility();
387 setRootDirAndCleanIt(htu, "testCleanParent");
388 Server server = new MockServer(htu);
389 try {
390 MasterServices services = new MockMasterServices(server);
391 CatalogJanitor janitor = new CatalogJanitor(server, services);
392
393 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("table"));
394 htd.addFamily(new HColumnDescriptor("f"));
395 HRegionInfo parent =
396 new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
397 Bytes.toBytes("eee"));
398 HRegionInfo splita =
399 new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
400 Bytes.toBytes("ccc"));
401 HRegionInfo splitb =
402 new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
403 Bytes.toBytes("eee"));
404
405
406 Result r = createResult(parent, splita, splitb);
407
408 Path rootdir = services.getMasterFileSystem().getRootDir();
409 Path tabledir =
410 FSUtils.getTableDir(rootdir, htd.getTableName());
411 Path storedir = HStore.getStoreHomedir(tabledir, splita,
412 htd.getColumnFamilies()[0].getName());
413 Reference ref = Reference.createTopReference(Bytes.toBytes("ccc"));
414 long now = System.currentTimeMillis();
415
416 Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
417 FileSystem fs = services.getMasterFileSystem().getFileSystem();
418 Path path = ref.write(fs, p);
419 assertTrue(fs.exists(path));
420 assertFalse(janitor.cleanParent(parent, r));
421
422 assertTrue(fs.delete(p, true));
423 assertTrue(janitor.cleanParent(parent, r));
424 } finally {
425 server.stop("shutdown");
426 }
427 }
428
429
430
431
432
433
434 @Test
435 public void testParentCleanedEvenIfDaughterGoneFirst()
436 throws IOException, InterruptedException {
437 parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
438 "testParentCleanedEvenIfDaughterGoneFirst", Bytes.toBytes("eee"));
439 }
440
441
442
443
444
445
446 @Test
447 public void testLastParentCleanedEvenIfDaughterGoneFirst()
448 throws IOException, InterruptedException {
449 parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
450 "testLastParentCleanedEvenIfDaughterGoneFirst", new byte[0]);
451 }
452
453
454
455
456
457
458
459
460
461 private void parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
462 final String rootDir, final byte[] lastEndKey)
463 throws IOException, InterruptedException {
464 HBaseTestingUtility htu = new HBaseTestingUtility();
465 setRootDirAndCleanIt(htu, rootDir);
466 Server server = new MockServer(htu);
467 MasterServices services = new MockMasterServices(server);
468 CatalogJanitor janitor = new CatalogJanitor(server, services);
469 final HTableDescriptor htd = createHTableDescriptor();
470
471
472
473
474 HRegionInfo parent = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
475 lastEndKey);
476
477
478 Thread.sleep(1001);
479
480
481 HRegionInfo splita = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
482 Bytes.toBytes("ccc"));
483 Thread.sleep(1001);
484
485 HRegionInfo splitaa = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
486 Bytes.toBytes("bbb"));
487 HRegionInfo splitab = new HRegionInfo(htd.getTableName(), Bytes.toBytes("bbb"),
488 Bytes.toBytes("ccc"));
489
490
491 HRegionInfo splitb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
492 lastEndKey);
493 Thread.sleep(1001);
494
495 HRegionInfo splitba = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
496 Bytes.toBytes("ddd"));
497 HRegionInfo splitbb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ddd"),
498 lastEndKey);
499
500
501
502 SortedMap<HRegionInfo, Result> regions =
503 new TreeMap<HRegionInfo, Result>(new CatalogJanitor.SplitParentFirstComparator());
504
505 regions.put(parent, createResult(parent, splita, splitb));
506 regions.put(splitb, createResult(splitb, splitba, splitbb));
507 regions.put(splita, createResult(splita, splitaa, splitab));
508
509 int index = 0;
510 for (Map.Entry<HRegionInfo, Result> e: regions.entrySet()) {
511 if (index == 0) {
512 assertTrue(e.getKey().getEncodedName().equals(parent.getEncodedName()));
513 } else if (index == 1) {
514 assertTrue(e.getKey().getEncodedName().equals(splita.getEncodedName()));
515 } else if (index == 2) {
516 assertTrue(e.getKey().getEncodedName().equals(splitb.getEncodedName()));
517 }
518 index++;
519 }
520
521
522
523 Path splitaRef =
524 createReferences(services, htd, parent, splita, Bytes.toBytes("ccc"), false);
525
526 assertFalse(janitor.cleanParent(parent, regions.get(parent)));
527
528
529
530 assertTrue(janitor.cleanParent(splitb, regions.get(splitb)));
531
532
533
534
535
536 FileSystem fs = FileSystem.get(htu.getConfiguration());
537 assertTrue(fs.delete(splitaRef, true));
538
539 Path splitaaRef =
540 createReferences(services, htd, splita, splitaa, Bytes.toBytes("bbb"), false);
541 Path splitabRef =
542 createReferences(services, htd, splita, splitab, Bytes.toBytes("bbb"), true);
543
544
545 assertFalse(janitor.cleanParent(splita, regions.get(splita)));
546
547
548 assertTrue(fs.delete(splitaaRef, true));
549 assertTrue(fs.delete(splitabRef, true));
550 assertTrue(janitor.cleanParent(splita, regions.get(splita)));
551
552
553 assertTrue(janitor.cleanParent(parent, regions.get(parent)));
554
555 services.stop("test finished");
556 janitor.join();
557 }
558
559
560
561
562
563
564 @Test
565 public void testScanDoesNotCleanRegionsWithExistingParents() throws Exception {
566 HBaseTestingUtility htu = new HBaseTestingUtility();
567 setRootDirAndCleanIt(htu, "testScanDoesNotCleanRegionsWithExistingParents");
568 Server server = new MockServer(htu);
569 MasterServices services = new MockMasterServices(server);
570
571 final HTableDescriptor htd = createHTableDescriptor();
572
573
574
575
576 HRegionInfo parent = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
577 new byte[0], true);
578
579
580 Thread.sleep(1001);
581
582
583 HRegionInfo splita = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
584 Bytes.toBytes("ccc"), true);
585 Thread.sleep(1001);
586
587 HRegionInfo splitaa = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
588 Bytes.toBytes("bbb"), false);
589 HRegionInfo splitab = new HRegionInfo(htd.getTableName(), Bytes.toBytes("bbb"),
590 Bytes.toBytes("ccc"), false);
591
592
593 HRegionInfo splitb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
594 new byte[0]);
595 Thread.sleep(1001);
596
597 final Map<HRegionInfo, Result> splitParents =
598 new TreeMap<HRegionInfo, Result>(new SplitParentFirstComparator());
599 splitParents.put(parent, createResult(parent, splita, splitb));
600 splita.setOffline(true);
601 splitParents.put(splita, createResult(splita, splitaa,splitab));
602
603 final Map<HRegionInfo, Result> mergedRegions = new TreeMap<HRegionInfo, Result>();
604 CatalogJanitor janitor = spy(new CatalogJanitor(server, services));
605 doReturn(new Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>>(
606 10, mergedRegions, splitParents)).when(janitor)
607 .getMergedRegionsAndSplitParents();
608
609
610 Path splitaRef =
611 createReferences(services, htd, parent, splita, Bytes.toBytes("ccc"), false);
612
613
614 assertEquals(0, janitor.scan());
615
616
617 FileSystem fs = FileSystem.get(htu.getConfiguration());
618 assertTrue(fs.delete(splitaRef, true));
619
620
621 assertEquals(2, janitor.scan());
622
623 services.stop("test finished");
624 janitor.join();
625 }
626
627
628
629
630
631 @Test
632 public void testSplitParentFirstComparator() {
633 SplitParentFirstComparator comp = new SplitParentFirstComparator();
634 final HTableDescriptor htd = createHTableDescriptor();
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655 HRegionInfo rootRegion = new HRegionInfo(htd.getTableName(),
656 HConstants.EMPTY_START_ROW,
657 HConstants.EMPTY_END_ROW, true);
658 HRegionInfo firstRegion = new HRegionInfo(htd.getTableName(),
659 HConstants.EMPTY_START_ROW,
660 Bytes.toBytes("bbb"), true);
661 HRegionInfo lastRegion = new HRegionInfo(htd.getTableName(),
662 Bytes.toBytes("bbb"),
663 HConstants.EMPTY_END_ROW, true);
664
665 assertTrue(comp.compare(rootRegion, rootRegion) == 0);
666 assertTrue(comp.compare(firstRegion, firstRegion) == 0);
667 assertTrue(comp.compare(lastRegion, lastRegion) == 0);
668 assertTrue(comp.compare(rootRegion, firstRegion) < 0);
669 assertTrue(comp.compare(rootRegion, lastRegion) < 0);
670 assertTrue(comp.compare(firstRegion, lastRegion) < 0);
671
672
673 HRegionInfo firstRegiona = new HRegionInfo(htd.getTableName(),
674 HConstants.EMPTY_START_ROW,
675 Bytes.toBytes("aaa"), true);
676 HRegionInfo firstRegionb = new HRegionInfo(htd.getTableName(),
677 Bytes.toBytes("aaa"),
678 Bytes.toBytes("bbb"), true);
679
680 HRegionInfo lastRegiona = new HRegionInfo(htd.getTableName(),
681 Bytes.toBytes("bbb"),
682 Bytes.toBytes("ddd"), true);
683 HRegionInfo lastRegionb = new HRegionInfo(htd.getTableName(),
684 Bytes.toBytes("ddd"),
685 HConstants.EMPTY_END_ROW, true);
686
687 assertTrue(comp.compare(firstRegiona, firstRegiona) == 0);
688 assertTrue(comp.compare(firstRegionb, firstRegionb) == 0);
689 assertTrue(comp.compare(rootRegion, firstRegiona) < 0);
690 assertTrue(comp.compare(rootRegion, firstRegionb) < 0);
691 assertTrue(comp.compare(firstRegion, firstRegiona) < 0);
692 assertTrue(comp.compare(firstRegion, firstRegionb) < 0);
693 assertTrue(comp.compare(firstRegiona, firstRegionb) < 0);
694
695 assertTrue(comp.compare(lastRegiona, lastRegiona) == 0);
696 assertTrue(comp.compare(lastRegionb, lastRegionb) == 0);
697 assertTrue(comp.compare(rootRegion, lastRegiona) < 0);
698 assertTrue(comp.compare(rootRegion, lastRegionb) < 0);
699 assertTrue(comp.compare(lastRegion, lastRegiona) < 0);
700 assertTrue(comp.compare(lastRegion, lastRegionb) < 0);
701 assertTrue(comp.compare(lastRegiona, lastRegionb) < 0);
702
703 assertTrue(comp.compare(firstRegiona, lastRegiona) < 0);
704 assertTrue(comp.compare(firstRegiona, lastRegionb) < 0);
705 assertTrue(comp.compare(firstRegionb, lastRegiona) < 0);
706 assertTrue(comp.compare(firstRegionb, lastRegionb) < 0);
707
708 HRegionInfo lastRegionaa = new HRegionInfo(htd.getTableName(),
709 Bytes.toBytes("bbb"),
710 Bytes.toBytes("ccc"), false);
711 HRegionInfo lastRegionab = new HRegionInfo(htd.getTableName(),
712 Bytes.toBytes("ccc"),
713 Bytes.toBytes("ddd"), false);
714
715 assertTrue(comp.compare(lastRegiona, lastRegionaa) < 0);
716 assertTrue(comp.compare(lastRegiona, lastRegionab) < 0);
717 assertTrue(comp.compare(lastRegionaa, lastRegionab) < 0);
718
719 }
720
721 @Test
722 public void testArchiveOldRegion() throws Exception {
723 String table = "table";
724 HBaseTestingUtility htu = new HBaseTestingUtility();
725 setRootDirAndCleanIt(htu, "testCleanParent");
726 Server server = new MockServer(htu);
727 MasterServices services = new MockMasterServices(server);
728
729
730 CatalogJanitor janitor = new CatalogJanitor(server, services);
731
732
733 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
734 htd.addFamily(new HColumnDescriptor("f"));
735 HRegionInfo parent = new HRegionInfo(htd.getTableName(),
736 Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
737 HRegionInfo splita = new HRegionInfo(htd.getTableName(),
738 Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
739 HRegionInfo splitb = new HRegionInfo(htd.getTableName(),
740 Bytes.toBytes("ccc"),
741 Bytes.toBytes("eee"));
742
743
744
745 Result parentMetaRow = createResult(parent, splita, splitb);
746 FileSystem fs = FileSystem.get(htu.getConfiguration());
747 Path rootdir = services.getMasterFileSystem().getRootDir();
748
749
750
751 FSUtils.setRootDir(fs.getConf(), rootdir);
752 Path tabledir = FSUtils.getTableDir(rootdir, htd.getTableName());
753 Path storedir = HStore.getStoreHomedir(tabledir, parent, htd.getColumnFamilies()[0].getName());
754 Path storeArchive = HFileArchiveUtil.getStoreArchivePath(services.getConfiguration(), parent,
755 tabledir, htd.getColumnFamilies()[0].getName());
756 LOG.debug("Table dir:" + tabledir);
757 LOG.debug("Store dir:" + storedir);
758 LOG.debug("Store archive dir:" + storeArchive);
759
760
761 FileStatus[] mockFiles = addMockStoreFiles(2, services, storedir);
762
763 FileStatus[] storeFiles = fs.listStatus(storedir);
764 int index = 0;
765 for (FileStatus file : storeFiles) {
766 LOG.debug("Have store file:" + file.getPath());
767 assertEquals("Got unexpected store file", mockFiles[index].getPath(),
768 storeFiles[index].getPath());
769 index++;
770 }
771
772
773 assertTrue(janitor.cleanParent(parent, parentMetaRow));
774 LOG.debug("Finished cleanup of parent region");
775
776
777 FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive);
778 logFiles("archived files", storeFiles);
779 logFiles("archived files", archivedStoreFiles);
780
781 assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs);
782
783
784 FSUtils.delete(fs, rootdir, true);
785 services.stop("Test finished");
786 server.stop("Test finished");
787 janitor.join();
788 }
789
790
791
792
793
794 private void logFiles(String description, FileStatus[] storeFiles) {
795 LOG.debug("Current " + description + ": ");
796 for (FileStatus file : storeFiles) {
797 LOG.debug(file.getPath());
798 }
799 }
800
801
802
803
804
805 @Test
806 public void testDuplicateHFileResolution() throws Exception {
807 String table = "table";
808 HBaseTestingUtility htu = new HBaseTestingUtility();
809 setRootDirAndCleanIt(htu, "testCleanParent");
810 Server server = new MockServer(htu);
811 MasterServices services = new MockMasterServices(server);
812
813
814 CatalogJanitor janitor = new CatalogJanitor(server, services);
815
816
817 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
818 htd.addFamily(new HColumnDescriptor("f"));
819 HRegionInfo parent = new HRegionInfo(htd.getTableName(),
820 Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
821 HRegionInfo splita = new HRegionInfo(htd.getTableName(),
822 Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
823 HRegionInfo splitb = new HRegionInfo(htd.getTableName(),
824 Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
825
826
827 Result r = createResult(parent, splita, splitb);
828
829 FileSystem fs = FileSystem.get(htu.getConfiguration());
830
831 Path rootdir = services.getMasterFileSystem().getRootDir();
832
833
834
835 FSUtils.setRootDir(fs.getConf(), rootdir);
836 Path tabledir = FSUtils.getTableDir(rootdir, parent.getTable());
837 Path storedir = HStore.getStoreHomedir(tabledir, parent, htd.getColumnFamilies()[0].getName());
838 System.out.println("Old root:" + rootdir);
839 System.out.println("Old table:" + tabledir);
840 System.out.println("Old store:" + storedir);
841
842 Path storeArchive = HFileArchiveUtil.getStoreArchivePath(services.getConfiguration(), parent,
843 tabledir, htd.getColumnFamilies()[0].getName());
844 System.out.println("Old archive:" + storeArchive);
845
846
847 addMockStoreFiles(2, services, storedir);
848
849 FileStatus[] storeFiles = fs.listStatus(storedir);
850
851 assertTrue(janitor.cleanParent(parent, r));
852
853
854 FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive);
855 assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs);
856
857
858
859 addMockStoreFiles(2, services, storedir);
860
861
862 assertTrue(janitor.cleanParent(parent, r));
863
864
865 archivedStoreFiles = fs.listStatus(storeArchive);
866 assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs, true);
867
868
869 services.stop("Test finished");
870 server.stop("shutdown");
871 janitor.join();
872 }
873
874 private FileStatus[] addMockStoreFiles(int count, MasterServices services, Path storedir)
875 throws IOException {
876
877 FileSystem fs = services.getMasterFileSystem().getFileSystem();
878 fs.mkdirs(storedir);
879
880 for (int i = 0; i < count; i++) {
881 Path storeFile = new Path(storedir, "_store" + i);
882 FSDataOutputStream dos = fs.create(storeFile, true);
883 dos.writeBytes("Some data: " + i);
884 dos.close();
885 }
886 LOG.debug("Adding " + count + " store files to the storedir:" + storedir);
887
888 FileStatus[] storeFiles = fs.listStatus(storedir);
889 assertEquals("Didn't have expected store files", count, storeFiles.length);
890 return storeFiles;
891 }
892
893 private String setRootDirAndCleanIt(final HBaseTestingUtility htu,
894 final String subdir)
895 throws IOException {
896 Path testdir = htu.getDataTestDir(subdir);
897 FileSystem fs = FileSystem.get(htu.getConfiguration());
898 if (fs.exists(testdir)) assertTrue(fs.delete(testdir, true));
899 FSUtils.setRootDir(htu.getConfiguration(), testdir);
900 return FSUtils.getRootDir(htu.getConfiguration()).toString();
901 }
902
903
904
905
906
907
908
909
910
911
912
913 private Path createReferences(final MasterServices services,
914 final HTableDescriptor htd, final HRegionInfo parent,
915 final HRegionInfo daughter, final byte [] midkey, final boolean top)
916 throws IOException {
917 Path rootdir = services.getMasterFileSystem().getRootDir();
918 Path tabledir = FSUtils.getTableDir(rootdir, parent.getTable());
919 Path storedir = HStore.getStoreHomedir(tabledir, daughter,
920 htd.getColumnFamilies()[0].getName());
921 Reference ref =
922 top? Reference.createTopReference(midkey): Reference.createBottomReference(midkey);
923 long now = System.currentTimeMillis();
924
925 Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
926 FileSystem fs = services.getMasterFileSystem().getFileSystem();
927 ref.write(fs, p);
928 return p;
929 }
930
931 private Result createResult(final HRegionInfo parent, final HRegionInfo a,
932 final HRegionInfo b)
933 throws IOException {
934 return MetaMockingUtil.getMetaTableRowResult(parent, null, a, b);
935 }
936
937 private HTableDescriptor createHTableDescriptor() {
938 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("t"));
939 htd.addFamily(new HColumnDescriptor("f"));
940 return htd;
941 }
942
943 }
944