1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertFalse;
22 import static org.junit.Assert.assertNotSame;
23 import static org.junit.Assert.assertTrue;
24 import static org.junit.Assert.fail;
25
26 import java.io.IOException;
27 import java.util.ArrayList;
28 import java.util.HashMap;
29 import java.util.List;
30 import java.util.Map;
31 import java.util.concurrent.atomic.AtomicBoolean;
32
33 import org.apache.hadoop.hbase.CellScannable;
34 import org.apache.hadoop.hbase.CellUtil;
35 import org.apache.hadoop.hbase.DoNotRetryIOException;
36 import org.apache.hadoop.hbase.HBaseConfiguration;
37 import org.apache.hadoop.hbase.HBaseTestingUtility;
38 import org.apache.hadoop.hbase.HConstants;
39 import org.apache.hadoop.hbase.HRegionInfo;
40 import org.apache.hadoop.hbase.MediumTests;
41 import org.apache.hadoop.hbase.RegionException;
42 import org.apache.hadoop.hbase.RegionTransition;
43 import org.apache.hadoop.hbase.Server;
44 import org.apache.hadoop.hbase.ServerLoad;
45 import org.apache.hadoop.hbase.ServerName;
46 import org.apache.hadoop.hbase.TableName;
47 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
48 import org.apache.hadoop.hbase.catalog.CatalogTracker;
49 import org.apache.hadoop.hbase.catalog.MetaMockingUtil;
50 import org.apache.hadoop.hbase.client.HConnection;
51 import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
52 import org.apache.hadoop.hbase.client.Result;
53 import org.apache.hadoop.hbase.exceptions.DeserializationException;
54 import org.apache.hadoop.hbase.executor.EventType;
55 import org.apache.hadoop.hbase.executor.ExecutorService;
56 import org.apache.hadoop.hbase.executor.ExecutorType;
57 import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController;
58 import org.apache.hadoop.hbase.master.RegionState.State;
59 import org.apache.hadoop.hbase.master.TableLockManager.NullTableLockManager;
60 import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
61 import org.apache.hadoop.hbase.master.balancer.SimpleLoadBalancer;
62 import org.apache.hadoop.hbase.master.handler.EnableTableHandler;
63 import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
64 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
65 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
66 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetRequest;
67 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetResponse;
68 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
69 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanResponse;
70 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.Table;
71 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
72 import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
73 import org.apache.hadoop.hbase.util.Bytes;
74 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
75 import org.apache.hadoop.hbase.util.Threads;
76 import org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper;
77 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
78 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
79 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
80 import org.apache.zookeeper.KeeperException;
81 import org.apache.zookeeper.KeeperException.NodeExistsException;
82 import org.apache.zookeeper.Watcher;
83 import org.junit.After;
84 import org.junit.AfterClass;
85 import org.junit.Before;
86 import org.junit.BeforeClass;
87 import org.junit.Test;
88 import org.junit.experimental.categories.Category;
89 import org.mockito.Mockito;
90 import org.mockito.internal.util.reflection.Whitebox;
91 import org.mockito.invocation.InvocationOnMock;
92 import org.mockito.stubbing.Answer;
93
94 import com.google.protobuf.RpcController;
95 import com.google.protobuf.ServiceException;
96
97
98
99
100
101 @Category(MediumTests.class)
102 public class TestAssignmentManager {
103 private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
104 private static final ServerName SERVERNAME_A =
105 ServerName.valueOf("example.org", 1234, 5678);
106 private static final ServerName SERVERNAME_B =
107 ServerName.valueOf("example.org", 0, 5678);
108 private static final HRegionInfo REGIONINFO =
109 new HRegionInfo(TableName.valueOf("t"),
110 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
111 private static int assignmentCount;
112 private static boolean enabling = false;
113
114
115 private Server server;
116 private ServerManager serverManager;
117 private ZooKeeperWatcher watcher;
118 private LoadBalancer balancer;
119 private HMaster master;
120
121 @BeforeClass
122 public static void beforeClass() throws Exception {
123 HTU.startMiniZKCluster();
124 }
125
126 @AfterClass
127 public static void afterClass() throws IOException {
128 HTU.shutdownMiniZKCluster();
129 }
130
131 @Before
132 public void before() throws ZooKeeperConnectionException, IOException {
133
134
135
136
137
138
139 this.server = Mockito.mock(Server.class);
140 Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("master,1,1"));
141 Mockito.when(server.getConfiguration()).thenReturn(HTU.getConfiguration());
142 this.watcher =
143 new ZooKeeperWatcher(HTU.getConfiguration(), "mockedServer", this.server, true);
144 Mockito.when(server.getZooKeeper()).thenReturn(this.watcher);
145 Mockito.doThrow(new RuntimeException("Aborted")).
146 when(server).abort(Mockito.anyString(), (Throwable)Mockito.anyObject());
147
148
149
150 this.serverManager = Mockito.mock(ServerManager.class);
151 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
152 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(true);
153 Mockito.when(this.serverManager.getDeadServers()).thenReturn(new DeadServer());
154 final Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();
155 onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);
156 onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD);
157 Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(
158 new ArrayList<ServerName>(onlineServers.keySet()));
159 Mockito.when(this.serverManager.getOnlineServers()).thenReturn(onlineServers);
160
161 List<ServerName> avServers = new ArrayList<ServerName>();
162 avServers.addAll(onlineServers.keySet());
163 Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(avServers);
164 Mockito.when(this.serverManager.createDestinationServersList(null)).thenReturn(avServers);
165
166 Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, REGIONINFO, -1)).
167 thenReturn(true);
168 Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_B, REGIONINFO, -1)).
169 thenReturn(true);
170
171 Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_A, REGIONINFO, -1, null)).
172 thenReturn(RegionOpeningState.OPENED);
173 Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_B, REGIONINFO, -1, null)).
174 thenReturn(RegionOpeningState.OPENED);
175 this.master = Mockito.mock(HMaster.class);
176
177 Mockito.when(this.master.getServerManager()).thenReturn(serverManager);
178 }
179
180 @After
181 public void after() throws KeeperException {
182 if (this.watcher != null) {
183
184 ZKAssign.deleteAllNodes(this.watcher);
185 this.watcher.close();
186 }
187 }
188
189
190
191
192
193
194
195
196
197 @Test(timeout = 60000)
198 public void testBalanceOnMasterFailoverScenarioWithOpenedNode()
199 throws IOException, KeeperException, InterruptedException, ServiceException, DeserializationException {
200 AssignmentManagerWithExtrasForTesting am =
201 setUpMockedAssignmentManager(this.server, this.serverManager);
202 try {
203 createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
204 startFakeFailedOverMasterAssignmentManager(am, this.watcher);
205 while (!am.processRITInvoked) Thread.sleep(1);
206
207
208
209 am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_B));
210
211 Mocking.waitForRegionFailedToCloseAndSetToPendingClose(am, REGIONINFO);
212
213
214
215
216
217
218 int versionid =
219 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
220 assertNotSame(versionid, -1);
221 Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
222
223
224
225 versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
226 assertNotSame(-1, versionid);
227
228 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
229 SERVERNAME_B, EventType.M_ZK_REGION_OFFLINE,
230 EventType.RS_ZK_REGION_OPENING, versionid);
231 assertNotSame(-1, versionid);
232
233 versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
234 SERVERNAME_B, versionid);
235 assertNotSame(-1, versionid);
236 am.gate.set(false);
237
238 ZKAssign.blockUntilNoRIT(watcher);
239 } finally {
240 am.getExecutorService().shutdown();
241 am.shutdown();
242 }
243 }
244
245 @Test(timeout = 60000)
246 public void testBalanceOnMasterFailoverScenarioWithClosedNode()
247 throws IOException, KeeperException, InterruptedException, ServiceException, DeserializationException {
248 AssignmentManagerWithExtrasForTesting am =
249 setUpMockedAssignmentManager(this.server, this.serverManager);
250 try {
251 createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
252 startFakeFailedOverMasterAssignmentManager(am, this.watcher);
253 while (!am.processRITInvoked) Thread.sleep(1);
254
255
256
257 am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_B));
258
259 Mocking.waitForRegionFailedToCloseAndSetToPendingClose(am, REGIONINFO);
260
261
262
263
264
265
266 int versionid =
267 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
268 assertNotSame(versionid, -1);
269 am.gate.set(false);
270 Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
271
272
273
274 versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
275 assertNotSame(-1, versionid);
276
277 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
278 SERVERNAME_B, EventType.M_ZK_REGION_OFFLINE,
279 EventType.RS_ZK_REGION_OPENING, versionid);
280 assertNotSame(-1, versionid);
281
282 versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
283 SERVERNAME_B, versionid);
284 assertNotSame(-1, versionid);
285
286
287 ZKAssign.blockUntilNoRIT(watcher);
288 } finally {
289 am.getExecutorService().shutdown();
290 am.shutdown();
291 }
292 }
293
294 @Test(timeout = 60000)
295 public void testBalanceOnMasterFailoverScenarioWithOfflineNode()
296 throws IOException, KeeperException, InterruptedException, ServiceException, DeserializationException {
297 AssignmentManagerWithExtrasForTesting am =
298 setUpMockedAssignmentManager(this.server, this.serverManager);
299 try {
300 createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO);
301 startFakeFailedOverMasterAssignmentManager(am, this.watcher);
302 while (!am.processRITInvoked) Thread.sleep(1);
303
304
305
306 am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_B));
307
308 Mocking.waitForRegionFailedToCloseAndSetToPendingClose(am, REGIONINFO);
309
310
311
312
313
314
315 int versionid =
316 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
317 assertNotSame(versionid, -1);
318 Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
319
320 am.gate.set(false);
321
322
323 versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
324 assertNotSame(-1, versionid);
325
326 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
327 SERVERNAME_B, EventType.M_ZK_REGION_OFFLINE,
328 EventType.RS_ZK_REGION_OPENING, versionid);
329 assertNotSame(-1, versionid);
330
331 versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO,
332 SERVERNAME_B, versionid);
333 assertNotSame(-1, versionid);
334
335 ZKAssign.blockUntilNoRIT(watcher);
336 } finally {
337 am.getExecutorService().shutdown();
338 am.shutdown();
339 }
340 }
341
342 private void createRegionPlanAndBalance(
343 final AssignmentManager am, final ServerName from,
344 final ServerName to, final HRegionInfo hri) throws RegionException {
345
346
347 am.regionOnline(hri, from);
348
349
350 am.balance(new RegionPlan(hri, from, to));
351 }
352
353
354
355
356
357
358
359
360 @Test
361 public void testBalance()
362 throws IOException, KeeperException, DeserializationException, InterruptedException {
363
364
365 ExecutorService executor = startupMasterExecutor("testBalanceExecutor");
366
367
368 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
369 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
370 .getConfiguration());
371
372 AssignmentManager am = new AssignmentManager(this.server,
373 this.serverManager, ct, balancer, executor, null, master.getTableLockManager());
374 am.failoverCleanupDone.set(true);
375 try {
376
377
378 this.watcher.registerListenerFirst(am);
379
380
381 am.regionOnline(REGIONINFO, SERVERNAME_A);
382
383 RegionPlan plan = new RegionPlan(REGIONINFO, SERVERNAME_A, SERVERNAME_B);
384 am.balance(plan);
385
386 RegionStates regionStates = am.getRegionStates();
387
388 assertTrue(regionStates.isRegionInTransition(REGIONINFO)
389 && regionStates.isRegionInState(REGIONINFO, State.FAILED_CLOSE));
390
391 regionStates.updateRegionState(REGIONINFO, State.PENDING_CLOSE);
392
393
394
395
396
397
398 int versionid =
399 ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1);
400 assertNotSame(versionid, -1);
401
402
403
404
405
406 Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
407
408
409 versionid = ZKAssign.getVersion(this.watcher, REGIONINFO);
410 assertNotSame(-1, versionid);
411
412 versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO,
413 SERVERNAME_B, EventType.M_ZK_REGION_OFFLINE,
414 EventType.RS_ZK_REGION_OPENING, versionid);
415 assertNotSame(-1, versionid);
416
417 versionid =
418 ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO, SERVERNAME_B, versionid);
419 assertNotSame(-1, versionid);
420
421 while(regionStates.isRegionInTransition(REGIONINFO)) Threads.sleep(1);
422 } finally {
423 executor.shutdown();
424 am.shutdown();
425
426 ZKAssign.deleteAllNodes(this.watcher);
427 }
428 }
429
430
431
432
433
434
435 @Test
436 public void testShutdownHandler()
437 throws KeeperException, IOException, ServiceException {
438
439
440 ExecutorService executor = startupMasterExecutor("testShutdownHandler");
441
442
443 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
444
445 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
446 this.server, this.serverManager);
447 try {
448 processServerShutdownHandler(ct, am, false);
449 } finally {
450 executor.shutdown();
451 am.shutdown();
452
453 ZKAssign.deleteAllNodes(this.watcher);
454 }
455 }
456
457
458
459
460
461
462
463
464
465
466 @Test
467 public void testSSHWhenDisableTableInProgress() throws KeeperException, IOException,
468 ServiceException {
469 testCaseWithPartiallyDisabledState(Table.State.DISABLING);
470 testCaseWithPartiallyDisabledState(Table.State.DISABLED);
471 }
472
473
474
475
476
477
478
479
480
481 @Test
482 public void testSSHWhenSplitRegionInProgress() throws KeeperException, IOException, Exception {
483
484 testCaseWithSplitRegionPartial(true);
485
486 testCaseWithSplitRegionPartial(false);
487 }
488
489 private void testCaseWithSplitRegionPartial(boolean regionSplitDone) throws KeeperException,
490 IOException, NodeExistsException, InterruptedException, ServiceException {
491
492
493 ExecutorService executor = startupMasterExecutor("testSSHWhenSplitRegionInProgress");
494
495 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
496 ZKAssign.deleteAllNodes(this.watcher);
497
498
499 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
500 this.server, this.serverManager);
501
502 am.regionOnline(REGIONINFO, SERVERNAME_A);
503
504 am.getRegionStates().updateRegionState(
505 REGIONINFO, State.SPLITTING, SERVERNAME_A);
506 am.getZKTable().setEnabledTable(REGIONINFO.getTable());
507 RegionTransition data = RegionTransition.createRegionTransition(EventType.RS_ZK_REGION_SPLITTING,
508 REGIONINFO.getRegionName(), SERVERNAME_A);
509 String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
510
511 ZKUtil.createAndWatch(this.watcher, node, data.toByteArray());
512
513 try {
514 processServerShutdownHandler(ct, am, regionSplitDone);
515
516
517
518 if (regionSplitDone) {
519 assertFalse("Region state of region in SPLITTING should be removed from rit.",
520 am.getRegionStates().isRegionsInTransition());
521 } else {
522 while (!am.assignInvoked) {
523 Thread.sleep(1);
524 }
525 assertTrue("Assign should be invoked.", am.assignInvoked);
526 }
527 } finally {
528 REGIONINFO.setOffline(false);
529 REGIONINFO.setSplit(false);
530 executor.shutdown();
531 am.shutdown();
532
533 ZKAssign.deleteAllNodes(this.watcher);
534 }
535 }
536
537 private void testCaseWithPartiallyDisabledState(Table.State state) throws KeeperException,
538 IOException, NodeExistsException, ServiceException {
539
540
541 ExecutorService executor = startupMasterExecutor("testSSHWhenDisableTableInProgress");
542
543 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
544 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
545 ZKAssign.deleteAllNodes(this.watcher);
546
547
548 AssignmentManager am = new AssignmentManager(this.server,
549 this.serverManager, ct, balancer, executor, null, master.getTableLockManager());
550
551 am.regionOnline(REGIONINFO, SERVERNAME_A);
552
553 am.getRegionStates().updateRegionState(REGIONINFO, State.PENDING_CLOSE);
554 if (state == Table.State.DISABLING) {
555 am.getZKTable().setDisablingTable(REGIONINFO.getTable());
556 } else {
557 am.getZKTable().setDisabledTable(REGIONINFO.getTable());
558 }
559 RegionTransition data = RegionTransition.createRegionTransition(EventType.M_ZK_REGION_CLOSING,
560 REGIONINFO.getRegionName(), SERVERNAME_A);
561
562
563
564 String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName());
565
566 ZKUtil.createAndWatch(this.watcher, node, data.toByteArray());
567
568 try {
569 processServerShutdownHandler(ct, am, false);
570
571
572 assertTrue("The znode should be deleted.", ZKUtil.checkExists(this.watcher, node) == -1);
573
574
575
576 if (state == Table.State.DISABLED) {
577 assertFalse("Region state of region in pending close should be removed from rit.",
578 am.getRegionStates().isRegionsInTransition());
579 }
580 } finally {
581 am.setEnabledTable(REGIONINFO.getTable());
582 executor.shutdown();
583 am.shutdown();
584
585 ZKAssign.deleteAllNodes(this.watcher);
586 }
587 }
588
589 private void processServerShutdownHandler(CatalogTracker ct, AssignmentManager am, boolean splitRegion)
590 throws IOException, ServiceException {
591
592
593 this.watcher.registerListenerFirst(am);
594
595
596
597 ClientProtos.ClientService.BlockingInterface implementation =
598 Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
599
600
601 Result r;
602 if (splitRegion) {
603 r = MetaMockingUtil.getMetaTableRowResultAsSplitRegion(REGIONINFO, SERVERNAME_A);
604 } else {
605 r = MetaMockingUtil.getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
606 }
607
608 final ScanResponse.Builder builder = ScanResponse.newBuilder();
609 builder.setMoreResults(true);
610 builder.addCellsPerResult(r.size());
611 final List<CellScannable> cellScannables = new ArrayList<CellScannable>(1);
612 cellScannables.add(r);
613 Mockito.when(implementation.scan(
614 (RpcController)Mockito.any(), (ScanRequest)Mockito.any())).
615 thenAnswer(new Answer<ScanResponse>() {
616 @Override
617 public ScanResponse answer(InvocationOnMock invocation) throws Throwable {
618 PayloadCarryingRpcController controller = (PayloadCarryingRpcController) invocation
619 .getArguments()[0];
620 if (controller != null) {
621 controller.setCellScanner(CellUtil.createCellScanner(cellScannables));
622 }
623 return builder.build();
624 }
625 });
626
627
628 HConnection connection =
629 HConnectionTestingUtility.getMockedConnectionAndDecorate(HTU.getConfiguration(),
630 null, implementation, SERVERNAME_B, REGIONINFO);
631
632
633
634 Mockito.when(ct.getConnection()).thenReturn(connection);
635 Mockito.when(this.server.getCatalogTracker()).thenReturn(ct);
636
637
638
639 DeadServer deadServers = new DeadServer();
640 deadServers.add(SERVERNAME_A);
641
642 MasterFileSystem fs = Mockito.mock(MasterFileSystem.class);
643 Mockito.doNothing().when(fs).setLogRecoveryMode();
644 Mockito.when(fs.getLogRecoveryMode()).thenReturn(RecoveryMode.LOG_REPLAY);
645 MasterServices services = Mockito.mock(MasterServices.class);
646 Mockito.when(services.getAssignmentManager()).thenReturn(am);
647 Mockito.when(services.getServerManager()).thenReturn(this.serverManager);
648 Mockito.when(services.getZooKeeper()).thenReturn(this.watcher);
649 Mockito.when(services.getMasterFileSystem()).thenReturn(fs);
650 ServerShutdownHandler handler = new ServerShutdownHandler(this.server,
651 services, deadServers, SERVERNAME_A, false);
652 am.failoverCleanupDone.set(true);
653 handler.process();
654
655 }
656
657
658
659
660
661
662
663 private ExecutorService startupMasterExecutor(final String name) {
664
665 ExecutorService executor = new ExecutorService(name);
666 executor.startExecutorService(ExecutorType.MASTER_OPEN_REGION, 3);
667 executor.startExecutorService(ExecutorType.MASTER_CLOSE_REGION, 3);
668 executor.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS, 3);
669 executor.startExecutorService(ExecutorType.MASTER_META_SERVER_OPERATIONS, 3);
670 return executor;
671 }
672
673 @Test
674 public void testUnassignWithSplitAtSameTime() throws KeeperException, IOException {
675
676 final HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO;
677
678
679
680 Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, hri, -1)).thenReturn(true);
681
682 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
683 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server
684 .getConfiguration());
685
686 AssignmentManager am = new AssignmentManager(this.server,
687 this.serverManager, ct, balancer, null, null, master.getTableLockManager());
688 try {
689
690 unassign(am, SERVERNAME_A, hri);
691
692 ZKAssign.deleteClosingNode(this.watcher, hri, SERVERNAME_A);
693
694
695
696 int version = createNodeSplitting(this.watcher, hri, SERVERNAME_A);
697
698
699
700
701 unassign(am, SERVERNAME_A, hri);
702
703 ZKAssign.transitionNode(this.watcher, hri, SERVERNAME_A,
704 EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version);
705 assertFalse(am.getRegionStates().isRegionInTransition(hri));
706 } finally {
707 am.shutdown();
708 }
709 }
710
711
712
713
714
715
716
717 @Test(timeout = 60000)
718 public void testProcessDeadServersAndRegionsInTransitionShouldNotFailWithNPE()
719 throws IOException, KeeperException, InterruptedException, ServiceException {
720 final RecoverableZooKeeper recoverableZk = Mockito
721 .mock(RecoverableZooKeeper.class);
722 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
723 this.server, this.serverManager);
724 Watcher zkw = new ZooKeeperWatcher(HBaseConfiguration.create(), "unittest",
725 null) {
726 @Override
727 public RecoverableZooKeeper getRecoverableZooKeeper() {
728 return recoverableZk;
729 }
730 };
731 ((ZooKeeperWatcher) zkw).registerListener(am);
732 Mockito.doThrow(new InterruptedException()).when(recoverableZk)
733 .getChildren("/hbase/region-in-transition", null);
734 am.setWatcher((ZooKeeperWatcher) zkw);
735 try {
736 am.processDeadServersAndRegionsInTransition(null);
737 fail("Expected to abort");
738 } catch (NullPointerException e) {
739 fail("Should not throw NPE");
740 } catch (RuntimeException e) {
741 assertEquals("Aborted", e.getLocalizedMessage());
742 }
743 }
744
745
746
747
748 @Test(timeout = 60000)
749 public void testRegionPlanIsUpdatedWhenRegionFailsToOpen() throws IOException, KeeperException,
750 ServiceException, InterruptedException {
751 this.server.getConfiguration().setClass(
752 HConstants.HBASE_MASTER_LOADBALANCER_CLASS, MockedLoadBalancer.class,
753 LoadBalancer.class);
754 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
755 this.server, this.serverManager);
756 try {
757
758
759
760 AtomicBoolean gate = new AtomicBoolean(false);
761 if (balancer instanceof MockedLoadBalancer) {
762 ((MockedLoadBalancer) balancer).setGateVariable(gate);
763 }
764 ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_A);
765 int v = ZKAssign.getVersion(this.watcher, REGIONINFO);
766 ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_A,
767 EventType.M_ZK_REGION_OFFLINE, EventType.RS_ZK_REGION_FAILED_OPEN, v);
768 String path = ZKAssign.getNodeName(this.watcher, REGIONINFO
769 .getEncodedName());
770 am.getRegionStates().updateRegionState(
771 REGIONINFO, State.OPENING, SERVERNAME_A);
772
773
774 am.regionPlans.put(REGIONINFO.getEncodedName(), new RegionPlan(
775 REGIONINFO, null, SERVERNAME_A));
776 RegionPlan regionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
777 List<ServerName> serverList = new ArrayList<ServerName>(2);
778 serverList.add(SERVERNAME_B);
779 Mockito.when(
780 this.serverManager.createDestinationServersList(SERVERNAME_A))
781 .thenReturn(serverList);
782 am.nodeDataChanged(path);
783
784
785 while (!gate.get()) {
786 Thread.sleep(10);
787 }
788
789
790
791 RegionPlan newRegionPlan = am.regionPlans
792 .get(REGIONINFO.getEncodedName());
793 while (newRegionPlan == null) {
794 Thread.sleep(10);
795 newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName());
796 }
797
798
799
800 assertNotSame("Same region plan should not come", regionPlan,
801 newRegionPlan);
802 assertTrue("Destination servers should be different.", !(regionPlan
803 .getDestination().equals(newRegionPlan.getDestination())));
804
805 Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName());
806 } finally {
807 this.server.getConfiguration().setClass(
808 HConstants.HBASE_MASTER_LOADBALANCER_CLASS, SimpleLoadBalancer.class,
809 LoadBalancer.class);
810 am.getExecutorService().shutdown();
811 am.shutdown();
812 }
813 }
814
815
816
817
818
819 public static class MockedLoadBalancer extends SimpleLoadBalancer {
820 private AtomicBoolean gate;
821
822 public void setGateVariable(AtomicBoolean gate) {
823 this.gate = gate;
824 }
825
826 @Override
827 public ServerName randomAssignment(HRegionInfo regionInfo, List<ServerName> servers) {
828 ServerName randomServerName = super.randomAssignment(regionInfo, servers);
829 this.gate.set(true);
830 return randomServerName;
831 }
832
833 @Override
834 public Map<ServerName, List<HRegionInfo>> retainAssignment(
835 Map<HRegionInfo, ServerName> regions, List<ServerName> servers) {
836 this.gate.set(true);
837 return super.retainAssignment(regions, servers);
838 }
839 }
840
841
842
843
844
845
846 @Test(timeout = 60000)
847 public void testRegionInOpeningStateOnDeadRSWhileMasterFailover() throws IOException,
848 KeeperException, ServiceException, InterruptedException {
849 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(
850 this.server, this.serverManager);
851 ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_A);
852 int version = ZKAssign.getVersion(this.watcher, REGIONINFO);
853 ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE,
854 EventType.RS_ZK_REGION_OPENING, version);
855 RegionTransition rt = RegionTransition.createRegionTransition(EventType.RS_ZK_REGION_OPENING,
856 REGIONINFO.getRegionName(), SERVERNAME_A, HConstants.EMPTY_BYTE_ARRAY);
857 version = ZKAssign.getVersion(this.watcher, REGIONINFO);
858 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(false);
859 am.getRegionStates().logSplit(SERVERNAME_A);
860 am.getRegionStates().createRegionState(REGIONINFO);
861 am.gate.set(false);
862 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
863 assertFalse(am.processRegionsInTransition(rt, REGIONINFO, version));
864 am.getZKTable().setEnabledTable(REGIONINFO.getTable());
865 processServerShutdownHandler(ct, am, false);
866
867 while (!am.gate.get()) {
868 Thread.sleep(10);
869 }
870 assertTrue("The region should be assigned immediately.", null != am.regionPlans.get(REGIONINFO
871 .getEncodedName()));
872 }
873
874
875
876
877
878
879
880
881
882 @Test(timeout = 60000)
883 public void testDisablingTableRegionsAssignmentDuringCleanClusterStartup()
884 throws KeeperException, IOException, Exception {
885 this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
886 MockedLoadBalancer.class, LoadBalancer.class);
887 Mockito.when(this.serverManager.getOnlineServers()).thenReturn(
888 new HashMap<ServerName, ServerLoad>(0));
889 List<ServerName> destServers = new ArrayList<ServerName>(1);
890 destServers.add(SERVERNAME_A);
891 Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(destServers);
892
893 HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
894 Server server = new HMaster(HTU.getConfiguration());
895 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
896 this.serverManager);
897 AtomicBoolean gate = new AtomicBoolean(false);
898 if (balancer instanceof MockedLoadBalancer) {
899 ((MockedLoadBalancer) balancer).setGateVariable(gate);
900 }
901 try{
902
903 am.getZKTable().setDisablingTable(REGIONINFO.getTable());
904 am.joinCluster();
905
906 assertFalse(
907 "Assign should not be invoked for disabling table regions during clean cluster startup.",
908 gate.get());
909
910 assertTrue("Table should be disabled.",
911 am.getZKTable().isDisabledTable(REGIONINFO.getTable()));
912 } finally {
913 this.server.getConfiguration().setClass(
914 HConstants.HBASE_MASTER_LOADBALANCER_CLASS, SimpleLoadBalancer.class,
915 LoadBalancer.class);
916 am.getZKTable().setEnabledTable(REGIONINFO.getTable());
917 am.shutdown();
918 }
919 }
920
921
922
923
924
925
926
927
928 @Test
929 public void testMasterRestartWhenTableInEnabling() throws KeeperException, IOException, Exception {
930 enabling = true;
931 List<ServerName> destServers = new ArrayList<ServerName>(1);
932 destServers.add(SERVERNAME_A);
933 Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(destServers);
934 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
935 HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
936 Server server = new HMaster(HTU.getConfiguration());
937 Whitebox.setInternalState(server, "serverManager", this.serverManager);
938 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
939 this.serverManager);
940 try {
941
942 am.getZKTable().setEnablingTable(REGIONINFO.getTable());
943 new EnableTableHandler(server, REGIONINFO.getTable(),
944 am.getCatalogTracker(), am, new NullTableLockManager(), true).prepare()
945 .process();
946 assertEquals("Number of assignments should be 1.", 1, assignmentCount);
947 assertTrue("Table should be enabled.",
948 am.getZKTable().isEnabledTable(REGIONINFO.getTable()));
949 } finally {
950 enabling = false;
951 assignmentCount = 0;
952 am.getZKTable().setEnabledTable(REGIONINFO.getTable());
953 am.shutdown();
954 ZKAssign.deleteAllNodes(this.watcher);
955 }
956 }
957
958
959
960
961
962
963
964
965 @Test
966 public void testMasterRestartShouldRemoveStaleZnodesOfUnknownTableAsForMeta()
967 throws KeeperException, IOException, Exception {
968 List<ServerName> destServers = new ArrayList<ServerName>(1);
969 destServers.add(SERVERNAME_A);
970 Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(destServers);
971 Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
972 HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
973 Server server = new HMaster(HTU.getConfiguration());
974 Whitebox.setInternalState(server, "serverManager", this.serverManager);
975 AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
976 this.serverManager);
977 try {
978 TableName tableName = TableName.valueOf("dummyTable");
979
980 am.getZKTable().setEnablingTable(tableName);
981 am.joinCluster();
982 assertFalse("Table should not be present in zookeeper.",
983 am.getZKTable().isTablePresent(tableName));
984 } finally {
985 }
986 }
987
988
989
990
991
992 @Test
993 public void testSSHTimesOutOpeningRegionTransition()
994 throws KeeperException, IOException, ServiceException {
995
996 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
997
998 AssignmentManagerWithExtrasForTesting am =
999 setUpMockedAssignmentManager(this.server, this.serverManager);
1000
1001 RegionState state = new RegionState(REGIONINFO,
1002 State.OPENING, System.currentTimeMillis(), SERVERNAME_A);
1003 am.getRegionStates().regionOnline(REGIONINFO, SERVERNAME_B);
1004 am.getRegionStates().regionsInTransition.put(REGIONINFO.getEncodedName(), state);
1005
1006 am.regionPlans.put(REGIONINFO.getEncodedName(),
1007 new RegionPlan(REGIONINFO, SERVERNAME_B, SERVERNAME_A));
1008 am.getZKTable().setEnabledTable(REGIONINFO.getTable());
1009
1010 try {
1011 am.assignInvoked = false;
1012 processServerShutdownHandler(ct, am, false);
1013 assertTrue(am.assignInvoked);
1014 } finally {
1015 am.getRegionStates().regionsInTransition.remove(REGIONINFO.getEncodedName());
1016 am.regionPlans.remove(REGIONINFO.getEncodedName());
1017 }
1018 }
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029 @Test
1030 public void testClosingFailureDuringRecovery() throws Exception {
1031
1032 AssignmentManagerWithExtrasForTesting am =
1033 setUpMockedAssignmentManager(this.server, this.serverManager);
1034 ZKAssign.createNodeClosing(this.watcher, REGIONINFO, SERVERNAME_A);
1035 am.getRegionStates().createRegionState(REGIONINFO);
1036
1037 assertFalse( am.getRegionStates().isRegionsInTransition() );
1038
1039 am.processRegionInTransition(REGIONINFO.getEncodedName(), REGIONINFO);
1040
1041 assertTrue( am.getRegionStates().isRegionsInTransition() );
1042 }
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060 private static int createNodeSplitting(final ZooKeeperWatcher zkw,
1061 final HRegionInfo region, final ServerName serverName)
1062 throws KeeperException, IOException {
1063 RegionTransition rt =
1064 RegionTransition.createRegionTransition(EventType.RS_ZK_REGION_SPLITTING,
1065 region.getRegionName(), serverName);
1066
1067 String node = ZKAssign.getNodeName(zkw, region.getEncodedName());
1068 if (!ZKUtil.createEphemeralNodeAndWatch(zkw, node, rt.toByteArray())) {
1069 throw new IOException("Failed create of ephemeral " + node);
1070 }
1071
1072
1073 return transitionNodeSplitting(zkw, region, serverName, -1);
1074 }
1075
1076
1077
1078 private static int transitionNodeSplitting(final ZooKeeperWatcher zkw,
1079 final HRegionInfo parent,
1080 final ServerName serverName, final int version)
1081 throws KeeperException, IOException {
1082 return ZKAssign.transitionNode(zkw, parent, serverName,
1083 EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version);
1084 }
1085
1086 private void unassign(final AssignmentManager am, final ServerName sn,
1087 final HRegionInfo hri) throws RegionException {
1088
1089 am.regionOnline(hri, sn);
1090
1091 am.unassign(hri);
1092 }
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103 private AssignmentManagerWithExtrasForTesting setUpMockedAssignmentManager(final Server server,
1104 final ServerManager manager) throws IOException, KeeperException, ServiceException {
1105
1106 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
1107
1108
1109
1110
1111
1112 ClientProtos.ClientService.BlockingInterface ri =
1113 Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
1114
1115 Result r = MetaMockingUtil.getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
1116 final ScanResponse.Builder builder = ScanResponse.newBuilder();
1117 builder.setMoreResults(true);
1118 builder.addCellsPerResult(r.size());
1119 final List<CellScannable> rows = new ArrayList<CellScannable>(1);
1120 rows.add(r);
1121 Answer<ScanResponse> ans = new Answer<ClientProtos.ScanResponse>() {
1122 @Override
1123 public ScanResponse answer(InvocationOnMock invocation) throws Throwable {
1124 PayloadCarryingRpcController controller = (PayloadCarryingRpcController) invocation
1125 .getArguments()[0];
1126 if (controller != null) {
1127 controller.setCellScanner(CellUtil.createCellScanner(rows));
1128 }
1129 return builder.build();
1130 }
1131 };
1132 if (enabling) {
1133 Mockito.when(ri.scan((RpcController) Mockito.any(), (ScanRequest) Mockito.any()))
1134 .thenAnswer(ans).thenAnswer(ans).thenAnswer(ans).thenAnswer(ans).thenAnswer(ans)
1135 .thenReturn(ScanResponse.newBuilder().setMoreResults(false).build());
1136 } else {
1137 Mockito.when(ri.scan((RpcController) Mockito.any(), (ScanRequest) Mockito.any())).thenAnswer(
1138 ans);
1139 }
1140
1141 GetResponse.Builder getBuilder = GetResponse.newBuilder();
1142 getBuilder.setResult(ProtobufUtil.toResult(r));
1143 Mockito.when(ri.get((RpcController)Mockito.any(), (GetRequest) Mockito.any())).
1144 thenReturn(getBuilder.build());
1145
1146 HConnection connection = HConnectionTestingUtility.
1147 getMockedConnectionAndDecorate(HTU.getConfiguration(), null,
1148 ri, SERVERNAME_B, REGIONINFO);
1149
1150 Mockito.when(ct.getConnection()).thenReturn(connection);
1151
1152 ExecutorService executor = startupMasterExecutor("mockedAMExecutor");
1153 this.balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
1154 AssignmentManagerWithExtrasForTesting am = new AssignmentManagerWithExtrasForTesting(
1155 server, manager, ct, this.balancer, executor, new NullTableLockManager());
1156 return am;
1157 }
1158
1159
1160
1161
1162 class AssignmentManagerWithExtrasForTesting extends AssignmentManager {
1163
1164 private final ExecutorService es;
1165
1166 private final CatalogTracker ct;
1167 boolean processRITInvoked = false;
1168 boolean assignInvoked = false;
1169 AtomicBoolean gate = new AtomicBoolean(true);
1170
1171 public AssignmentManagerWithExtrasForTesting(
1172 final Server master, final ServerManager serverManager,
1173 final CatalogTracker catalogTracker, final LoadBalancer balancer,
1174 final ExecutorService service, final TableLockManager tableLockManager)
1175 throws KeeperException, IOException {
1176 super(master, serverManager, catalogTracker, balancer, service, null, tableLockManager);
1177 this.es = service;
1178 this.ct = catalogTracker;
1179 }
1180
1181 @Override
1182 boolean processRegionInTransition(String encodedRegionName,
1183 HRegionInfo regionInfo) throws KeeperException, IOException {
1184 this.processRITInvoked = true;
1185 return super.processRegionInTransition(encodedRegionName, regionInfo);
1186 }
1187
1188 @Override
1189 public void assign(HRegionInfo region, boolean setOfflineInZK, boolean forceNewPlan) {
1190 if (enabling) {
1191 assignmentCount++;
1192 this.regionOnline(region, SERVERNAME_A);
1193 } else {
1194 super.assign(region, setOfflineInZK, forceNewPlan);
1195 this.gate.set(true);
1196 }
1197 }
1198
1199 @Override
1200 boolean assign(ServerName destination, List<HRegionInfo> regions) {
1201 if (enabling) {
1202 for (HRegionInfo region : regions) {
1203 assignmentCount++;
1204 this.regionOnline(region, SERVERNAME_A);
1205 }
1206 return true;
1207 }
1208 return super.assign(destination, regions);
1209 }
1210
1211 @Override
1212 public void assign(List<HRegionInfo> regions)
1213 throws IOException, InterruptedException {
1214 assignInvoked = (regions != null && regions.size() > 0);
1215 super.assign(regions);
1216 this.gate.set(true);
1217 }
1218
1219
1220 void setWatcher(ZooKeeperWatcher watcher) {
1221 this.watcher = watcher;
1222 }
1223
1224
1225
1226
1227 ExecutorService getExecutorService() {
1228 return this.es;
1229 }
1230
1231
1232
1233
1234 CatalogTracker getCatalogTracker() {
1235 return this.ct;
1236 }
1237 }
1238
1239
1240
1241
1242
1243
1244
1245 private void startFakeFailedOverMasterAssignmentManager(final AssignmentManager am,
1246 final ZooKeeperWatcher watcher) {
1247
1248
1249 watcher.registerListenerFirst(am);
1250 Thread t = new Thread("RunAmJoinCluster") {
1251 @Override
1252 public void run() {
1253
1254
1255
1256
1257
1258 am.getRegionStates().regionsInTransition.clear();
1259 am.regionPlans.clear();
1260 try {
1261 am.joinCluster();
1262 } catch (IOException e) {
1263 throw new RuntimeException(e);
1264 } catch (KeeperException e) {
1265 throw new RuntimeException(e);
1266 } catch (InterruptedException e) {
1267 throw new RuntimeException(e);
1268 }
1269 }
1270 };
1271 t.start();
1272 while (!t.isAlive()) Threads.sleep(1);
1273 }
1274
1275 @Test
1276 public void testForceAssignMergingRegion() throws Exception {
1277
1278 final HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO;
1279
1280 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
1281 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(
1282 server.getConfiguration());
1283
1284 AssignmentManager am = new AssignmentManager(this.server,
1285 this.serverManager, ct, balancer, null, null, master.getTableLockManager());
1286 RegionStates regionStates = am.getRegionStates();
1287 try {
1288
1289 regionStates.updateRegionState(hri, RegionState.State.MERGING);
1290
1291 am.assign(hri, true, true);
1292 assertEquals("The region should be still in merging state",
1293 RegionState.State.MERGING, regionStates.getRegionState(hri).getState());
1294 } finally {
1295 am.shutdown();
1296 }
1297 }
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309 @Test
1310 public void testAssignmentEventIgnoredIfNotExpected() throws KeeperException, IOException {
1311
1312 final HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO;
1313
1314 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
1315 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(
1316 server.getConfiguration());
1317 final AtomicBoolean zkEventProcessed = new AtomicBoolean(false);
1318
1319 AssignmentManager am = new AssignmentManager(this.server,
1320 this.serverManager, ct, balancer, null, null, master.getTableLockManager()) {
1321
1322 @Override
1323 void handleRegion(final RegionTransition rt, int expectedVersion) {
1324 super.handleRegion(rt, expectedVersion);
1325 if (rt != null && Bytes.equals(hri.getRegionName(),
1326 rt.getRegionName()) && rt.getEventType() == EventType.RS_ZK_REGION_OPENING) {
1327 zkEventProcessed.set(true);
1328 }
1329 }
1330 };
1331 try {
1332
1333 am.getRegionStates().regionOffline(hri);
1334 zkEventProcessed.set(false);
1335 this.watcher.registerListenerFirst(am);
1336 assertFalse("The region should not be in transition",
1337 am.getRegionStates().isRegionInTransition(hri));
1338 ZKAssign.createNodeOffline(this.watcher, hri, SERVERNAME_A);
1339
1340 ZKAssign.transitionNodeOpening(this.watcher, hri, SERVERNAME_A);
1341 long startTime = EnvironmentEdgeManager.currentTimeMillis();
1342 while (!zkEventProcessed.get()) {
1343 assertTrue("Timed out in waiting for ZK event to be processed",
1344 EnvironmentEdgeManager.currentTimeMillis() - startTime < 30000);
1345 Threads.sleepWithoutInterrupt(100);
1346 }
1347 assertFalse(am.getRegionStates().isRegionInTransition(hri));
1348 } finally {
1349 am.shutdown();
1350 }
1351 }
1352
1353
1354
1355
1356
1357
1358 @Test
1359 public void testBalanceRegionOfDeletedTable() throws Exception {
1360 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
1361 AssignmentManager am = new AssignmentManager(this.server, this.serverManager,
1362 ct, balancer, null, null, master.getTableLockManager());
1363 RegionStates regionStates = am.getRegionStates();
1364 HRegionInfo hri = REGIONINFO;
1365 regionStates.createRegionState(hri);
1366 assertFalse(regionStates.isRegionInTransition(hri));
1367 RegionPlan plan = new RegionPlan(hri, SERVERNAME_A, SERVERNAME_B);
1368
1369 regionStates.tableDeleted(hri.getTable());
1370 am.balance(plan);
1371 assertFalse("The region should not in transition",
1372 regionStates.isRegionInTransition(hri));
1373 }
1374
1375
1376
1377
1378
1379 @SuppressWarnings("unchecked")
1380 @Test
1381 public void testOpenCloseRegionRPCIntendedForPreviousServer() throws Exception {
1382 Mockito.when(this.serverManager.sendRegionOpen(Mockito.eq(SERVERNAME_B), Mockito.eq(REGIONINFO),
1383 Mockito.anyInt(), (List<ServerName>)Mockito.any()))
1384 .thenThrow(new DoNotRetryIOException());
1385 this.server.getConfiguration().setInt("hbase.assignment.maximum.attempts", 100);
1386
1387 HRegionInfo hri = REGIONINFO;
1388 CatalogTracker ct = Mockito.mock(CatalogTracker.class);
1389 LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(
1390 server.getConfiguration());
1391
1392 AssignmentManager am = new AssignmentManager(this.server,
1393 this.serverManager, ct, balancer, null, null, master.getTableLockManager());
1394 RegionStates regionStates = am.getRegionStates();
1395 try {
1396 am.regionPlans.put(REGIONINFO.getEncodedName(),
1397 new RegionPlan(REGIONINFO, null, SERVERNAME_B));
1398
1399
1400 am.assign(hri, true, false);
1401 } finally {
1402 assertEquals(SERVERNAME_A, regionStates.getRegionState(REGIONINFO).getServerName());
1403 }
1404 }
1405 }