1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master.procedure;
20
21 import static org.junit.Assert.assertEquals;
22
23 import java.io.IOException;
24 import java.util.Arrays;
25 import java.util.Collection;
26
27 import org.apache.hadoop.hbase.HBaseTestingUtility;
28 import org.apache.hadoop.hbase.MiniHBaseCluster;
29 import org.apache.hadoop.hbase.TableName;
30 import org.apache.hadoop.hbase.client.ResultScanner;
31 import org.apache.hadoop.hbase.client.Scan;
32 import org.apache.hadoop.hbase.client.Table;
33 import org.apache.hadoop.hbase.master.AssignmentManager;
34 import org.apache.hadoop.hbase.master.HMaster;
35 import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
36 import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
37 import org.apache.hadoop.hbase.regionserver.HRegionServer;
38 import org.apache.hadoop.hbase.testclassification.LargeTests;
39 import org.apache.hadoop.hbase.util.Threads;
40 import org.junit.After;
41 import org.junit.Before;
42 import org.junit.Test;
43 import org.junit.experimental.categories.Category;
44 import org.junit.runner.RunWith;
45 import org.junit.runners.Parameterized;
46 import org.junit.runners.Parameterized.Parameters;
47
48
49
50
51 @Category(LargeTests.class)
52 @RunWith(Parameterized.class)
53 public class TestServerCrashProcedure {
54
55
56 @Parameters(name = "{index}: setting={0}")
57 public static Collection<Object []> data() {
58 return Arrays.asList(new Object[] [] {{Boolean.FALSE, -1}});
59 }
60
61 private final HBaseTestingUtility util = new HBaseTestingUtility();
62
63 @Before
64 public void setup() throws Exception {
65 this.util.startMiniCluster(3);
66 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(
67 this.util.getHBaseCluster().getMaster().getMasterProcedureExecutor(), false);
68 }
69
70 @After
71 public void tearDown() throws Exception {
72 MiniHBaseCluster cluster = this.util.getHBaseCluster();
73 HMaster master = cluster == null? null: cluster.getMaster();
74 if (master != null && master.getMasterProcedureExecutor() != null) {
75 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(master.getMasterProcedureExecutor(),
76 false);
77 }
78 this.util.shutdownMiniCluster();
79 }
80
81 public TestServerCrashProcedure(final Boolean b, final int ignore) {
82 this.util.getConfiguration().setBoolean("hbase.master.distributed.log.replay", b);
83 this.util.getConfiguration().setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
84 }
85
86
87
88
89
90
91 @Test(timeout = 300000)
92 public void testRecoveryAndDoubleExecutionOnline() throws Exception {
93 final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecutionOnline");
94 this.util.createTable(tableName, HBaseTestingUtility.COLUMNS,
95 HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
96 try (Table t = this.util.getConnection().getTable(tableName)) {
97
98 this.util.loadTable(t, HBaseTestingUtility.COLUMNS[0]);
99 int count = countRows(t);
100
101
102 HMaster master = this.util.getHBaseCluster().getMaster();
103 final ProcedureExecutor<MasterProcedureEnv> procExec = master.getMasterProcedureExecutor();
104 master.setServerCrashProcessingEnabled(false);
105
106 HRegionServer hrs = this.util.getHBaseCluster().getRegionServer(0);
107 boolean carryingMeta = (master.getAssignmentManager().isCarryingMeta(hrs.getServerName()) ==
108 AssignmentManager.ServerHostRegion.HOSTING_REGION);
109 this.util.getHBaseCluster().killRegionServer(hrs.getServerName());
110 hrs.join();
111
112
113
114 while (!master.getServerManager().isServerDead(hrs.getServerName())) Threads.sleep(10);
115
116 master.setServerCrashProcessingEnabled(true);
117
118 master.getServerManager().moveFromOnelineToDeadServers(hrs.getServerName());
119
120 ProcedureTestingUtility.waitNoProcedureRunning(procExec);
121 ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
122 long procId =
123 procExec.submitProcedure(new ServerCrashProcedure(hrs.getServerName(), true, carryingMeta));
124
125 MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
126
127 assertEquals(count, countRows(t));
128 }
129 }
130
131 int countRows(final Table t) throws IOException {
132 int count = 0;
133 try (ResultScanner scanner = t.getScanner(new Scan())) {
134 while(scanner.next() != null) count++;
135 }
136 return count;
137 }
138 }