1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master;
20
21 import static org.junit.Assert.assertFalse;
22 import static org.junit.Assert.assertNotNull;
23 import static org.junit.Assert.assertTrue;
24
25 import java.io.IOException;
26 import java.util.concurrent.Semaphore;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.hbase.HBaseTestingUtility;
32 import org.apache.hadoop.hbase.MediumTests;
33 import org.apache.hadoop.hbase.Server;
34 import org.apache.hadoop.hbase.ServerName;
35 import org.apache.hadoop.hbase.catalog.CatalogTracker;
36 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
37 import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
38 import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
39 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
40 import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
41 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
42 import org.apache.zookeeper.KeeperException;
43 import org.junit.AfterClass;
44 import org.junit.BeforeClass;
45 import org.junit.Test;
46 import org.junit.experimental.categories.Category;
47 import org.mockito.Mockito;
48
49
50
51
52 @Category(MediumTests.class)
53 public class TestActiveMasterManager {
54 private final static Log LOG = LogFactory.getLog(TestActiveMasterManager.class);
55 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
56
57 @BeforeClass
58 public static void setUpBeforeClass() throws Exception {
59 TEST_UTIL.startMiniZKCluster();
60 }
61
62 @AfterClass
63 public static void tearDownAfterClass() throws Exception {
64 TEST_UTIL.shutdownMiniZKCluster();
65 }
66
67 @Test public void testRestartMaster() throws IOException, KeeperException {
68 ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
69 "testActiveMasterManagerFromZK", null, true);
70 try {
71 ZKUtil.deleteNode(zk, zk.getMasterAddressZNode());
72 ZKUtil.deleteNode(zk, zk.clusterStateZNode);
73 } catch(KeeperException.NoNodeException nne) {}
74
75
76 ServerName master = ServerName.valueOf("localhost", 1, System.currentTimeMillis());
77
78 DummyMaster dummyMaster = new DummyMaster(zk,master);
79 ClusterStatusTracker clusterStatusTracker =
80 dummyMaster.getClusterStatusTracker();
81 ActiveMasterManager activeMasterManager =
82 dummyMaster.getActiveMasterManager();
83 assertFalse(activeMasterManager.clusterHasActiveMaster.get());
84
85
86 MonitoredTask status = Mockito.mock(MonitoredTask.class);
87 clusterStatusTracker.setClusterUp();
88
89 activeMasterManager.blockUntilBecomingActiveMaster(status);
90 assertTrue(activeMasterManager.clusterHasActiveMaster.get());
91 assertMaster(zk, master);
92
93
94 DummyMaster secondDummyMaster = new DummyMaster(zk,master);
95 ActiveMasterManager secondActiveMasterManager =
96 secondDummyMaster.getActiveMasterManager();
97 assertFalse(secondActiveMasterManager.clusterHasActiveMaster.get());
98 activeMasterManager.blockUntilBecomingActiveMaster(status);
99 assertTrue(activeMasterManager.clusterHasActiveMaster.get());
100 assertMaster(zk, master);
101 }
102
103
104
105
106
107
108 @Test
109 public void testActiveMasterManagerFromZK() throws Exception {
110 ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
111 "testActiveMasterManagerFromZK", null, true);
112 try {
113 ZKUtil.deleteNode(zk, zk.getMasterAddressZNode());
114 ZKUtil.deleteNode(zk, zk.clusterStateZNode);
115 } catch(KeeperException.NoNodeException nne) {}
116
117
118 ServerName firstMasterAddress =
119 ServerName.valueOf("localhost", 1, System.currentTimeMillis());
120 ServerName secondMasterAddress =
121 ServerName.valueOf("localhost", 2, System.currentTimeMillis());
122
123
124 DummyMaster ms1 = new DummyMaster(zk,firstMasterAddress);
125 ActiveMasterManager activeMasterManager =
126 ms1.getActiveMasterManager();
127 assertFalse(activeMasterManager.clusterHasActiveMaster.get());
128
129
130 ClusterStatusTracker clusterStatusTracker =
131 ms1.getClusterStatusTracker();
132 clusterStatusTracker.setClusterUp();
133 activeMasterManager.blockUntilBecomingActiveMaster(
134 Mockito.mock(MonitoredTask.class));
135 assertTrue(activeMasterManager.clusterHasActiveMaster.get());
136 assertMaster(zk, firstMasterAddress);
137
138
139 WaitToBeMasterThread t = new WaitToBeMasterThread(zk, secondMasterAddress);
140 t.start();
141
142
143 int sleeps = 0;
144 while(!t.manager.clusterHasActiveMaster.get() && sleeps < 100) {
145 Thread.sleep(10);
146 sleeps++;
147 }
148
149
150 assertTrue(activeMasterManager.clusterHasActiveMaster.get());
151 assertTrue(t.manager.clusterHasActiveMaster.get());
152
153 assertFalse(t.isActiveMaster);
154
155
156 ms1.stop("stopping first server");
157
158
159 NodeDeletionListener listener = new NodeDeletionListener(zk, zk.getMasterAddressZNode());
160 zk.registerListener(listener);
161
162 LOG.info("Deleting master node");
163 ZKUtil.deleteNode(zk, zk.getMasterAddressZNode());
164
165
166 LOG.info("Waiting for active master manager to be notified");
167 listener.waitForDeletion();
168 LOG.info("Master node deleted");
169
170
171
172 sleeps = 0;
173 while(!t.isActiveMaster && sleeps < 100) {
174 Thread.sleep(10);
175 sleeps++;
176 }
177 LOG.debug("Slept " + sleeps + " times");
178
179 assertTrue(t.manager.clusterHasActiveMaster.get());
180 assertTrue(t.isActiveMaster);
181
182 LOG.info("Deleting master node");
183 ZKUtil.deleteNode(zk, zk.getMasterAddressZNode());
184 }
185
186
187
188
189
190
191
192
193 private void assertMaster(ZooKeeperWatcher zk,
194 ServerName expectedAddress)
195 throws KeeperException, IOException {
196 ServerName readAddress = MasterAddressTracker.getMasterAddress(zk);
197 assertNotNull(readAddress);
198 assertTrue(expectedAddress.equals(readAddress));
199 }
200
201 public static class WaitToBeMasterThread extends Thread {
202
203 ActiveMasterManager manager;
204 DummyMaster dummyMaster;
205 boolean isActiveMaster;
206
207 public WaitToBeMasterThread(ZooKeeperWatcher zk, ServerName address) {
208 this.dummyMaster = new DummyMaster(zk,address);
209 this.manager = this.dummyMaster.getActiveMasterManager();
210 isActiveMaster = false;
211 }
212
213 @Override
214 public void run() {
215 manager.blockUntilBecomingActiveMaster(
216 Mockito.mock(MonitoredTask.class));
217 LOG.info("Second master has become the active master!");
218 isActiveMaster = true;
219 }
220 }
221
222 public static class NodeDeletionListener extends ZooKeeperListener {
223 private static final Log LOG = LogFactory.getLog(NodeDeletionListener.class);
224
225 private Semaphore lock;
226 private String node;
227
228 public NodeDeletionListener(ZooKeeperWatcher watcher, String node) {
229 super(watcher);
230 lock = new Semaphore(0);
231 this.node = node;
232 }
233
234 @Override
235 public void nodeDeleted(String path) {
236 if(path.equals(node)) {
237 LOG.debug("nodeDeleted(" + path + ")");
238 lock.release();
239 }
240 }
241
242 public void waitForDeletion() throws InterruptedException {
243 lock.acquire();
244 }
245 }
246
247
248
249
250 public static class DummyMaster implements Server {
251 private volatile boolean stopped;
252 private ClusterStatusTracker clusterStatusTracker;
253 private ActiveMasterManager activeMasterManager;
254
255 public DummyMaster(ZooKeeperWatcher zk, ServerName master) {
256 this.clusterStatusTracker =
257 new ClusterStatusTracker(zk, this);
258 clusterStatusTracker.start();
259
260 this.activeMasterManager =
261 new ActiveMasterManager(zk, master, this);
262 zk.registerListener(activeMasterManager);
263 }
264
265 @Override
266 public void abort(final String msg, final Throwable t) {}
267
268 @Override
269 public boolean isAborted() {
270 return false;
271 }
272
273 @Override
274 public Configuration getConfiguration() {
275 return null;
276 }
277
278 @Override
279 public ZooKeeperWatcher getZooKeeper() {
280 return null;
281 }
282
283 @Override
284 public ServerName getServerName() {
285 return null;
286 }
287
288 @Override
289 public boolean isStopped() {
290 return this.stopped;
291 }
292
293 @Override
294 public void stop(String why) {
295 this.stopped = true;
296 }
297
298 @Override
299 public CatalogTracker getCatalogTracker() {
300 return null;
301 }
302
303 public ClusterStatusTracker getClusterStatusTracker() {
304 return clusterStatusTracker;
305 }
306
307 public ActiveMasterManager getActiveMasterManager() {
308 return activeMasterManager;
309 }
310 }
311
312 }
313