View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master;
20  
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertNotNull;
23  import static org.junit.Assert.assertTrue;
24  
25  import java.io.IOException;
26  import java.util.concurrent.Semaphore;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.hbase.HBaseTestingUtility;
32  import org.apache.hadoop.hbase.MediumTests;
33  import org.apache.hadoop.hbase.Server;
34  import org.apache.hadoop.hbase.ServerName;
35  import org.apache.hadoop.hbase.catalog.CatalogTracker;
36  import org.apache.hadoop.hbase.monitoring.MonitoredTask;
37  import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
38  import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
39  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
40  import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
41  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
42  import org.apache.zookeeper.KeeperException;
43  import org.junit.AfterClass;
44  import org.junit.BeforeClass;
45  import org.junit.Test;
46  import org.junit.experimental.categories.Category;
47  import org.mockito.Mockito;
48  
49  /**
50   * Test the {@link ActiveMasterManager}.
51   */
52  @Category(MediumTests.class)
53  public class TestActiveMasterManager {
54    private final static Log LOG = LogFactory.getLog(TestActiveMasterManager.class);
55    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
56  
57    @BeforeClass
58    public static void setUpBeforeClass() throws Exception {
59      TEST_UTIL.startMiniZKCluster();
60    }
61  
62    @AfterClass
63    public static void tearDownAfterClass() throws Exception {
64      TEST_UTIL.shutdownMiniZKCluster();
65    }
66  
67    @Test public void testRestartMaster() throws IOException, KeeperException {
68      ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
69        "testActiveMasterManagerFromZK", null, true);
70      try {
71        ZKUtil.deleteNode(zk, zk.getMasterAddressZNode());
72        ZKUtil.deleteNode(zk, zk.clusterStateZNode);
73      } catch(KeeperException.NoNodeException nne) {}
74  
75      // Create the master node with a dummy address
76      ServerName master = ServerName.valueOf("localhost", 1, System.currentTimeMillis());
77      // Should not have a master yet
78      DummyMaster dummyMaster = new DummyMaster(zk,master);
79      ClusterStatusTracker clusterStatusTracker =
80        dummyMaster.getClusterStatusTracker();
81      ActiveMasterManager activeMasterManager =
82        dummyMaster.getActiveMasterManager();
83      assertFalse(activeMasterManager.clusterHasActiveMaster.get());
84  
85      // First test becoming the active master uninterrupted
86      MonitoredTask status = Mockito.mock(MonitoredTask.class);
87      clusterStatusTracker.setClusterUp();
88  
89      activeMasterManager.blockUntilBecomingActiveMaster(status);
90      assertTrue(activeMasterManager.clusterHasActiveMaster.get());
91      assertMaster(zk, master);
92  
93      // Now pretend master restart
94      DummyMaster secondDummyMaster = new DummyMaster(zk,master);
95      ActiveMasterManager secondActiveMasterManager =
96        secondDummyMaster.getActiveMasterManager();
97      assertFalse(secondActiveMasterManager.clusterHasActiveMaster.get());
98      activeMasterManager.blockUntilBecomingActiveMaster(status);
99      assertTrue(activeMasterManager.clusterHasActiveMaster.get());
100     assertMaster(zk, master);
101   }
102 
103   /**
104    * Unit tests that uses ZooKeeper but does not use the master-side methods
105    * but rather acts directly on ZK.
106    * @throws Exception
107    */
108   @Test
109   public void testActiveMasterManagerFromZK() throws Exception {
110     ZooKeeperWatcher zk = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
111       "testActiveMasterManagerFromZK", null, true);
112     try {
113       ZKUtil.deleteNode(zk, zk.getMasterAddressZNode());
114       ZKUtil.deleteNode(zk, zk.clusterStateZNode);
115     } catch(KeeperException.NoNodeException nne) {}
116 
117     // Create the master node with a dummy address
118     ServerName firstMasterAddress =
119         ServerName.valueOf("localhost", 1, System.currentTimeMillis());
120     ServerName secondMasterAddress =
121         ServerName.valueOf("localhost", 2, System.currentTimeMillis());
122 
123     // Should not have a master yet
124     DummyMaster ms1 = new DummyMaster(zk,firstMasterAddress);
125     ActiveMasterManager activeMasterManager =
126       ms1.getActiveMasterManager();
127     assertFalse(activeMasterManager.clusterHasActiveMaster.get());
128 
129     // First test becoming the active master uninterrupted
130     ClusterStatusTracker clusterStatusTracker =
131       ms1.getClusterStatusTracker();
132     clusterStatusTracker.setClusterUp();
133     activeMasterManager.blockUntilBecomingActiveMaster(
134         Mockito.mock(MonitoredTask.class));
135     assertTrue(activeMasterManager.clusterHasActiveMaster.get());
136     assertMaster(zk, firstMasterAddress);
137 
138     // New manager will now try to become the active master in another thread
139     WaitToBeMasterThread t = new WaitToBeMasterThread(zk, secondMasterAddress);
140     t.start();
141     // Wait for this guy to figure out there is another active master
142     // Wait for 1 second at most
143     int sleeps = 0;
144     while(!t.manager.clusterHasActiveMaster.get() && sleeps < 100) {
145       Thread.sleep(10);
146       sleeps++;
147     }
148 
149     // Both should see that there is an active master
150     assertTrue(activeMasterManager.clusterHasActiveMaster.get());
151     assertTrue(t.manager.clusterHasActiveMaster.get());
152     // But secondary one should not be the active master
153     assertFalse(t.isActiveMaster);
154 
155     // Close the first server and delete it's master node
156     ms1.stop("stopping first server");
157 
158     // Use a listener to capture when the node is actually deleted
159     NodeDeletionListener listener = new NodeDeletionListener(zk, zk.getMasterAddressZNode());
160     zk.registerListener(listener);
161 
162     LOG.info("Deleting master node");
163     ZKUtil.deleteNode(zk, zk.getMasterAddressZNode());
164 
165     // Wait for the node to be deleted
166     LOG.info("Waiting for active master manager to be notified");
167     listener.waitForDeletion();
168     LOG.info("Master node deleted");
169 
170     // Now we expect the secondary manager to have and be the active master
171     // Wait for 1 second at most
172     sleeps = 0;
173     while(!t.isActiveMaster && sleeps < 100) {
174       Thread.sleep(10);
175       sleeps++;
176     }
177     LOG.debug("Slept " + sleeps + " times");
178 
179     assertTrue(t.manager.clusterHasActiveMaster.get());
180     assertTrue(t.isActiveMaster);
181 
182     LOG.info("Deleting master node");
183     ZKUtil.deleteNode(zk, zk.getMasterAddressZNode());
184   }
185 
186   /**
187    * Assert there is an active master and that it has the specified address.
188    * @param zk
189    * @param thisMasterAddress
190    * @throws KeeperException
191    * @throws IOException 
192    */
193   private void assertMaster(ZooKeeperWatcher zk,
194       ServerName expectedAddress)
195   throws KeeperException, IOException {
196     ServerName readAddress = MasterAddressTracker.getMasterAddress(zk);
197     assertNotNull(readAddress);
198     assertTrue(expectedAddress.equals(readAddress));
199   }
200 
201   public static class WaitToBeMasterThread extends Thread {
202 
203     ActiveMasterManager manager;
204     DummyMaster dummyMaster;
205     boolean isActiveMaster;
206 
207     public WaitToBeMasterThread(ZooKeeperWatcher zk, ServerName address) {
208       this.dummyMaster = new DummyMaster(zk,address);
209       this.manager = this.dummyMaster.getActiveMasterManager();
210       isActiveMaster = false;
211     }
212 
213     @Override
214     public void run() {
215       manager.blockUntilBecomingActiveMaster(
216           Mockito.mock(MonitoredTask.class));
217       LOG.info("Second master has become the active master!");
218       isActiveMaster = true;
219     }
220   }
221 
222   public static class NodeDeletionListener extends ZooKeeperListener {
223     private static final Log LOG = LogFactory.getLog(NodeDeletionListener.class);
224 
225     private Semaphore lock;
226     private String node;
227 
228     public NodeDeletionListener(ZooKeeperWatcher watcher, String node) {
229       super(watcher);
230       lock = new Semaphore(0);
231       this.node = node;
232     }
233 
234     @Override
235     public void nodeDeleted(String path) {
236       if(path.equals(node)) {
237         LOG.debug("nodeDeleted(" + path + ")");
238         lock.release();
239       }
240     }
241 
242     public void waitForDeletion() throws InterruptedException {
243       lock.acquire();
244     }
245   }
246 
247   /**
248    * Dummy Master Implementation.
249    */
250   public static class DummyMaster implements Server {
251     private volatile boolean stopped;
252     private ClusterStatusTracker clusterStatusTracker;
253     private ActiveMasterManager activeMasterManager;
254 
255     public DummyMaster(ZooKeeperWatcher zk, ServerName master) {
256       this.clusterStatusTracker =
257         new ClusterStatusTracker(zk, this);
258       clusterStatusTracker.start();
259       
260       this.activeMasterManager =
261         new ActiveMasterManager(zk, master, this);
262       zk.registerListener(activeMasterManager);
263     }
264 
265     @Override
266     public void abort(final String msg, final Throwable t) {}
267     
268     @Override
269     public boolean isAborted() {
270       return false;
271     }
272 
273     @Override
274     public Configuration getConfiguration() {
275       return null;
276     }
277 
278     @Override
279     public ZooKeeperWatcher getZooKeeper() {
280       return null;
281     }
282 
283     @Override
284     public ServerName getServerName() {
285       return null;
286     }
287 
288     @Override
289     public boolean isStopped() {
290       return this.stopped;
291     }
292 
293     @Override
294     public void stop(String why) {
295       this.stopped = true;
296     }
297 
298     @Override
299     public CatalogTracker getCatalogTracker() {
300       return null;
301     }
302 
303     public ClusterStatusTracker getClusterStatusTracker() {
304       return clusterStatusTracker;
305     }
306 
307     public ActiveMasterManager getActiveMasterManager() {
308       return activeMasterManager;
309     }
310   }
311 
312 }
313