View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master;
19  
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertTrue;
24  
25  import java.io.IOException;
26  import java.net.InetAddress;
27  import java.net.UnknownHostException;
28  import java.util.ArrayList;
29  import java.util.List;
30  import java.util.Set;
31  
32  import org.apache.hadoop.conf.Configuration;
33  import org.apache.hadoop.hbase.Abortable;
34  import org.apache.hadoop.hbase.TableName;
35  import org.apache.hadoop.hbase.exceptions.DeserializationException;
36  import org.apache.hadoop.hbase.HBaseTestingUtility;
37  import org.apache.hadoop.hbase.HConstants;
38  import org.apache.hadoop.hbase.HRegionInfo;
39  import org.apache.hadoop.hbase.ServerLoad;
40  import org.apache.hadoop.hbase.Server;
41  import org.apache.hadoop.hbase.ServerName;
42  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
43  import org.apache.hadoop.hbase.catalog.CatalogTracker;
44  import org.apache.hadoop.hbase.catalog.MetaMockingUtil;
45  import org.apache.hadoop.hbase.client.HConnection;
46  import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
47  import org.apache.hadoop.hbase.client.Result;
48  import org.apache.hadoop.hbase.monitoring.MonitoredTask;
49  import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
50  import org.apache.hadoop.hbase.util.FSUtils;
51  import org.apache.hadoop.hbase.util.Threads;
52  import org.apache.hadoop.hbase.zookeeper.MetaRegionTracker;
53  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
54  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
55  import org.apache.hadoop.hbase.MediumTests;
56  import org.apache.zookeeper.KeeperException;
57  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
58  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair;
59  import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest;
60  import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStartupRequest;
61  import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStartupResponse;
62  import com.google.protobuf.ServiceException;
63  import org.junit.After;
64  import org.junit.AfterClass;
65  import org.junit.BeforeClass;
66  import org.junit.Test;
67  import org.mockito.Mockito;
68  import org.junit.experimental.categories.Category;
69  import org.apache.commons.logging.Log;
70  import org.apache.commons.logging.LogFactory;
71  
72  /**
73   * Standup the master and fake it to test various aspects of master function.
74   * Does NOT spin up a mini hbase nor mini dfs cluster testing master (it does
75   * put up a zk cluster but this is usually pretty fast compared).  Also, should
76   * be possible to inject faults at points difficult to get at in cluster context.
77   * TODO: Speed up the zk connection by Master.  It pauses 5 seconds establishing
78   * session.
79   */
80  @Category(MediumTests.class)
81  public class TestMasterNoCluster {
82    private static final Log LOG = LogFactory.getLog(TestMasterNoCluster.class);
83    private static final HBaseTestingUtility TESTUTIL = new HBaseTestingUtility();
84  
85    @BeforeClass
86    public static void setUpBeforeClass() throws Exception {
87      Configuration c = TESTUTIL.getConfiguration();
88      // We use local filesystem.  Set it so it writes into the testdir.
89      FSUtils.setRootDir(c, TESTUTIL.getDataTestDir());
90      // Startup a mini zk cluster.
91      TESTUTIL.startMiniZKCluster();
92    }
93  
94    @AfterClass
95    public static void tearDownAfterClass() throws Exception {
96      TESTUTIL.shutdownMiniZKCluster();
97    }
98  
99    @After
100   public void tearDown()
101   throws KeeperException, ZooKeeperConnectionException, IOException {
102     // Make sure zk is clean before we run the next test.
103     ZooKeeperWatcher zkw = new ZooKeeperWatcher(TESTUTIL.getConfiguration(),
104         "@Before", new Abortable() {
105       @Override
106       public void abort(String why, Throwable e) {
107         throw new RuntimeException(why, e);
108       }
109 
110       @Override
111       public boolean isAborted() {
112         return false;
113       }
114     });
115     ZKUtil.deleteNodeRecursively(zkw, zkw.baseZNode);
116     zkw.close();
117   }
118 
119   /**
120    * Test starting master then stopping it before its fully up.
121    * @throws IOException
122    * @throws KeeperException
123    * @throws InterruptedException
124    */
125   @Test (timeout=30000)
126   public void testStopDuringStart()
127   throws IOException, KeeperException, InterruptedException {
128     HMaster master = new HMaster(TESTUTIL.getConfiguration());
129     master.start();
130     // Immediately have it stop.  We used hang in assigning meta.
131     master.stopMaster();
132     master.join();
133   }
134 
135   /**
136    * Test master failover.
137    * Start up three fake regionservers and a master.
138    * @throws IOException
139    * @throws KeeperException
140    * @throws InterruptedException
141    */
142   @Test (timeout=30000)
143   public void testFailover()
144   throws IOException, KeeperException, InterruptedException, ServiceException {
145     final long now = System.currentTimeMillis();
146     // Names for our three servers.  Make the port numbers match hostname.
147     // Will come in use down in the server when we need to figure how to respond.
148     final ServerName sn0 = ServerName.valueOf("0.example.org", 0, now);
149     final ServerName sn1 = ServerName.valueOf("1.example.org", 1, now);
150     final ServerName sn2 = ServerName.valueOf("2.example.org", 2, now);
151     final ServerName [] sns = new ServerName [] {sn0, sn1, sn2};
152     // Put up the mock servers
153     final Configuration conf = TESTUTIL.getConfiguration();
154     final MockRegionServer rs0 = new MockRegionServer(conf, sn0);
155     final MockRegionServer rs1 = new MockRegionServer(conf, sn1);
156     final MockRegionServer rs2 = new MockRegionServer(conf, sn2);
157     // Put some data into the servers.  Make it look like sn0 has the metaH
158     // Put data into sn2 so it looks like it has a few regions for a table named 't'.
159     MetaRegionTracker.setMetaLocation(rs0.getZooKeeper(), rs0.getServerName());
160     final TableName tableName = TableName.valueOf("t");
161     Result [] results = new Result [] {
162       MetaMockingUtil.getMetaTableRowResult(
163         new HRegionInfo(tableName, HConstants.EMPTY_START_ROW, HBaseTestingUtility.KEYS[1]),
164         rs2.getServerName()),
165       MetaMockingUtil.getMetaTableRowResult(
166         new HRegionInfo(tableName, HBaseTestingUtility.KEYS[1], HBaseTestingUtility.KEYS[2]),
167         rs2.getServerName()),
168       MetaMockingUtil.getMetaTableRowResult(new HRegionInfo(tableName, HBaseTestingUtility.KEYS[2],
169           HConstants.EMPTY_END_ROW),
170         rs2.getServerName())
171     };
172     rs1.setNextResults(HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), results);
173 
174     // Create master.  Subclass to override a few methods so we can insert mocks
175     // and get notification on transitions.  We need to fake out any rpcs the
176     // master does opening/closing regions.  Also need to fake out the address
177     // of the 'remote' mocked up regionservers.
178     HMaster master = new HMaster(conf) {
179       InetAddress getRemoteInetAddress(final int port, final long serverStartCode)
180       throws UnknownHostException {
181         // Return different address dependent on port passed.
182         ServerName sn = sns[port];
183         return InetAddress.getByAddress(sn.getHostname(),
184           new byte [] {10, 0, 0, (byte)sn.getPort()});
185       }
186 
187       @Override
188       ServerManager createServerManager(Server master, MasterServices services)
189       throws IOException {
190         ServerManager sm = super.createServerManager(master, services);
191         // Spy on the created servermanager
192         ServerManager spy = Mockito.spy(sm);
193         // Fake a successful open.
194         Mockito.doReturn(RegionOpeningState.OPENED).when(spy).
195           sendRegionOpen((ServerName)Mockito.any(), (HRegionInfo)Mockito.any(),
196             Mockito.anyInt(), Mockito.anyListOf(ServerName.class));
197         return spy;
198       }
199 
200       @Override
201       CatalogTracker createCatalogTracker(ZooKeeperWatcher zk,
202           Configuration conf, Abortable abortable)
203       throws IOException {
204         // Insert a mock for the connection used by the CatalogTracker.  Any
205         // regionserver should do.  Use TESTUTIL.getConfiguration rather than
206         // the conf from the master; the conf will already have an HConnection
207         // associate so the below mocking of a connection will fail.
208         HConnection connection =
209           HConnectionTestingUtility.getMockedConnectionAndDecorate(TESTUTIL.getConfiguration(),
210             rs0, rs0, rs0.getServerName(), HRegionInfo.FIRST_META_REGIONINFO);
211         return new CatalogTracker(zk, conf, connection, abortable);
212       }
213 
214       @Override
215       void initNamespace() {
216       }
217     };
218     master.start();
219 
220     try {
221       // Wait till master is up ready for RPCs.
222       while (!master.isRpcServerOpen()) Threads.sleep(10);
223       // Fake master that there are regionservers out there.  Report in.
224       for (int i = 0; i < sns.length; i++) {
225         RegionServerReportRequest.Builder request = RegionServerReportRequest.newBuilder();;
226         ServerName sn = ServerName.parseVersionedServerName(sns[i].getVersionedBytes());
227         request.setServer(ProtobufUtil.toServerName(sn));
228         request.setLoad(ServerLoad.EMPTY_SERVERLOAD.obtainServerLoadPB());
229         master.regionServerReport(null, request.build());
230       }
231       // Master should now come up.
232       while (!master.isInitialized()) {Threads.sleep(10);}
233       assertTrue(master.isInitialized());
234     } finally {
235       rs0.stop("Test is done");
236       rs1.stop("Test is done");
237       rs2.stop("Test is done");
238       master.stopMaster();
239       master.join();
240     }
241   }
242 
243   /**
244    * Test starting master getting it up post initialized state using mocks.
245    * @throws IOException
246    * @throws KeeperException
247    * @throws InterruptedException
248    * @throws DeserializationException
249    * @throws ServiceException
250    */
251   @Test (timeout=60000)
252   public void testCatalogDeploys()
253       throws Exception {
254     final Configuration conf = TESTUTIL.getConfiguration();
255     conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
256     conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 1);
257 
258     final long now = System.currentTimeMillis();
259     // Name for our single mocked up regionserver.
260     final ServerName sn = ServerName.valueOf("0.example.org", 0, now);
261     // Here is our mocked up regionserver.  Create it now.  Need it setting up
262     // master next.
263     final MockRegionServer rs0 = new MockRegionServer(conf, sn);
264 
265     // Create master.  Subclass to override a few methods so we can insert mocks
266     // and get notification on transitions.  We need to fake out any rpcs the
267     // master does opening/closing regions.  Also need to fake out the address
268     // of the 'remote' mocked up regionservers.
269     HMaster master = new HMaster(conf) {
270       InetAddress getRemoteInetAddress(final int port, final long serverStartCode)
271       throws UnknownHostException {
272         // Interject an unchecked, nonsense InetAddress; i.e. no resolve.
273         return InetAddress.getByAddress(rs0.getServerName().getHostname(),
274           new byte [] {10, 0, 0, 0});
275       }
276 
277       @Override
278       ServerManager createServerManager(Server master, MasterServices services)
279       throws IOException {
280         ServerManager sm = super.createServerManager(master, services);
281         // Spy on the created servermanager
282         ServerManager spy = Mockito.spy(sm);
283         // Fake a successful open.
284         Mockito.doReturn(RegionOpeningState.OPENED).when(spy).
285           sendRegionOpen((ServerName)Mockito.any(), (HRegionInfo)Mockito.any(),
286             Mockito.anyInt(), Mockito.anyListOf(ServerName.class));
287         return spy;
288       }
289 
290       @Override
291       CatalogTracker createCatalogTracker(ZooKeeperWatcher zk,
292           Configuration conf, Abortable abortable)
293       throws IOException {
294         // Insert a mock for the connection used by the CatalogTracker.   Use
295         // TESTUTIL.getConfiguration rather than the conf from the master; the
296         // conf will already have an HConnection associate so the below mocking
297         // of a connection will fail.
298         HConnection connection =
299           HConnectionTestingUtility.getMockedConnectionAndDecorate(TESTUTIL.getConfiguration(),
300             rs0, rs0, rs0.getServerName(), HRegionInfo.FIRST_META_REGIONINFO);
301         return new CatalogTracker(zk, conf, connection, abortable);
302       }
303 
304       @Override
305       void initNamespace() {
306       }
307     };
308     master.start();
309     LOG.info("Master has started");
310 
311     try {
312       // Wait till master is up ready for RPCs.
313       while (!master.isRpcServerOpen()) Threads.sleep(10);
314       LOG.info("RpcServerOpen has started");
315 
316       // Fake master that there is a regionserver out there.  Report in.
317       RegionServerStartupRequest.Builder request = RegionServerStartupRequest.newBuilder();
318       request.setPort(rs0.getServerName().getPort());
319       request.setServerStartCode(rs0.getServerName().getStartcode());
320       request.setServerCurrentTime(now);
321       RegionServerStartupResponse result =
322         master.regionServerStartup(null, request.build());
323       String rshostname = new String();
324       for (NameStringPair e : result.getMapEntriesList()) {
325         if (e.getName().toString().equals(HConstants.KEY_FOR_HOSTNAME_SEEN_BY_MASTER)) {
326           rshostname = e.getValue();
327         }
328       }
329       // Assert hostname is as expected.
330       assertEquals(rs0.getServerName().getHostname(), rshostname);
331       // Now master knows there is at least one regionserver checked in and so
332       // it'll wait a while to see if more and when none, will assign meta
333       // to this single server.  Will do an rpc open but we've
334       // mocked it above in our master override to return 'success'.  As part of
335       // region open, master will have set an unassigned znode for the region up
336       // into zk for the regionserver to transition.  Lets do that now to
337       // complete fake of a successful open.
338       Mocking.fakeRegionServerRegionOpenInZK(master, rs0.getZooKeeper(),
339         rs0.getServerName(), HRegionInfo.FIRST_META_REGIONINFO);
340       LOG.info("fakeRegionServerRegionOpenInZK has started");
341 
342       // Need to set meta location as r0.  Usually the regionserver does this
343       // when its figured it just opened the meta region by setting the meta
344       // location up into zk.  Since we're mocking regionserver, need to do this
345       // ourselves.
346       MetaRegionTracker.setMetaLocation(rs0.getZooKeeper(), rs0.getServerName());
347       // Master should now come up.
348       while (!master.isInitialized()) {Threads.sleep(10);}
349       assertTrue(master.isInitialized());
350     } finally {
351       rs0.stop("Test is done");
352       master.stopMaster();
353       master.join();
354     }
355   }
356 
357   @Test
358   public void testNotPullingDeadRegionServerFromZK()
359       throws IOException, KeeperException, InterruptedException {
360     final Configuration conf = TESTUTIL.getConfiguration();
361     final ServerName newServer = ServerName.valueOf("test.sample", 1, 101);
362     final ServerName deadServer = ServerName.valueOf("test.sample", 1, 100);
363     final MockRegionServer rs0 = new MockRegionServer(conf, newServer);
364 
365     HMaster master = new HMaster(conf) {
366       @Override
367       void assignMeta(MonitoredTask status, Set<ServerName> previouslyFailedMeatRSs) {
368       }
369 
370       @Override
371       void initializeZKBasedSystemTrackers() throws IOException,
372       InterruptedException, KeeperException {
373         super.initializeZKBasedSystemTrackers();
374         // Record a newer server in server manager at first
375         serverManager.recordNewServerWithLock(newServer, ServerLoad.EMPTY_SERVERLOAD);
376 
377         List<ServerName> onlineServers = new ArrayList<ServerName>();
378         onlineServers.add(deadServer);
379         onlineServers.add(newServer);
380         // Mock the region server tracker to pull the dead server from zk
381         regionServerTracker = Mockito.spy(regionServerTracker);
382         Mockito.doReturn(onlineServers).when(
383           regionServerTracker).getOnlineServers();
384       }
385 
386       @Override
387       CatalogTracker createCatalogTracker(ZooKeeperWatcher zk,
388           Configuration conf, Abortable abortable)
389       throws IOException {
390         // Insert a mock for the connection used by the CatalogTracker.  Any
391         // regionserver should do.  Use TESTUTIL.getConfiguration rather than
392         // the conf from the master; the conf will already have an HConnection
393         // associate so the below mocking of a connection will fail.
394         HConnection connection =
395           HConnectionTestingUtility.getMockedConnectionAndDecorate(TESTUTIL.getConfiguration(),
396             rs0, rs0, rs0.getServerName(), HRegionInfo.FIRST_META_REGIONINFO);
397         return new CatalogTracker(zk, conf, connection, abortable);
398       }
399 
400       @Override
401       void initNamespace() {
402       }
403     };
404     master.start();
405 
406     try {
407       // Wait till master is initialized.
408       while (!master.initialized) Threads.sleep(10);
409       LOG.info("Master is initialized");
410 
411       assertFalse("The dead server should not be pulled in",
412         master.serverManager.isServerOnline(deadServer));
413     } finally {
414       master.stopMaster();
415       master.join();
416     }
417   }
418 }