1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master;
19
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertTrue;
24
25 import java.io.IOException;
26 import java.net.InetAddress;
27 import java.net.UnknownHostException;
28 import java.util.ArrayList;
29 import java.util.List;
30 import java.util.Set;
31
32 import org.apache.hadoop.conf.Configuration;
33 import org.apache.hadoop.hbase.Abortable;
34 import org.apache.hadoop.hbase.TableName;
35 import org.apache.hadoop.hbase.exceptions.DeserializationException;
36 import org.apache.hadoop.hbase.HBaseTestingUtility;
37 import org.apache.hadoop.hbase.HConstants;
38 import org.apache.hadoop.hbase.HRegionInfo;
39 import org.apache.hadoop.hbase.ServerLoad;
40 import org.apache.hadoop.hbase.Server;
41 import org.apache.hadoop.hbase.ServerName;
42 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
43 import org.apache.hadoop.hbase.catalog.CatalogTracker;
44 import org.apache.hadoop.hbase.catalog.MetaMockingUtil;
45 import org.apache.hadoop.hbase.client.HConnection;
46 import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
47 import org.apache.hadoop.hbase.client.Result;
48 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
49 import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
50 import org.apache.hadoop.hbase.util.FSUtils;
51 import org.apache.hadoop.hbase.util.Threads;
52 import org.apache.hadoop.hbase.zookeeper.MetaRegionTracker;
53 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
54 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
55 import org.apache.hadoop.hbase.MediumTests;
56 import org.apache.zookeeper.KeeperException;
57 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
58 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair;
59 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest;
60 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStartupRequest;
61 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStartupResponse;
62 import com.google.protobuf.ServiceException;
63 import org.junit.After;
64 import org.junit.AfterClass;
65 import org.junit.BeforeClass;
66 import org.junit.Test;
67 import org.mockito.Mockito;
68 import org.junit.experimental.categories.Category;
69 import org.apache.commons.logging.Log;
70 import org.apache.commons.logging.LogFactory;
71
72
73
74
75
76
77
78
79
80 @Category(MediumTests.class)
81 public class TestMasterNoCluster {
82 private static final Log LOG = LogFactory.getLog(TestMasterNoCluster.class);
83 private static final HBaseTestingUtility TESTUTIL = new HBaseTestingUtility();
84
85 @BeforeClass
86 public static void setUpBeforeClass() throws Exception {
87 Configuration c = TESTUTIL.getConfiguration();
88
89 FSUtils.setRootDir(c, TESTUTIL.getDataTestDir());
90
91 TESTUTIL.startMiniZKCluster();
92 }
93
94 @AfterClass
95 public static void tearDownAfterClass() throws Exception {
96 TESTUTIL.shutdownMiniZKCluster();
97 }
98
99 @After
100 public void tearDown()
101 throws KeeperException, ZooKeeperConnectionException, IOException {
102
103 ZooKeeperWatcher zkw = new ZooKeeperWatcher(TESTUTIL.getConfiguration(),
104 "@Before", new Abortable() {
105 @Override
106 public void abort(String why, Throwable e) {
107 throw new RuntimeException(why, e);
108 }
109
110 @Override
111 public boolean isAborted() {
112 return false;
113 }
114 });
115 ZKUtil.deleteNodeRecursively(zkw, zkw.baseZNode);
116 zkw.close();
117 }
118
119
120
121
122
123
124
125 @Test (timeout=30000)
126 public void testStopDuringStart()
127 throws IOException, KeeperException, InterruptedException {
128 HMaster master = new HMaster(TESTUTIL.getConfiguration());
129 master.start();
130
131 master.stopMaster();
132 master.join();
133 }
134
135
136
137
138
139
140
141
142 @Test (timeout=30000)
143 public void testFailover()
144 throws IOException, KeeperException, InterruptedException, ServiceException {
145 final long now = System.currentTimeMillis();
146
147
148 final ServerName sn0 = ServerName.valueOf("0.example.org", 0, now);
149 final ServerName sn1 = ServerName.valueOf("1.example.org", 1, now);
150 final ServerName sn2 = ServerName.valueOf("2.example.org", 2, now);
151 final ServerName [] sns = new ServerName [] {sn0, sn1, sn2};
152
153 final Configuration conf = TESTUTIL.getConfiguration();
154 final MockRegionServer rs0 = new MockRegionServer(conf, sn0);
155 final MockRegionServer rs1 = new MockRegionServer(conf, sn1);
156 final MockRegionServer rs2 = new MockRegionServer(conf, sn2);
157
158
159 MetaRegionTracker.setMetaLocation(rs0.getZooKeeper(), rs0.getServerName());
160 final TableName tableName = TableName.valueOf("t");
161 Result [] results = new Result [] {
162 MetaMockingUtil.getMetaTableRowResult(
163 new HRegionInfo(tableName, HConstants.EMPTY_START_ROW, HBaseTestingUtility.KEYS[1]),
164 rs2.getServerName()),
165 MetaMockingUtil.getMetaTableRowResult(
166 new HRegionInfo(tableName, HBaseTestingUtility.KEYS[1], HBaseTestingUtility.KEYS[2]),
167 rs2.getServerName()),
168 MetaMockingUtil.getMetaTableRowResult(new HRegionInfo(tableName, HBaseTestingUtility.KEYS[2],
169 HConstants.EMPTY_END_ROW),
170 rs2.getServerName())
171 };
172 rs1.setNextResults(HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), results);
173
174
175
176
177
178 HMaster master = new HMaster(conf) {
179 InetAddress getRemoteInetAddress(final int port, final long serverStartCode)
180 throws UnknownHostException {
181
182 ServerName sn = sns[port];
183 return InetAddress.getByAddress(sn.getHostname(),
184 new byte [] {10, 0, 0, (byte)sn.getPort()});
185 }
186
187 @Override
188 ServerManager createServerManager(Server master, MasterServices services)
189 throws IOException {
190 ServerManager sm = super.createServerManager(master, services);
191
192 ServerManager spy = Mockito.spy(sm);
193
194 Mockito.doReturn(RegionOpeningState.OPENED).when(spy).
195 sendRegionOpen((ServerName)Mockito.any(), (HRegionInfo)Mockito.any(),
196 Mockito.anyInt(), Mockito.anyListOf(ServerName.class));
197 return spy;
198 }
199
200 @Override
201 CatalogTracker createCatalogTracker(ZooKeeperWatcher zk,
202 Configuration conf, Abortable abortable)
203 throws IOException {
204
205
206
207
208 HConnection connection =
209 HConnectionTestingUtility.getMockedConnectionAndDecorate(TESTUTIL.getConfiguration(),
210 rs0, rs0, rs0.getServerName(), HRegionInfo.FIRST_META_REGIONINFO);
211 return new CatalogTracker(zk, conf, connection, abortable);
212 }
213
214 @Override
215 void initNamespace() {
216 }
217 };
218 master.start();
219
220 try {
221
222 while (!master.isRpcServerOpen()) Threads.sleep(10);
223
224 for (int i = 0; i < sns.length; i++) {
225 RegionServerReportRequest.Builder request = RegionServerReportRequest.newBuilder();;
226 ServerName sn = ServerName.parseVersionedServerName(sns[i].getVersionedBytes());
227 request.setServer(ProtobufUtil.toServerName(sn));
228 request.setLoad(ServerLoad.EMPTY_SERVERLOAD.obtainServerLoadPB());
229 master.regionServerReport(null, request.build());
230 }
231
232 while (!master.isInitialized()) {Threads.sleep(10);}
233 assertTrue(master.isInitialized());
234 } finally {
235 rs0.stop("Test is done");
236 rs1.stop("Test is done");
237 rs2.stop("Test is done");
238 master.stopMaster();
239 master.join();
240 }
241 }
242
243
244
245
246
247
248
249
250
251 @Test (timeout=60000)
252 public void testCatalogDeploys()
253 throws Exception {
254 final Configuration conf = TESTUTIL.getConfiguration();
255 conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
256 conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 1);
257
258 final long now = System.currentTimeMillis();
259
260 final ServerName sn = ServerName.valueOf("0.example.org", 0, now);
261
262
263 final MockRegionServer rs0 = new MockRegionServer(conf, sn);
264
265
266
267
268
269 HMaster master = new HMaster(conf) {
270 InetAddress getRemoteInetAddress(final int port, final long serverStartCode)
271 throws UnknownHostException {
272
273 return InetAddress.getByAddress(rs0.getServerName().getHostname(),
274 new byte [] {10, 0, 0, 0});
275 }
276
277 @Override
278 ServerManager createServerManager(Server master, MasterServices services)
279 throws IOException {
280 ServerManager sm = super.createServerManager(master, services);
281
282 ServerManager spy = Mockito.spy(sm);
283
284 Mockito.doReturn(RegionOpeningState.OPENED).when(spy).
285 sendRegionOpen((ServerName)Mockito.any(), (HRegionInfo)Mockito.any(),
286 Mockito.anyInt(), Mockito.anyListOf(ServerName.class));
287 return spy;
288 }
289
290 @Override
291 CatalogTracker createCatalogTracker(ZooKeeperWatcher zk,
292 Configuration conf, Abortable abortable)
293 throws IOException {
294
295
296
297
298 HConnection connection =
299 HConnectionTestingUtility.getMockedConnectionAndDecorate(TESTUTIL.getConfiguration(),
300 rs0, rs0, rs0.getServerName(), HRegionInfo.FIRST_META_REGIONINFO);
301 return new CatalogTracker(zk, conf, connection, abortable);
302 }
303
304 @Override
305 void initNamespace() {
306 }
307 };
308 master.start();
309 LOG.info("Master has started");
310
311 try {
312
313 while (!master.isRpcServerOpen()) Threads.sleep(10);
314 LOG.info("RpcServerOpen has started");
315
316
317 RegionServerStartupRequest.Builder request = RegionServerStartupRequest.newBuilder();
318 request.setPort(rs0.getServerName().getPort());
319 request.setServerStartCode(rs0.getServerName().getStartcode());
320 request.setServerCurrentTime(now);
321 RegionServerStartupResponse result =
322 master.regionServerStartup(null, request.build());
323 String rshostname = new String();
324 for (NameStringPair e : result.getMapEntriesList()) {
325 if (e.getName().toString().equals(HConstants.KEY_FOR_HOSTNAME_SEEN_BY_MASTER)) {
326 rshostname = e.getValue();
327 }
328 }
329
330 assertEquals(rs0.getServerName().getHostname(), rshostname);
331
332
333
334
335
336
337
338 Mocking.fakeRegionServerRegionOpenInZK(master, rs0.getZooKeeper(),
339 rs0.getServerName(), HRegionInfo.FIRST_META_REGIONINFO);
340 LOG.info("fakeRegionServerRegionOpenInZK has started");
341
342
343
344
345
346 MetaRegionTracker.setMetaLocation(rs0.getZooKeeper(), rs0.getServerName());
347
348 while (!master.isInitialized()) {Threads.sleep(10);}
349 assertTrue(master.isInitialized());
350 } finally {
351 rs0.stop("Test is done");
352 master.stopMaster();
353 master.join();
354 }
355 }
356
357 @Test
358 public void testNotPullingDeadRegionServerFromZK()
359 throws IOException, KeeperException, InterruptedException {
360 final Configuration conf = TESTUTIL.getConfiguration();
361 final ServerName newServer = ServerName.valueOf("test.sample", 1, 101);
362 final ServerName deadServer = ServerName.valueOf("test.sample", 1, 100);
363 final MockRegionServer rs0 = new MockRegionServer(conf, newServer);
364
365 HMaster master = new HMaster(conf) {
366 @Override
367 void assignMeta(MonitoredTask status, Set<ServerName> previouslyFailedMeatRSs) {
368 }
369
370 @Override
371 void initializeZKBasedSystemTrackers() throws IOException,
372 InterruptedException, KeeperException {
373 super.initializeZKBasedSystemTrackers();
374
375 serverManager.recordNewServerWithLock(newServer, ServerLoad.EMPTY_SERVERLOAD);
376
377 List<ServerName> onlineServers = new ArrayList<ServerName>();
378 onlineServers.add(deadServer);
379 onlineServers.add(newServer);
380
381 regionServerTracker = Mockito.spy(regionServerTracker);
382 Mockito.doReturn(onlineServers).when(
383 regionServerTracker).getOnlineServers();
384 }
385
386 @Override
387 CatalogTracker createCatalogTracker(ZooKeeperWatcher zk,
388 Configuration conf, Abortable abortable)
389 throws IOException {
390
391
392
393
394 HConnection connection =
395 HConnectionTestingUtility.getMockedConnectionAndDecorate(TESTUTIL.getConfiguration(),
396 rs0, rs0, rs0.getServerName(), HRegionInfo.FIRST_META_REGIONINFO);
397 return new CatalogTracker(zk, conf, connection, abortable);
398 }
399
400 @Override
401 void initNamespace() {
402 }
403 };
404 master.start();
405
406 try {
407
408 while (!master.initialized) Threads.sleep(10);
409 LOG.info("Master is initialized");
410
411 assertFalse("The dead server should not be pulled in",
412 master.serverManager.isServerOnline(deadServer));
413 } finally {
414 master.stopMaster();
415 master.join();
416 }
417 }
418 }