1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertNotNull;
24 import static org.junit.Assert.assertNull;
25 import static org.junit.Assert.assertTrue;
26 import static org.junit.Assert.fail;
27
28 import java.io.IOException;
29 import java.lang.reflect.InvocationTargetException;
30 import java.lang.reflect.Method;
31 import java.util.List;
32 import java.util.Map;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.hbase.client.Get;
38 import org.apache.hadoop.hbase.client.HBaseAdmin;
39 import org.apache.hadoop.hbase.client.HConnection;
40 import org.apache.hadoop.hbase.client.HConnectionManager;
41 import org.apache.hadoop.hbase.client.HTable;
42 import org.apache.hadoop.hbase.client.Put;
43 import org.apache.hadoop.hbase.client.Result;
44 import org.apache.hadoop.hbase.client.ResultScanner;
45 import org.apache.hadoop.hbase.client.Scan;
46 import org.apache.hadoop.hbase.master.HMaster;
47 import org.apache.hadoop.hbase.master.LoadBalancer;
48 import org.apache.hadoop.hbase.master.balancer.SimpleLoadBalancer;
49 import org.apache.hadoop.hbase.util.Bytes;
50 import org.apache.hadoop.hbase.util.FSUtils;
51 import org.apache.hadoop.hbase.util.Threads;
52 import org.apache.hadoop.hbase.zookeeper.EmptyWatcher;
53 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
54 import org.apache.hadoop.hbase.zookeeper.ZKConfig;
55 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
56 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
57 import org.apache.zookeeper.CreateMode;
58 import org.apache.zookeeper.KeeperException;
59 import org.apache.zookeeper.ZooDefs;
60 import org.apache.zookeeper.ZooKeeper;
61 import org.apache.zookeeper.ZooKeeper.States;
62 import org.apache.zookeeper.data.ACL;
63 import org.apache.zookeeper.data.Stat;
64 import org.junit.After;
65 import org.junit.AfterClass;
66 import org.junit.Assert;
67 import org.junit.Before;
68 import org.junit.BeforeClass;
69 import org.junit.Test;
70 import org.junit.experimental.categories.Category;
71
72
73
74 @Category(LargeTests.class)
75 public class TestZooKeeper {
76 private final Log LOG = LogFactory.getLog(this.getClass());
77
78 private final static HBaseTestingUtility
79 TEST_UTIL = new HBaseTestingUtility();
80
81
82
83
84 @BeforeClass
85 public static void setUpBeforeClass() throws Exception {
86
87 Configuration conf = TEST_UTIL.getConfiguration();
88 TEST_UTIL.startMiniDFSCluster(2);
89 TEST_UTIL.startMiniZKCluster();
90 conf.setBoolean("dfs.support.append", true);
91 conf.setInt(HConstants.ZK_SESSION_TIMEOUT, 1000);
92 conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, MockLoadBalancer.class,
93 LoadBalancer.class);
94 }
95
96
97
98
99 @AfterClass
100 public static void tearDownAfterClass() throws Exception {
101 TEST_UTIL.shutdownMiniCluster();
102 }
103
104
105
106
107 @Before
108 public void setUp() throws Exception {
109 TEST_UTIL.startMiniHBaseCluster(1, 2);
110 }
111
112 @After
113 public void after() throws Exception {
114 try {
115 TEST_UTIL.shutdownMiniHBaseCluster();
116 } finally {
117 TEST_UTIL.getTestFileSystem().delete(FSUtils.getRootDir(TEST_UTIL.getConfiguration()), true);
118 ZKUtil.deleteNodeRecursively(TEST_UTIL.getZooKeeperWatcher(), "/hbase");
119 }
120 }
121
122 private ZooKeeperWatcher getZooKeeperWatcher(HConnection c)
123 throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
124 Method getterZK = c.getClass().getDeclaredMethod("getKeepAliveZooKeeperWatcher");
125 getterZK.setAccessible(true);
126 return (ZooKeeperWatcher) getterZK.invoke(c);
127 }
128
129
130
131
132
133
134
135
136
137 public void testClientSessionExpired() throws Exception {
138 Configuration c = new Configuration(TEST_UTIL.getConfiguration());
139
140
141 c.set(HConstants.HBASE_CLIENT_INSTANCE_ID, "1111");
142
143 HConnection connection = HConnectionManager.getConnection(c);
144
145 ZooKeeperWatcher connectionZK = getZooKeeperWatcher(connection);
146 LOG.info("ZooKeeperWatcher= 0x"+ Integer.toHexString(
147 connectionZK.hashCode()));
148 LOG.info("getRecoverableZooKeeper= 0x"+ Integer.toHexString(
149 connectionZK.getRecoverableZooKeeper().hashCode()));
150 LOG.info("session="+Long.toHexString(
151 connectionZK.getRecoverableZooKeeper().getSessionId()));
152
153 TEST_UTIL.expireSession(connectionZK);
154
155 LOG.info("Before using zkw state=" +
156 connectionZK.getRecoverableZooKeeper().getState());
157
158 try {
159 connectionZK.getRecoverableZooKeeper().getZooKeeper().exists(
160 "/1/1", false);
161 } catch (KeeperException ignored) {
162 }
163
164
165 States state = connectionZK.getRecoverableZooKeeper().getState();
166 LOG.info("After using zkw state=" + state);
167 LOG.info("session="+Long.toHexString(
168 connectionZK.getRecoverableZooKeeper().getSessionId()));
169
170
171 final long limit1 = System.currentTimeMillis() + 3000;
172 while (System.currentTimeMillis() < limit1 && state != States.CLOSED){
173 state = connectionZK.getRecoverableZooKeeper().getState();
174 }
175 LOG.info("After using zkw loop=" + state);
176 LOG.info("ZooKeeper should have timed out");
177 LOG.info("session="+Long.toHexString(
178 connectionZK.getRecoverableZooKeeper().getSessionId()));
179
180
181
182
183
184
185
186 ZooKeeperWatcher newConnectionZK = getZooKeeperWatcher(connection);
187
188 States state2 = newConnectionZK.getRecoverableZooKeeper().getState();
189 LOG.info("After new get state=" +state2);
190
191
192
193 final long limit2 = System.currentTimeMillis() + 3000;
194 while (System.currentTimeMillis() < limit2 &&
195 state2 != States.CONNECTED && state2 != States.CONNECTING) {
196
197 newConnectionZK = getZooKeeperWatcher(connection);
198 state2 = newConnectionZK.getRecoverableZooKeeper().getState();
199 }
200 LOG.info("After new get state loop=" + state2);
201
202 Assert.assertTrue(
203 state2 == States.CONNECTED || state2 == States.CONNECTING);
204
205 connection.close();
206 }
207
208 @Test (timeout = 60000)
209 public void testRegionServerSessionExpired() throws Exception {
210 LOG.info("Starting testRegionServerSessionExpired");
211 int metaIndex = TEST_UTIL.getMiniHBaseCluster().getServerWithMeta();
212 TEST_UTIL.expireRegionServerSession(metaIndex);
213 testSanity("testRegionServerSessionExpired");
214 }
215
216
217
218
219 public void testMasterSessionExpired() throws Exception {
220 LOG.info("Starting testMasterSessionExpired");
221 TEST_UTIL.expireMasterSession();
222 testSanity("testMasterSessionExpired");
223 }
224
225
226
227
228
229
230 @Test(timeout = 60000)
231 public void testMasterZKSessionRecoveryFailure() throws Exception {
232 LOG.info("Starting testMasterZKSessionRecoveryFailure");
233 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
234 HMaster m = cluster.getMaster();
235 m.abort("Test recovery from zk session expired",
236 new KeeperException.SessionExpiredException());
237 assertFalse(m.isStopped());
238 testSanity("testMasterZKSessionRecoveryFailure");
239 }
240
241
242
243
244
245 private void testSanity(final String testName) throws Exception{
246 String tableName = testName + "_" + System.currentTimeMillis();
247 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
248 HColumnDescriptor family = new HColumnDescriptor("fam");
249 desc.addFamily(family);
250 LOG.info("Creating table " + tableName);
251 HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
252 try {
253 admin.createTable(desc);
254 } finally {
255 admin.close();
256 }
257
258 HTable table =
259 new HTable(new Configuration(TEST_UTIL.getConfiguration()), tableName);
260 Put put = new Put(Bytes.toBytes("testrow"));
261 put.add(Bytes.toBytes("fam"),
262 Bytes.toBytes("col"), Bytes.toBytes("testdata"));
263 LOG.info("Putting table " + tableName);
264 table.put(put);
265 table.close();
266 }
267
268 @Test
269 public void testMultipleZK()
270 throws IOException, NoSuchMethodException, InvocationTargetException, IllegalAccessException {
271 HTable localMeta =
272 new HTable(new Configuration(TEST_UTIL.getConfiguration()), TableName.META_TABLE_NAME);
273 Configuration otherConf = new Configuration(TEST_UTIL.getConfiguration());
274 otherConf.set(HConstants.ZOOKEEPER_QUORUM, "127.0.0.1");
275 HTable ipMeta = new HTable(otherConf, TableName.META_TABLE_NAME);
276
277
278 final byte [] row = new byte [] {'r'};
279 localMeta.exists(new Get(row));
280 ipMeta.exists(new Get(row));
281
282
283 ZooKeeperWatcher z1 =
284 getZooKeeperWatcher(HConnectionManager.getConnection(localMeta.getConfiguration()));
285 ZooKeeperWatcher z2 =
286 getZooKeeperWatcher(HConnectionManager.getConnection(otherConf));
287 assertFalse(z1 == z2);
288 assertFalse(z1.getQuorum().equals(z2.getQuorum()));
289
290 localMeta.close();
291 ipMeta.close();
292 }
293
294
295
296
297
298 @Test
299 public void testCreateWithParents() throws Exception {
300 ZooKeeperWatcher zkw =
301 new ZooKeeperWatcher(new Configuration(TEST_UTIL.getConfiguration()),
302 TestZooKeeper.class.getName(), null);
303 byte[] expectedData = new byte[] { 1, 2, 3 };
304 ZKUtil.createWithParents(zkw, "/l1/l2/l3/l4/testCreateWithParents", expectedData);
305 byte[] data = ZKUtil.getData(zkw, "/l1/l2/l3/l4/testCreateWithParents");
306 assertTrue(Bytes.equals(expectedData, data));
307 ZKUtil.deleteNodeRecursively(zkw, "/l1");
308
309 ZKUtil.createWithParents(zkw, "/testCreateWithParents", expectedData);
310 data = ZKUtil.getData(zkw, "/testCreateWithParents");
311 assertTrue(Bytes.equals(expectedData, data));
312 ZKUtil.deleteNodeRecursively(zkw, "/testCreateWithParents");
313 }
314
315
316
317
318
319
320 @Test
321 public void testZNodeDeletes() throws Exception {
322 ZooKeeperWatcher zkw = new ZooKeeperWatcher(
323 new Configuration(TEST_UTIL.getConfiguration()),
324 TestZooKeeper.class.getName(), null);
325 ZKUtil.createWithParents(zkw, "/l1/l2/l3/l4");
326 try {
327 ZKUtil.deleteNode(zkw, "/l1/l2");
328 fail("We should not be able to delete if znode has childs");
329 } catch (KeeperException ex) {
330 assertNotNull(ZKUtil.getDataNoWatch(zkw, "/l1/l2/l3/l4", null));
331 }
332 ZKUtil.deleteNodeRecursively(zkw, "/l1/l2");
333
334 assertNull(ZKUtil.getDataNoWatch(zkw, "/l1/l2/l3/l4", null));
335
336
337 ZKUtil.deleteNodeRecursively(zkw, "/l1/l2");
338
339 ZKUtil.deleteNode(zkw, "/l1");
340 assertNull(ZKUtil.getDataNoWatch(zkw, "/l1/l2", null));
341 }
342
343 @Test
344 public void testClusterKey() throws Exception {
345 testKey("server", "2181", "hbase");
346 testKey("server1,server2,server3", "2181", "hbase");
347 try {
348 ZKUtil.transformClusterKey("2181:hbase");
349 } catch (IOException ex) {
350
351 }
352 }
353
354 private void testKey(String ensemble, String port, String znode)
355 throws IOException {
356 Configuration conf = new Configuration();
357 String key = ensemble+":"+port+":"+znode;
358 String[] parts = ZKUtil.transformClusterKey(key);
359 assertEquals(ensemble, parts[0]);
360 assertEquals(port, parts[1]);
361 assertEquals(znode, parts[2]);
362 ZKUtil.applyClusterKeyToConf(conf, key);
363 assertEquals(parts[0], conf.get(HConstants.ZOOKEEPER_QUORUM));
364 assertEquals(parts[1], conf.get(HConstants.ZOOKEEPER_CLIENT_PORT));
365 assertEquals(parts[2], conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT));
366 String reconstructedKey = ZKUtil.getZooKeeperClusterKey(conf);
367 assertEquals(key, reconstructedKey);
368 }
369
370
371
372
373
374
375
376
377 @Test
378 public void testCreateSilentIsReallySilent() throws InterruptedException,
379 KeeperException, IOException {
380 Configuration c = TEST_UTIL.getConfiguration();
381
382 String aclZnode = "/aclRoot";
383 String quorumServers = ZKConfig.getZKQuorumServersString(c);
384 int sessionTimeout = 5 * 1000;
385 ZooKeeper zk = new ZooKeeper(quorumServers, sessionTimeout, EmptyWatcher.instance);
386 zk.addAuthInfo("digest", "hbase:rox".getBytes());
387
388
389
390 ZooKeeperWatcher zk2 = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
391 "testCreateSilentIsReallySilent", null);
392
393
394 Stat s = null;
395 List<ACL> oldACL = null;
396 while (true) {
397 try {
398 s = new Stat();
399 oldACL = zk.getACL("/", s);
400 break;
401 } catch (KeeperException e) {
402 switch (e.code()) {
403 case CONNECTIONLOSS:
404 case SESSIONEXPIRED:
405 case OPERATIONTIMEOUT:
406 LOG.warn("Possibly transient ZooKeeper exception", e);
407 Threads.sleep(100);
408 break;
409 default:
410 throw e;
411 }
412 }
413 }
414
415
416
417 while (true) {
418 try {
419 zk.setACL("/", ZooDefs.Ids.CREATOR_ALL_ACL, -1);
420 break;
421 } catch (KeeperException e) {
422 switch (e.code()) {
423 case CONNECTIONLOSS:
424 case SESSIONEXPIRED:
425 case OPERATIONTIMEOUT:
426 LOG.warn("Possibly transient ZooKeeper exception: " + e);
427 Threads.sleep(100);
428 break;
429 default:
430 throw e;
431 }
432 }
433 }
434
435 while (true) {
436 try {
437 zk.create(aclZnode, null, ZooDefs.Ids.CREATOR_ALL_ACL, CreateMode.PERSISTENT);
438 break;
439 } catch (KeeperException e) {
440 switch (e.code()) {
441 case CONNECTIONLOSS:
442 case SESSIONEXPIRED:
443 case OPERATIONTIMEOUT:
444 LOG.warn("Possibly transient ZooKeeper exception: " + e);
445 Threads.sleep(100);
446 break;
447 default:
448 throw e;
449 }
450 }
451 }
452 zk.close();
453 ZKUtil.createAndFailSilent(zk2, aclZnode);
454
455
456 ZooKeeper zk3 = new ZooKeeper(quorumServers, sessionTimeout, EmptyWatcher.instance);
457 zk3.addAuthInfo("digest", "hbase:rox".getBytes());
458 try {
459 zk3.setACL("/", oldACL, -1);
460 } finally {
461 zk3.close();
462 }
463 }
464
465
466
467
468
469 @Test
470 @SuppressWarnings("deprecation")
471 public void testGetChildDataAndWatchForNewChildrenShouldNotThrowNPE()
472 throws Exception {
473 ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
474 "testGetChildDataAndWatchForNewChildrenShouldNotThrowNPE", null);
475 ZKUtil.getChildDataAndWatchForNewChildren(zkw, "/wrongNode");
476 }
477
478
479
480
481
482
483 @Test
484 public void testRegionAssignmentAfterMasterRecoveryDueToZKExpiry() throws Exception {
485 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
486 cluster.startRegionServer();
487 cluster.waitForActiveAndReadyMaster(10000);
488 HMaster m = cluster.getMaster();
489 ZooKeeperWatcher zkw = m.getZooKeeperWatcher();
490 int expectedNumOfListeners = zkw.getNumberOfListeners();
491
492 HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
493 try {
494 byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"),
495 Bytes.toBytes("c"), Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"),
496 Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i"), Bytes.toBytes("j") };
497 String tableName = "testRegionAssignmentAfterMasterRecoveryDueToZKExpiry";
498 admin.createTable(new HTableDescriptor(TableName.valueOf(tableName)), SPLIT_KEYS);
499 ZooKeeperWatcher zooKeeperWatcher = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
500 ZKAssign.blockUntilNoRIT(zooKeeperWatcher);
501 m.getZooKeeperWatcher().close();
502 MockLoadBalancer.retainAssignCalled = false;
503 m.abort("Test recovery from zk session expired",
504 new KeeperException.SessionExpiredException());
505 assertFalse(m.isStopped());
506
507
508 assertFalse("Retain assignment should not be called", MockLoadBalancer.retainAssignCalled);
509
510
511 cluster.waitForActiveAndReadyMaster(10000);
512 assertEquals(expectedNumOfListeners, zkw.getNumberOfListeners());
513 } finally {
514 admin.close();
515 }
516 }
517
518
519
520
521
522 @Test(timeout = 240000)
523 public void testLogSplittingAfterMasterRecoveryDueToZKExpiry() throws IOException,
524 KeeperException, InterruptedException {
525 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
526 cluster.startRegionServer();
527 HMaster m = cluster.getMaster();
528
529 HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
530 HTable table = null;
531 try {
532 byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("1"), Bytes.toBytes("2"),
533 Bytes.toBytes("3"), Bytes.toBytes("4"), Bytes.toBytes("5") };
534
535 String tableName = "testLogSplittingAfterMasterRecoveryDueToZKExpiry";
536 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
537 HColumnDescriptor hcd = new HColumnDescriptor("col");
538 htd.addFamily(hcd);
539 admin.createTable(htd, SPLIT_KEYS);
540 ZooKeeperWatcher zooKeeperWatcher = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
541 ZKAssign.blockUntilNoRIT(zooKeeperWatcher);
542 table = new HTable(TEST_UTIL.getConfiguration(), tableName);
543 Put p;
544 int numberOfPuts;
545 for (numberOfPuts = 0; numberOfPuts < 6; numberOfPuts++) {
546 p = new Put(Bytes.toBytes(numberOfPuts));
547 p.add(Bytes.toBytes("col"), Bytes.toBytes("ql"), Bytes.toBytes("value" + numberOfPuts));
548 table.put(p);
549 }
550 m.getZooKeeperWatcher().close();
551 m.abort("Test recovery from zk session expired",
552 new KeeperException.SessionExpiredException());
553 assertFalse(m.isStopped());
554 cluster.getRegionServer(0).abort("Aborting");
555
556
557 Scan scan = new Scan();
558 int numberOfRows = 0;
559 ResultScanner scanner = table.getScanner(scan);
560 Result[] result = scanner.next(1);
561 while (result != null && result.length > 0) {
562 numberOfRows++;
563 result = scanner.next(1);
564 }
565 assertEquals("Number of rows should be equal to number of puts.", numberOfPuts,
566 numberOfRows);
567 } finally {
568 if (table != null) table.close();
569 admin.close();
570 }
571 }
572
573 static class MockLoadBalancer extends SimpleLoadBalancer {
574 static boolean retainAssignCalled = false;
575
576 @Override
577 public Map<ServerName, List<HRegionInfo>> retainAssignment(
578 Map<HRegionInfo, ServerName> regions, List<ServerName> servers) {
579 retainAssignCalled = true;
580 return super.retainAssignment(regions, servers);
581 }
582 }
583
584 }
585