View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertNotNull;
24  import static org.junit.Assert.assertNull;
25  import static org.junit.Assert.assertTrue;
26  import static org.junit.Assert.fail;
27  
28  import java.io.IOException;
29  import java.lang.reflect.InvocationTargetException;
30  import java.lang.reflect.Method;
31  import java.util.List;
32  import java.util.Map;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.hbase.client.Get;
38  import org.apache.hadoop.hbase.client.HBaseAdmin;
39  import org.apache.hadoop.hbase.client.HConnection;
40  import org.apache.hadoop.hbase.client.HConnectionManager;
41  import org.apache.hadoop.hbase.client.HTable;
42  import org.apache.hadoop.hbase.client.Put;
43  import org.apache.hadoop.hbase.client.Result;
44  import org.apache.hadoop.hbase.client.ResultScanner;
45  import org.apache.hadoop.hbase.client.Scan;
46  import org.apache.hadoop.hbase.master.HMaster;
47  import org.apache.hadoop.hbase.master.LoadBalancer;
48  import org.apache.hadoop.hbase.master.balancer.SimpleLoadBalancer;
49  import org.apache.hadoop.hbase.util.Bytes;
50  import org.apache.hadoop.hbase.util.FSUtils;
51  import org.apache.hadoop.hbase.util.Threads;
52  import org.apache.hadoop.hbase.zookeeper.EmptyWatcher;
53  import org.apache.hadoop.hbase.zookeeper.ZKAssign;
54  import org.apache.hadoop.hbase.zookeeper.ZKConfig;
55  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
56  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
57  import org.apache.zookeeper.CreateMode;
58  import org.apache.zookeeper.KeeperException;
59  import org.apache.zookeeper.ZooDefs;
60  import org.apache.zookeeper.ZooKeeper;
61  import org.apache.zookeeper.ZooKeeper.States;
62  import org.apache.zookeeper.data.ACL;
63  import org.apache.zookeeper.data.Stat;
64  import org.junit.After;
65  import org.junit.AfterClass;
66  import org.junit.Assert;
67  import org.junit.Before;
68  import org.junit.BeforeClass;
69  import org.junit.Test;
70  import org.junit.experimental.categories.Category;
71  
72  
73  
74  @Category(LargeTests.class)
75  public class TestZooKeeper {
76    private final Log LOG = LogFactory.getLog(this.getClass());
77  
78    private final static HBaseTestingUtility
79        TEST_UTIL = new HBaseTestingUtility();
80  
81    /**
82     * @throws java.lang.Exception
83     */
84    @BeforeClass
85    public static void setUpBeforeClass() throws Exception {
86      // Test we can first start the ZK cluster by itself
87      Configuration conf = TEST_UTIL.getConfiguration();
88      TEST_UTIL.startMiniDFSCluster(2);
89      TEST_UTIL.startMiniZKCluster();
90      conf.setBoolean("dfs.support.append", true);
91      conf.setInt(HConstants.ZK_SESSION_TIMEOUT, 1000);
92      conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, MockLoadBalancer.class,
93          LoadBalancer.class);
94    }
95  
96    /**
97     * @throws java.lang.Exception
98     */
99    @AfterClass
100   public static void tearDownAfterClass() throws Exception {
101     TEST_UTIL.shutdownMiniCluster();
102   }
103 
104   /**
105    * @throws java.lang.Exception
106    */
107   @Before
108   public void setUp() throws Exception {
109     TEST_UTIL.startMiniHBaseCluster(1, 2);
110   }
111 
112   @After
113   public void after() throws Exception {
114     try {
115       TEST_UTIL.shutdownMiniHBaseCluster();
116     } finally {
117       TEST_UTIL.getTestFileSystem().delete(FSUtils.getRootDir(TEST_UTIL.getConfiguration()), true);
118       ZKUtil.deleteNodeRecursively(TEST_UTIL.getZooKeeperWatcher(), "/hbase");
119     }
120   }
121 
122   private ZooKeeperWatcher getZooKeeperWatcher(HConnection c)
123   throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
124     Method getterZK = c.getClass().getDeclaredMethod("getKeepAliveZooKeeperWatcher");
125     getterZK.setAccessible(true);
126     return (ZooKeeperWatcher) getterZK.invoke(c);
127   }
128 
129 
130   /**
131    * See HBASE-1232 and http://wiki.apache.org/hadoop/ZooKeeper/FAQ#4.
132    * @throws IOException
133    * @throws InterruptedException
134    */
135   // fails frequently, disabled for now, see HBASE-6406
136   //@Test
137   public void testClientSessionExpired() throws Exception {
138     Configuration c = new Configuration(TEST_UTIL.getConfiguration());
139 
140     // We don't want to share the connection as we will check its state
141     c.set(HConstants.HBASE_CLIENT_INSTANCE_ID, "1111");
142 
143     HConnection connection = HConnectionManager.getConnection(c);
144 
145     ZooKeeperWatcher connectionZK = getZooKeeperWatcher(connection);
146     LOG.info("ZooKeeperWatcher= 0x"+ Integer.toHexString(
147       connectionZK.hashCode()));
148     LOG.info("getRecoverableZooKeeper= 0x"+ Integer.toHexString(
149       connectionZK.getRecoverableZooKeeper().hashCode()));
150     LOG.info("session="+Long.toHexString(
151       connectionZK.getRecoverableZooKeeper().getSessionId()));
152 
153     TEST_UTIL.expireSession(connectionZK);
154 
155     LOG.info("Before using zkw state=" +
156       connectionZK.getRecoverableZooKeeper().getState());
157     // provoke session expiration by doing something with ZK
158     try {
159       connectionZK.getRecoverableZooKeeper().getZooKeeper().exists(
160         "/1/1", false);
161     } catch (KeeperException ignored) {
162     }
163 
164     // Check that the old ZK connection is closed, means we did expire
165     States state = connectionZK.getRecoverableZooKeeper().getState();
166     LOG.info("After using zkw state=" + state);
167     LOG.info("session="+Long.toHexString(
168       connectionZK.getRecoverableZooKeeper().getSessionId()));
169 
170     // It's asynchronous, so we may have to wait a little...
171     final long limit1 = System.currentTimeMillis() + 3000;
172     while (System.currentTimeMillis() < limit1 && state != States.CLOSED){
173       state = connectionZK.getRecoverableZooKeeper().getState();
174     }
175     LOG.info("After using zkw loop=" + state);
176     LOG.info("ZooKeeper should have timed out");
177     LOG.info("session="+Long.toHexString(
178       connectionZK.getRecoverableZooKeeper().getSessionId()));
179 
180     // It's surprising but sometimes we can still be in connected state.
181     // As it's known (even if not understood) we don't make the the test fail
182     // for this reason.)
183     // Assert.assertTrue("state=" + state, state == States.CLOSED);
184 
185     // Check that the client recovered
186     ZooKeeperWatcher newConnectionZK = getZooKeeperWatcher(connection);
187 
188     States state2 = newConnectionZK.getRecoverableZooKeeper().getState();
189     LOG.info("After new get state=" +state2);
190 
191     // As it's an asynchronous event we may got the same ZKW, if it's not
192     //  yet invalidated. Hence this loop.
193     final long limit2 = System.currentTimeMillis() + 3000;
194     while (System.currentTimeMillis() < limit2 &&
195       state2 != States.CONNECTED && state2 != States.CONNECTING) {
196 
197       newConnectionZK = getZooKeeperWatcher(connection);
198       state2 = newConnectionZK.getRecoverableZooKeeper().getState();
199     }
200     LOG.info("After new get state loop=" + state2);
201 
202     Assert.assertTrue(
203       state2 == States.CONNECTED || state2 == States.CONNECTING);
204 
205     connection.close();
206   }
207 
208   @Test (timeout = 60000)
209   public void testRegionServerSessionExpired() throws Exception {
210     LOG.info("Starting testRegionServerSessionExpired");
211     int metaIndex = TEST_UTIL.getMiniHBaseCluster().getServerWithMeta();
212     TEST_UTIL.expireRegionServerSession(metaIndex);
213     testSanity("testRegionServerSessionExpired");
214   }
215 
216   // @Test Disabled because seems to make no sense expiring master session
217   // and then trying to create table (down in testSanity); on master side
218   // it will fail because the master's session has expired -- St.Ack 07/24/2012
219   public void testMasterSessionExpired() throws Exception {
220     LOG.info("Starting testMasterSessionExpired");
221     TEST_UTIL.expireMasterSession();
222     testSanity("testMasterSessionExpired");
223   }
224 
225   /**
226    * Master recovery when the znode already exists. Internally, this
227    *  test differs from {@link #testMasterSessionExpired} because here
228    *  the master znode will exist in ZK.
229    */
230   @Test(timeout = 60000)
231   public void testMasterZKSessionRecoveryFailure() throws Exception {
232     LOG.info("Starting testMasterZKSessionRecoveryFailure");
233     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
234     HMaster m = cluster.getMaster();
235     m.abort("Test recovery from zk session expired",
236       new KeeperException.SessionExpiredException());
237     assertFalse(m.isStopped());
238     testSanity("testMasterZKSessionRecoveryFailure");
239   }
240 
241   /**
242    * Make sure we can use the cluster
243    * @throws Exception
244    */
245   private void testSanity(final String testName) throws Exception{
246     String tableName = testName + "_" + System.currentTimeMillis();
247     HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
248     HColumnDescriptor family = new HColumnDescriptor("fam");
249     desc.addFamily(family);
250     LOG.info("Creating table " + tableName);
251     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
252     try {
253       admin.createTable(desc);
254     } finally {
255       admin.close();
256     }
257 
258     HTable table =
259       new HTable(new Configuration(TEST_UTIL.getConfiguration()), tableName);
260     Put put = new Put(Bytes.toBytes("testrow"));
261     put.add(Bytes.toBytes("fam"),
262         Bytes.toBytes("col"), Bytes.toBytes("testdata"));
263     LOG.info("Putting table " + tableName);
264     table.put(put);
265     table.close();
266   }
267 
268   @Test
269   public void testMultipleZK()
270   throws IOException, NoSuchMethodException, InvocationTargetException, IllegalAccessException {
271     HTable localMeta =
272       new HTable(new Configuration(TEST_UTIL.getConfiguration()), TableName.META_TABLE_NAME);
273     Configuration otherConf = new Configuration(TEST_UTIL.getConfiguration());
274     otherConf.set(HConstants.ZOOKEEPER_QUORUM, "127.0.0.1");
275     HTable ipMeta = new HTable(otherConf, TableName.META_TABLE_NAME);
276 
277     // dummy, just to open the connection
278     final byte [] row = new byte [] {'r'};
279     localMeta.exists(new Get(row));
280     ipMeta.exists(new Get(row));
281 
282     // make sure they aren't the same
283     ZooKeeperWatcher z1 =
284       getZooKeeperWatcher(HConnectionManager.getConnection(localMeta.getConfiguration()));
285     ZooKeeperWatcher z2 =
286       getZooKeeperWatcher(HConnectionManager.getConnection(otherConf));
287     assertFalse(z1 == z2);
288     assertFalse(z1.getQuorum().equals(z2.getQuorum()));
289 
290     localMeta.close();
291     ipMeta.close();
292   }
293 
294   /**
295    * Create a znode with data
296    * @throws Exception
297    */
298   @Test
299   public void testCreateWithParents() throws Exception {
300     ZooKeeperWatcher zkw =
301         new ZooKeeperWatcher(new Configuration(TEST_UTIL.getConfiguration()),
302             TestZooKeeper.class.getName(), null);
303     byte[] expectedData = new byte[] { 1, 2, 3 };
304     ZKUtil.createWithParents(zkw, "/l1/l2/l3/l4/testCreateWithParents", expectedData);
305     byte[] data = ZKUtil.getData(zkw, "/l1/l2/l3/l4/testCreateWithParents");
306     assertTrue(Bytes.equals(expectedData, data));
307     ZKUtil.deleteNodeRecursively(zkw, "/l1");
308 
309     ZKUtil.createWithParents(zkw, "/testCreateWithParents", expectedData);
310     data = ZKUtil.getData(zkw, "/testCreateWithParents");
311     assertTrue(Bytes.equals(expectedData, data));
312     ZKUtil.deleteNodeRecursively(zkw, "/testCreateWithParents");
313   }
314 
315   /**
316    * Create a bunch of znodes in a hierarchy, try deleting one that has childs (it will fail), then
317    * delete it recursively, then delete the last znode
318    * @throws Exception
319    */
320   @Test
321   public void testZNodeDeletes() throws Exception {
322     ZooKeeperWatcher zkw = new ZooKeeperWatcher(
323       new Configuration(TEST_UTIL.getConfiguration()),
324       TestZooKeeper.class.getName(), null);
325     ZKUtil.createWithParents(zkw, "/l1/l2/l3/l4");
326     try {
327       ZKUtil.deleteNode(zkw, "/l1/l2");
328       fail("We should not be able to delete if znode has childs");
329     } catch (KeeperException ex) {
330       assertNotNull(ZKUtil.getDataNoWatch(zkw, "/l1/l2/l3/l4", null));
331     }
332     ZKUtil.deleteNodeRecursively(zkw, "/l1/l2");
333     // make sure it really is deleted
334     assertNull(ZKUtil.getDataNoWatch(zkw, "/l1/l2/l3/l4", null));
335 
336     // do the same delete again and make sure it doesn't crash
337     ZKUtil.deleteNodeRecursively(zkw, "/l1/l2");
338 
339     ZKUtil.deleteNode(zkw, "/l1");
340     assertNull(ZKUtil.getDataNoWatch(zkw, "/l1/l2", null));
341   }
342 
343   @Test
344   public void testClusterKey() throws Exception {
345     testKey("server", "2181", "hbase");
346     testKey("server1,server2,server3", "2181", "hbase");
347     try {
348       ZKUtil.transformClusterKey("2181:hbase");
349     } catch (IOException ex) {
350       // OK
351     }
352   }
353 
354   private void testKey(String ensemble, String port, String znode)
355       throws IOException {
356     Configuration conf = new Configuration();
357     String key = ensemble+":"+port+":"+znode;
358     String[] parts = ZKUtil.transformClusterKey(key);
359     assertEquals(ensemble, parts[0]);
360     assertEquals(port, parts[1]);
361     assertEquals(znode, parts[2]);
362     ZKUtil.applyClusterKeyToConf(conf, key);
363     assertEquals(parts[0], conf.get(HConstants.ZOOKEEPER_QUORUM));
364     assertEquals(parts[1], conf.get(HConstants.ZOOKEEPER_CLIENT_PORT));
365     assertEquals(parts[2], conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT));
366     String reconstructedKey = ZKUtil.getZooKeeperClusterKey(conf);
367     assertEquals(key, reconstructedKey);
368   }
369 
370   /**
371    * A test for HBASE-3238
372    * @throws IOException A connection attempt to zk failed
373    * @throws InterruptedException One of the non ZKUtil actions was interrupted
374    * @throws KeeperException Any of the zookeeper connections had a
375    * KeeperException
376    */
377   @Test
378   public void testCreateSilentIsReallySilent() throws InterruptedException,
379       KeeperException, IOException {
380     Configuration c = TEST_UTIL.getConfiguration();
381 
382     String aclZnode = "/aclRoot";
383     String quorumServers = ZKConfig.getZKQuorumServersString(c);
384     int sessionTimeout = 5 * 1000; // 5 seconds
385     ZooKeeper zk = new ZooKeeper(quorumServers, sessionTimeout, EmptyWatcher.instance);
386     zk.addAuthInfo("digest", "hbase:rox".getBytes());
387 
388     // Assumes the  root of the ZooKeeper space is writable as it creates a node
389     // wherever the cluster home is defined.
390     ZooKeeperWatcher zk2 = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
391       "testCreateSilentIsReallySilent", null);
392 
393     // Save the previous ACL
394     Stat s =  null;
395     List<ACL> oldACL = null;
396     while (true) {
397       try {
398         s = new Stat();
399         oldACL = zk.getACL("/", s);
400         break;
401       } catch (KeeperException e) {
402         switch (e.code()) {
403           case CONNECTIONLOSS:
404           case SESSIONEXPIRED:
405           case OPERATIONTIMEOUT:
406             LOG.warn("Possibly transient ZooKeeper exception", e);
407             Threads.sleep(100);
408             break;
409          default:
410             throw e;
411         }
412       }
413     }
414 
415     // I set this acl after the attempted creation of the cluster home node.
416     // Add retries in case of retryable zk exceptions.
417     while (true) {
418       try {
419         zk.setACL("/", ZooDefs.Ids.CREATOR_ALL_ACL, -1);
420         break;
421       } catch (KeeperException e) {
422         switch (e.code()) {
423           case CONNECTIONLOSS:
424           case SESSIONEXPIRED:
425           case OPERATIONTIMEOUT:
426             LOG.warn("Possibly transient ZooKeeper exception: " + e);
427             Threads.sleep(100);
428             break;
429          default:
430             throw e;
431         }
432       }
433     }
434 
435     while (true) {
436       try {
437         zk.create(aclZnode, null, ZooDefs.Ids.CREATOR_ALL_ACL, CreateMode.PERSISTENT);
438         break;
439       } catch (KeeperException e) {
440         switch (e.code()) {
441           case CONNECTIONLOSS:
442           case SESSIONEXPIRED:
443           case OPERATIONTIMEOUT:
444             LOG.warn("Possibly transient ZooKeeper exception: " + e);
445             Threads.sleep(100);
446             break;
447          default:
448             throw e;
449         }
450       }
451     }
452     zk.close();
453     ZKUtil.createAndFailSilent(zk2, aclZnode);
454 
455     // Restore the ACL
456     ZooKeeper zk3 = new ZooKeeper(quorumServers, sessionTimeout, EmptyWatcher.instance);
457     zk3.addAuthInfo("digest", "hbase:rox".getBytes());
458     try {
459       zk3.setACL("/", oldACL, -1);
460     } finally {
461       zk3.close();
462     }
463  }
464 
465   /**
466    * Test should not fail with NPE when getChildDataAndWatchForNewChildren
467    * invoked with wrongNode
468    */
469   @Test
470   @SuppressWarnings("deprecation")
471   public void testGetChildDataAndWatchForNewChildrenShouldNotThrowNPE()
472       throws Exception {
473     ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
474         "testGetChildDataAndWatchForNewChildrenShouldNotThrowNPE", null);
475     ZKUtil.getChildDataAndWatchForNewChildren(zkw, "/wrongNode");
476   }
477 
478   /**
479    * Tests that the master does not call retainAssignment after recovery from expired zookeeper
480    * session. Without the HBASE-6046 fix master always tries to assign all the user regions by
481    * calling retainAssignment.
482    */
483   @Test
484   public void testRegionAssignmentAfterMasterRecoveryDueToZKExpiry() throws Exception {
485     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
486     cluster.startRegionServer();
487     cluster.waitForActiveAndReadyMaster(10000);
488     HMaster m = cluster.getMaster();
489     ZooKeeperWatcher zkw = m.getZooKeeperWatcher();
490     int expectedNumOfListeners = zkw.getNumberOfListeners();
491     // now the cluster is up. So assign some regions.
492     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
493     try {
494       byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"),
495         Bytes.toBytes("c"), Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"),
496         Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i"), Bytes.toBytes("j") };
497       String tableName = "testRegionAssignmentAfterMasterRecoveryDueToZKExpiry";
498       admin.createTable(new HTableDescriptor(TableName.valueOf(tableName)), SPLIT_KEYS);
499       ZooKeeperWatcher zooKeeperWatcher = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
500       ZKAssign.blockUntilNoRIT(zooKeeperWatcher);
501       m.getZooKeeperWatcher().close();
502       MockLoadBalancer.retainAssignCalled = false;
503       m.abort("Test recovery from zk session expired",
504         new KeeperException.SessionExpiredException());
505       assertFalse(m.isStopped());
506       // The recovered master should not call retainAssignment, as it is not a
507       // clean startup.
508       assertFalse("Retain assignment should not be called", MockLoadBalancer.retainAssignCalled);
509       // number of listeners should be same as the value before master aborted
510       // wait for new master is initialized
511       cluster.waitForActiveAndReadyMaster(10000);
512       assertEquals(expectedNumOfListeners, zkw.getNumberOfListeners());
513     } finally {
514       admin.close();
515     }
516   }
517 
518   /**
519    * Tests whether the logs are split when master recovers from a expired zookeeper session and an
520    * RS goes down.
521    */
522   @Test(timeout = 240000)
523   public void testLogSplittingAfterMasterRecoveryDueToZKExpiry() throws IOException,
524       KeeperException, InterruptedException {
525     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
526     cluster.startRegionServer();
527     HMaster m = cluster.getMaster();
528     // now the cluster is up. So assign some regions.
529     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
530     HTable table = null;
531     try {
532       byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("1"), Bytes.toBytes("2"),
533         Bytes.toBytes("3"), Bytes.toBytes("4"), Bytes.toBytes("5") };
534 
535       String tableName = "testLogSplittingAfterMasterRecoveryDueToZKExpiry";
536       HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
537       HColumnDescriptor hcd = new HColumnDescriptor("col");
538       htd.addFamily(hcd);
539       admin.createTable(htd, SPLIT_KEYS);
540       ZooKeeperWatcher zooKeeperWatcher = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
541       ZKAssign.blockUntilNoRIT(zooKeeperWatcher);
542       table = new HTable(TEST_UTIL.getConfiguration(), tableName);
543       Put p;
544       int numberOfPuts;
545       for (numberOfPuts = 0; numberOfPuts < 6; numberOfPuts++) {
546         p = new Put(Bytes.toBytes(numberOfPuts));
547         p.add(Bytes.toBytes("col"), Bytes.toBytes("ql"), Bytes.toBytes("value" + numberOfPuts));
548         table.put(p);
549       }
550       m.getZooKeeperWatcher().close();
551       m.abort("Test recovery from zk session expired",
552         new KeeperException.SessionExpiredException());
553       assertFalse(m.isStopped());
554       cluster.getRegionServer(0).abort("Aborting");
555       // Without patch for HBASE-6046 this test case will always timeout
556       // with patch the test case should pass.
557       Scan scan = new Scan();
558       int numberOfRows = 0;
559       ResultScanner scanner = table.getScanner(scan);
560       Result[] result = scanner.next(1);
561       while (result != null && result.length > 0) {
562         numberOfRows++;
563         result = scanner.next(1);
564       }
565       assertEquals("Number of rows should be equal to number of puts.", numberOfPuts,
566         numberOfRows);
567     } finally {
568       if (table != null) table.close();
569       admin.close();
570     }
571   }
572 
573   static class MockLoadBalancer extends SimpleLoadBalancer {
574     static boolean retainAssignCalled = false;
575 
576     @Override
577     public Map<ServerName, List<HRegionInfo>> retainAssignment(
578         Map<HRegionInfo, ServerName> regions, List<ServerName> servers) {
579       retainAssignCalled = true;
580       return super.retainAssignment(regions, servers);
581     }
582   }
583 
584 }
585