View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.coprocessor;
20  
21  import org.apache.commons.logging.Log;
22  import org.apache.commons.logging.LogFactory;
23  
24  import org.apache.hadoop.conf.Configuration;
25  import org.apache.hadoop.hbase.*;
26  import org.apache.hadoop.hbase.client.HBaseAdmin;
27  import org.apache.hadoop.hbase.regionserver.HRegion;
28  import org.apache.hadoop.hbase.regionserver.TestServerCustomProtocol;
29  import org.apache.hadoop.hbase.util.ClassLoaderTestHelper;
30  import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
31  import org.apache.hadoop.hdfs.MiniDFSCluster;
32  import org.apache.hadoop.fs.FileSystem;
33  import org.apache.hadoop.fs.Path;
34  import org.apache.hadoop.hbase.ServerLoad;
35  import org.apache.hadoop.hbase.RegionLoad;
36  
37  import java.io.*;
38  import java.util.*;
39  
40  import org.junit.*;
41  import org.junit.experimental.categories.Category;
42  
43  import static org.junit.Assert.assertEquals;
44  import static org.junit.Assert.assertNotNull;
45  import static org.junit.Assert.assertTrue;
46  import static org.junit.Assert.assertFalse;
47  
48  /**
49   * Test coprocessors class loading.
50   */
51  @Category(MediumTests.class)
52  public class TestClassLoading {
53    private static final Log LOG = LogFactory.getLog(TestClassLoading.class);
54    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
55  
56    private static MiniDFSCluster cluster;
57  
58    static final String tableName = "TestClassLoading";
59    static final String cpName1 = "TestCP1";
60    static final String cpName2 = "TestCP2";
61    static final String cpName3 = "TestCP3";
62    static final String cpName4 = "TestCP4";
63    static final String cpName5 = "TestCP5";
64    static final String cpName6 = "TestCP6";
65  
66    private static Class<?> regionCoprocessor1 = ColumnAggregationEndpoint.class;
67    // TOOD: Fix the import of this handler.  It is coming in from a package that is far away.
68    private static Class<?> regionCoprocessor2 = TestServerCustomProtocol.PingHandler.class;
69    private static Class<?> regionServerCoprocessor = SampleRegionWALObserver.class;
70    private static Class<?> masterCoprocessor = BaseMasterObserver.class;
71  
72    private static final String[] regionServerSystemCoprocessors =
73        new String[]{
74        regionServerCoprocessor.getSimpleName()
75    };
76  
77    @BeforeClass
78    public static void setUpBeforeClass() throws Exception {
79      Configuration conf = TEST_UTIL.getConfiguration();
80  
81      // regionCoprocessor1 will be loaded on all regionservers, since it is
82      // loaded for any tables (user or meta).
83      conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
84          regionCoprocessor1.getName());
85  
86      // regionCoprocessor2 will be loaded only on regionservers that serve a
87      // user table region. Therefore, if there are no user tables loaded,
88      // this coprocessor will not be loaded on any regionserver.
89      conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
90          regionCoprocessor2.getName());
91  
92      conf.setStrings(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY,
93          regionServerCoprocessor.getName());
94      conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
95          masterCoprocessor.getName());
96      TEST_UTIL.startMiniCluster(1);
97      cluster = TEST_UTIL.getDFSCluster();
98    }
99  
100   @AfterClass
101   public static void tearDownAfterClass() throws Exception {
102     TEST_UTIL.shutdownMiniCluster();
103   }
104 
105   static File buildCoprocessorJar(String className) throws Exception {
106     String code = "import org.apache.hadoop.hbase.coprocessor.*;" +
107       "public class " + className + " extends BaseRegionObserver {}";
108     return ClassLoaderTestHelper.buildJar(
109       TEST_UTIL.getDataTestDir().toString(), className, code);
110   }
111 
112   @Test
113   // HBASE-3516: Test CP Class loading from HDFS
114   public void testClassLoadingFromHDFS() throws Exception {
115     FileSystem fs = cluster.getFileSystem();
116 
117     File jarFile1 = buildCoprocessorJar(cpName1);
118     File jarFile2 = buildCoprocessorJar(cpName2);
119 
120     // copy the jars into dfs
121     fs.copyFromLocalFile(new Path(jarFile1.getPath()),
122       new Path(fs.getUri().toString() + Path.SEPARATOR));
123     String jarFileOnHDFS1 = fs.getUri().toString() + Path.SEPARATOR +
124       jarFile1.getName();
125     Path pathOnHDFS1 = new Path(jarFileOnHDFS1);
126     assertTrue("Copy jar file to HDFS failed.",
127       fs.exists(pathOnHDFS1));
128     LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS1);
129 
130     fs.copyFromLocalFile(new Path(jarFile2.getPath()),
131         new Path(fs.getUri().toString() + Path.SEPARATOR));
132     String jarFileOnHDFS2 = fs.getUri().toString() + Path.SEPARATOR +
133       jarFile2.getName();
134     Path pathOnHDFS2 = new Path(jarFileOnHDFS2);
135     assertTrue("Copy jar file to HDFS failed.",
136       fs.exists(pathOnHDFS2));
137     LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS2);
138 
139     // create a table that references the coprocessors
140     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
141     htd.addFamily(new HColumnDescriptor("test"));
142       // without configuration values
143     htd.setValue("COPROCESSOR$1", jarFileOnHDFS1.toString() + "|" + cpName1 +
144       "|" + Coprocessor.PRIORITY_USER);
145       // with configuration values
146     htd.setValue("COPROCESSOR$2", jarFileOnHDFS2.toString() + "|" + cpName2 +
147       "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
148     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
149     if (admin.tableExists(tableName)) {
150       if (admin.isTableEnabled(tableName)) {
151         admin.disableTable(tableName);
152       }
153       admin.deleteTable(tableName);
154     }
155     CoprocessorClassLoader.clearCache();
156     byte[] startKey = {10, 63};
157     byte[] endKey = {12, 43};
158     admin.createTable(htd, startKey, endKey, 4);
159     waitForTable(htd.getTableName());
160 
161     // verify that the coprocessors were loaded
162     boolean foundTableRegion=false;
163     boolean found1 = true, found2 = true, found2_k1 = true, found2_k2 = true, found2_k3 = true;
164     Map<HRegion, Set<ClassLoader>> regionsActiveClassLoaders =
165         new HashMap<HRegion, Set<ClassLoader>>();
166     MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
167     for (HRegion region:
168         hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
169       if (region.getRegionNameAsString().startsWith(tableName)) {
170         foundTableRegion = true;
171         CoprocessorEnvironment env;
172         env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
173         found1 = found1 && (env != null);
174         env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
175         found2 = found2 && (env != null);
176         if (env != null) {
177           Configuration conf = env.getConfiguration();
178           found2_k1 = found2_k1 && (conf.get("k1") != null);
179           found2_k2 = found2_k2 && (conf.get("k2") != null);
180           found2_k3 = found2_k3 && (conf.get("k3") != null);
181         } else {
182           found2_k1 = found2_k2 = found2_k3 = false;
183         }
184         regionsActiveClassLoaders
185             .put(region, ((CoprocessorHost) region.getCoprocessorHost()).getExternalClassLoaders());
186       }
187     }
188 
189     assertTrue("No region was found for table " + tableName, foundTableRegion);
190     assertTrue("Class " + cpName1 + " was missing on a region", found1);
191     assertTrue("Class " + cpName2 + " was missing on a region", found2);
192     assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
193     assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
194     assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
195     // check if CP classloaders are cached
196     assertNotNull(jarFileOnHDFS1 + " was not cached",
197       CoprocessorClassLoader.getIfCached(pathOnHDFS1));
198     assertNotNull(jarFileOnHDFS2 + " was not cached",
199       CoprocessorClassLoader.getIfCached(pathOnHDFS2));
200     //two external jar used, should be one classloader per jar
201     assertEquals("The number of cached classloaders should be equal to the number" +
202       " of external jar files",
203       2, CoprocessorClassLoader.getAllCached().size());
204     //check if region active classloaders are shared across all RS regions
205     Set<ClassLoader> externalClassLoaders = new HashSet<ClassLoader>(
206       CoprocessorClassLoader.getAllCached());
207     for (Map.Entry<HRegion, Set<ClassLoader>> regionCP : regionsActiveClassLoaders.entrySet()) {
208       assertTrue("Some CP classloaders for region " + regionCP.getKey() + " are not cached."
209         + " ClassLoader Cache:" + externalClassLoaders
210         + " Region ClassLoaders:" + regionCP.getValue(),
211         externalClassLoaders.containsAll(regionCP.getValue()));
212     }
213   }
214 
215   private String getLocalPath(File file) {
216     return new Path(file.toURI()).toString();
217   }
218 
219   @Test
220   // HBASE-3516: Test CP Class loading from local file system
221   public void testClassLoadingFromLocalFS() throws Exception {
222     File jarFile = buildCoprocessorJar(cpName3);
223 
224     // create a table that references the jar
225     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(cpName3));
226     htd.addFamily(new HColumnDescriptor("test"));
227     htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName3 + "|" +
228       Coprocessor.PRIORITY_USER);
229     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
230     admin.createTable(htd);
231     waitForTable(htd.getTableName());
232 
233     // verify that the coprocessor was loaded
234     boolean found = false;
235     MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
236     for (HRegion region:
237         hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
238       if (region.getRegionNameAsString().startsWith(cpName3)) {
239         found = (region.getCoprocessorHost().findCoprocessor(cpName3) != null);
240       }
241     }
242     assertTrue("Class " + cpName3 + " was missing on a region", found);
243   }
244 
245   @Test
246   // HBASE-6308: Test CP classloader is the CoprocessorClassLoader
247   public void testPrivateClassLoader() throws Exception {
248     File jarFile = buildCoprocessorJar(cpName4);
249 
250     // create a table that references the jar
251     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(cpName4));
252     htd.addFamily(new HColumnDescriptor("test"));
253     htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName4 + "|" +
254       Coprocessor.PRIORITY_USER);
255     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
256     admin.createTable(htd);
257     waitForTable(htd.getTableName());
258 
259     // verify that the coprocessor was loaded correctly
260     boolean found = false;
261     MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
262     for (HRegion region:
263         hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
264       if (region.getRegionNameAsString().startsWith(cpName4)) {
265         Coprocessor cp = region.getCoprocessorHost().findCoprocessor(cpName4);
266         if (cp != null) {
267           found = true;
268           assertEquals("Class " + cpName4 + " was not loaded by CoprocessorClassLoader",
269             cp.getClass().getClassLoader().getClass(), CoprocessorClassLoader.class);
270         }
271       }
272     }
273     assertTrue("Class " + cpName4 + " was missing on a region", found);
274   }
275 
276   @Test
277   // HBase-3810: Registering a Coprocessor at HTableDescriptor should be
278   // less strict
279   public void testHBase3810() throws Exception {
280     // allowed value pattern: [path] | class name | [priority] | [key values]
281 
282     File jarFile1 = buildCoprocessorJar(cpName1);
283     File jarFile2 = buildCoprocessorJar(cpName2);
284     File jarFile5 = buildCoprocessorJar(cpName5);
285     File jarFile6 = buildCoprocessorJar(cpName6);
286 
287     String cpKey1 = "COPROCESSOR$1";
288     String cpKey2 = " Coprocessor$2 ";
289     String cpKey3 = " coprocessor$03 ";
290 
291     String cpValue1 = getLocalPath(jarFile1) + "|" + cpName1 + "|" +
292         Coprocessor.PRIORITY_USER;
293     String cpValue2 = getLocalPath(jarFile2) + " | " + cpName2 + " | ";
294     // load from default class loader
295     String cpValue3 =
296         " | org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver | | k=v ";
297 
298     // create a table that references the jar
299     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
300     htd.addFamily(new HColumnDescriptor("test"));
301 
302     // add 3 coprocessors by setting htd attributes directly.
303     htd.setValue(cpKey1, cpValue1);
304     htd.setValue(cpKey2, cpValue2);
305     htd.setValue(cpKey3, cpValue3);
306 
307     // add 2 coprocessor by using new htd.addCoprocessor() api
308     htd.addCoprocessor(cpName5, new Path(getLocalPath(jarFile5)),
309         Coprocessor.PRIORITY_USER, null);
310     Map<String, String> kvs = new HashMap<String, String>();
311     kvs.put("k1", "v1");
312     kvs.put("k2", "v2");
313     kvs.put("k3", "v3");
314     htd.addCoprocessor(cpName6, new Path(getLocalPath(jarFile6)),
315         Coprocessor.PRIORITY_USER, kvs);
316 
317     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
318     if (admin.tableExists(tableName)) {
319       if (admin.isTableEnabled(tableName)) {
320         admin.disableTable(tableName);
321       }
322       admin.deleteTable(tableName);
323     }
324     admin.createTable(htd);
325     waitForTable(htd.getTableName());
326 
327     // verify that the coprocessor was loaded
328     boolean found_2 = false, found_1 = false, found_3 = false,
329         found_5 = false, found_6 = false;
330     boolean found6_k1 = false, found6_k2 = false, found6_k3 = false,
331         found6_k4 = false;
332 
333     MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
334     for (HRegion region:
335         hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
336       if (region.getRegionNameAsString().startsWith(tableName)) {
337         found_1 = found_1 ||
338             (region.getCoprocessorHost().findCoprocessor(cpName1) != null);
339         found_2 = found_2 ||
340             (region.getCoprocessorHost().findCoprocessor(cpName2) != null);
341         found_3 = found_3 ||
342             (region.getCoprocessorHost().findCoprocessor("SimpleRegionObserver")
343                 != null);
344         found_5 = found_5 ||
345             (region.getCoprocessorHost().findCoprocessor(cpName5) != null);
346 
347         CoprocessorEnvironment env =
348             region.getCoprocessorHost().findCoprocessorEnvironment(cpName6);
349         if (env != null) {
350           found_6 = true;
351           Configuration conf = env.getConfiguration();
352           found6_k1 = conf.get("k1") != null;
353           found6_k2 = conf.get("k2") != null;
354           found6_k3 = conf.get("k3") != null;
355         }
356       }
357     }
358 
359     assertTrue("Class " + cpName1 + " was missing on a region", found_1);
360     assertTrue("Class " + cpName2 + " was missing on a region", found_2);
361     assertTrue("Class SimpleRegionObserver was missing on a region", found_3);
362     assertTrue("Class " + cpName5 + " was missing on a region", found_5);
363     assertTrue("Class " + cpName6 + " was missing on a region", found_6);
364 
365     assertTrue("Configuration key 'k1' was missing on a region", found6_k1);
366     assertTrue("Configuration key 'k2' was missing on a region", found6_k2);
367     assertTrue("Configuration key 'k3' was missing on a region", found6_k3);
368     assertFalse("Configuration key 'k4' wasn't configured", found6_k4);
369   }
370 
371   @Test
372   public void testClassLoadingFromLibDirInJar() throws Exception {
373     loadingClassFromLibDirInJar("/lib/");
374   }
375 
376   @Test
377   public void testClassLoadingFromRelativeLibDirInJar() throws Exception {
378     loadingClassFromLibDirInJar("lib/");
379   }
380 
381   void loadingClassFromLibDirInJar(String libPrefix) throws Exception {
382     FileSystem fs = cluster.getFileSystem();
383 
384     File innerJarFile1 = buildCoprocessorJar(cpName1);
385     File innerJarFile2 = buildCoprocessorJar(cpName2);
386     File outerJarFile = new File(TEST_UTIL.getDataTestDir().toString(), "outer.jar");
387 
388     ClassLoaderTestHelper.addJarFilesToJar(
389       outerJarFile, libPrefix, innerJarFile1, innerJarFile2);
390 
391     // copy the jars into dfs
392     fs.copyFromLocalFile(new Path(outerJarFile.getPath()),
393       new Path(fs.getUri().toString() + Path.SEPARATOR));
394     String jarFileOnHDFS = fs.getUri().toString() + Path.SEPARATOR +
395       outerJarFile.getName();
396     assertTrue("Copy jar file to HDFS failed.",
397       fs.exists(new Path(jarFileOnHDFS)));
398     LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS);
399 
400     // create a table that references the coprocessors
401     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
402     htd.addFamily(new HColumnDescriptor("test"));
403       // without configuration values
404     htd.setValue("COPROCESSOR$1", jarFileOnHDFS.toString() + "|" + cpName1 +
405       "|" + Coprocessor.PRIORITY_USER);
406       // with configuration values
407     htd.setValue("COPROCESSOR$2", jarFileOnHDFS.toString() + "|" + cpName2 +
408       "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
409     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
410     if (admin.tableExists(tableName)) {
411       if (admin.isTableEnabled(tableName)) {
412         admin.disableTable(tableName);
413       }
414       admin.deleteTable(tableName);
415     }
416     admin.createTable(htd);
417     waitForTable(htd.getTableName());
418 
419     // verify that the coprocessors were loaded
420     boolean found1 = false, found2 = false, found2_k1 = false,
421         found2_k2 = false, found2_k3 = false;
422     MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
423     for (HRegion region:
424         hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
425       if (region.getRegionNameAsString().startsWith(tableName)) {
426         CoprocessorEnvironment env;
427         env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
428         if (env != null) {
429           found1 = true;
430         }
431         env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
432         if (env != null) {
433           found2 = true;
434           Configuration conf = env.getConfiguration();
435           found2_k1 = conf.get("k1") != null;
436           found2_k2 = conf.get("k2") != null;
437           found2_k3 = conf.get("k3") != null;
438         }
439       }
440     }
441     assertTrue("Class " + cpName1 + " was missing on a region", found1);
442     assertTrue("Class " + cpName2 + " was missing on a region", found2);
443     assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
444     assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
445     assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
446   }
447 
448   @Test
449   public void testRegionServerCoprocessorsReported() throws Exception {
450     // This was a test for HBASE-4070.
451     // We are removing coprocessors from region load in HBASE-5258.
452     // Therefore, this test now only checks system coprocessors.
453     assertAllRegionServers(regionServerSystemCoprocessors,null);
454   }
455 
456   /**
457    * return the subset of all regionservers
458    * (actually returns set of ServerLoads)
459    * which host some region in a given table.
460    * used by assertAllRegionServers() below to
461    * test reporting of loaded coprocessors.
462    * @param tableName : given table.
463    * @return subset of all servers.
464    */
465   Map<ServerName, ServerLoad> serversForTable(String tableName) {
466     Map<ServerName, ServerLoad> serverLoadHashMap =
467         new HashMap<ServerName, ServerLoad>();
468     for(Map.Entry<ServerName,ServerLoad> server:
469         TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager().
470             getOnlineServers().entrySet()) {
471       for( Map.Entry<byte[], RegionLoad> region:
472           server.getValue().getRegionsLoad().entrySet()) {
473         if (region.getValue().getNameAsString().equals(tableName)) {
474           // this server hosts a region of tableName: add this server..
475           serverLoadHashMap.put(server.getKey(),server.getValue());
476           // .. and skip the rest of the regions that it hosts.
477           break;
478         }
479       }
480     }
481     return serverLoadHashMap;
482   }
483 
484   void assertAllRegionServers(String[] expectedCoprocessors, String tableName)
485       throws InterruptedException {
486     Map<ServerName, ServerLoad> servers;
487     String[] actualCoprocessors = null;
488     boolean success = false;
489     for(int i = 0; i < 5; i++) {
490       if (tableName == null) {
491         //if no tableName specified, use all servers.
492         servers =
493             TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager().
494                 getOnlineServers();
495       } else {
496         servers = serversForTable(tableName);
497       }
498       boolean any_failed = false;
499       for(Map.Entry<ServerName,ServerLoad> server: servers.entrySet()) {
500         actualCoprocessors = server.getValue().getRsCoprocessors();
501         if (!Arrays.equals(actualCoprocessors, expectedCoprocessors)) {
502           LOG.debug("failed comparison: actual: " +
503               Arrays.toString(actualCoprocessors) +
504               " ; expected: " + Arrays.toString(expectedCoprocessors));
505           any_failed = true;
506           break;
507         }
508       }
509       if (any_failed == false) {
510         success = true;
511         break;
512       }
513       LOG.debug("retrying after failed comparison: " + i);
514       Thread.sleep(1000);
515     }
516     assertTrue(success);
517   }
518 
519   @Test
520   public void testMasterCoprocessorsReported() {
521     // HBASE 4070: Improve region server metrics to report loaded coprocessors
522     // to master: verify that the master is reporting the correct set of
523     // loaded coprocessors.
524     final String loadedMasterCoprocessorsVerify =
525         "[" + masterCoprocessor.getSimpleName() + "]";
526     String loadedMasterCoprocessors =
527         java.util.Arrays.toString(
528             TEST_UTIL.getHBaseCluster().getMaster().getCoprocessors());
529     assertEquals(loadedMasterCoprocessorsVerify, loadedMasterCoprocessors);
530   }
531 
532   private void waitForTable(TableName name) throws InterruptedException, IOException {
533     // First wait until all regions are online
534     TEST_UTIL.waitTableEnabled(name.getName());
535     // Now wait a bit longer for the coprocessor hosts to load the CPs
536     Thread.sleep(1000);
537   }
538 
539 }
540