1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.coprocessor;
20
21 import org.apache.commons.logging.Log;
22 import org.apache.commons.logging.LogFactory;
23
24 import org.apache.hadoop.conf.Configuration;
25 import org.apache.hadoop.hbase.*;
26 import org.apache.hadoop.hbase.client.HBaseAdmin;
27 import org.apache.hadoop.hbase.regionserver.HRegion;
28 import org.apache.hadoop.hbase.regionserver.TestServerCustomProtocol;
29 import org.apache.hadoop.hbase.util.ClassLoaderTestHelper;
30 import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
31 import org.apache.hadoop.hdfs.MiniDFSCluster;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.hbase.ServerLoad;
35 import org.apache.hadoop.hbase.RegionLoad;
36
37 import java.io.*;
38 import java.util.*;
39
40 import org.junit.*;
41 import org.junit.experimental.categories.Category;
42
43 import static org.junit.Assert.assertEquals;
44 import static org.junit.Assert.assertNotNull;
45 import static org.junit.Assert.assertTrue;
46 import static org.junit.Assert.assertFalse;
47
48
49
50
51 @Category(MediumTests.class)
52 public class TestClassLoading {
53 private static final Log LOG = LogFactory.getLog(TestClassLoading.class);
54 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
55
56 private static MiniDFSCluster cluster;
57
58 static final String tableName = "TestClassLoading";
59 static final String cpName1 = "TestCP1";
60 static final String cpName2 = "TestCP2";
61 static final String cpName3 = "TestCP3";
62 static final String cpName4 = "TestCP4";
63 static final String cpName5 = "TestCP5";
64 static final String cpName6 = "TestCP6";
65
66 private static Class<?> regionCoprocessor1 = ColumnAggregationEndpoint.class;
67
68 private static Class<?> regionCoprocessor2 = TestServerCustomProtocol.PingHandler.class;
69 private static Class<?> regionServerCoprocessor = SampleRegionWALObserver.class;
70 private static Class<?> masterCoprocessor = BaseMasterObserver.class;
71
72 private static final String[] regionServerSystemCoprocessors =
73 new String[]{
74 regionServerCoprocessor.getSimpleName()
75 };
76
77 @BeforeClass
78 public static void setUpBeforeClass() throws Exception {
79 Configuration conf = TEST_UTIL.getConfiguration();
80
81
82
83 conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
84 regionCoprocessor1.getName());
85
86
87
88
89 conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
90 regionCoprocessor2.getName());
91
92 conf.setStrings(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY,
93 regionServerCoprocessor.getName());
94 conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
95 masterCoprocessor.getName());
96 TEST_UTIL.startMiniCluster(1);
97 cluster = TEST_UTIL.getDFSCluster();
98 }
99
100 @AfterClass
101 public static void tearDownAfterClass() throws Exception {
102 TEST_UTIL.shutdownMiniCluster();
103 }
104
105 static File buildCoprocessorJar(String className) throws Exception {
106 String code = "import org.apache.hadoop.hbase.coprocessor.*;" +
107 "public class " + className + " extends BaseRegionObserver {}";
108 return ClassLoaderTestHelper.buildJar(
109 TEST_UTIL.getDataTestDir().toString(), className, code);
110 }
111
112 @Test
113
114 public void testClassLoadingFromHDFS() throws Exception {
115 FileSystem fs = cluster.getFileSystem();
116
117 File jarFile1 = buildCoprocessorJar(cpName1);
118 File jarFile2 = buildCoprocessorJar(cpName2);
119
120
121 fs.copyFromLocalFile(new Path(jarFile1.getPath()),
122 new Path(fs.getUri().toString() + Path.SEPARATOR));
123 String jarFileOnHDFS1 = fs.getUri().toString() + Path.SEPARATOR +
124 jarFile1.getName();
125 Path pathOnHDFS1 = new Path(jarFileOnHDFS1);
126 assertTrue("Copy jar file to HDFS failed.",
127 fs.exists(pathOnHDFS1));
128 LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS1);
129
130 fs.copyFromLocalFile(new Path(jarFile2.getPath()),
131 new Path(fs.getUri().toString() + Path.SEPARATOR));
132 String jarFileOnHDFS2 = fs.getUri().toString() + Path.SEPARATOR +
133 jarFile2.getName();
134 Path pathOnHDFS2 = new Path(jarFileOnHDFS2);
135 assertTrue("Copy jar file to HDFS failed.",
136 fs.exists(pathOnHDFS2));
137 LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS2);
138
139
140 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
141 htd.addFamily(new HColumnDescriptor("test"));
142
143 htd.setValue("COPROCESSOR$1", jarFileOnHDFS1.toString() + "|" + cpName1 +
144 "|" + Coprocessor.PRIORITY_USER);
145
146 htd.setValue("COPROCESSOR$2", jarFileOnHDFS2.toString() + "|" + cpName2 +
147 "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
148 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
149 if (admin.tableExists(tableName)) {
150 if (admin.isTableEnabled(tableName)) {
151 admin.disableTable(tableName);
152 }
153 admin.deleteTable(tableName);
154 }
155 CoprocessorClassLoader.clearCache();
156 byte[] startKey = {10, 63};
157 byte[] endKey = {12, 43};
158 admin.createTable(htd, startKey, endKey, 4);
159 waitForTable(htd.getTableName());
160
161
162 boolean foundTableRegion=false;
163 boolean found1 = true, found2 = true, found2_k1 = true, found2_k2 = true, found2_k3 = true;
164 Map<HRegion, Set<ClassLoader>> regionsActiveClassLoaders =
165 new HashMap<HRegion, Set<ClassLoader>>();
166 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
167 for (HRegion region:
168 hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
169 if (region.getRegionNameAsString().startsWith(tableName)) {
170 foundTableRegion = true;
171 CoprocessorEnvironment env;
172 env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
173 found1 = found1 && (env != null);
174 env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
175 found2 = found2 && (env != null);
176 if (env != null) {
177 Configuration conf = env.getConfiguration();
178 found2_k1 = found2_k1 && (conf.get("k1") != null);
179 found2_k2 = found2_k2 && (conf.get("k2") != null);
180 found2_k3 = found2_k3 && (conf.get("k3") != null);
181 } else {
182 found2_k1 = found2_k2 = found2_k3 = false;
183 }
184 regionsActiveClassLoaders
185 .put(region, ((CoprocessorHost) region.getCoprocessorHost()).getExternalClassLoaders());
186 }
187 }
188
189 assertTrue("No region was found for table " + tableName, foundTableRegion);
190 assertTrue("Class " + cpName1 + " was missing on a region", found1);
191 assertTrue("Class " + cpName2 + " was missing on a region", found2);
192 assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
193 assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
194 assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
195
196 assertNotNull(jarFileOnHDFS1 + " was not cached",
197 CoprocessorClassLoader.getIfCached(pathOnHDFS1));
198 assertNotNull(jarFileOnHDFS2 + " was not cached",
199 CoprocessorClassLoader.getIfCached(pathOnHDFS2));
200
201 assertEquals("The number of cached classloaders should be equal to the number" +
202 " of external jar files",
203 2, CoprocessorClassLoader.getAllCached().size());
204
205 Set<ClassLoader> externalClassLoaders = new HashSet<ClassLoader>(
206 CoprocessorClassLoader.getAllCached());
207 for (Map.Entry<HRegion, Set<ClassLoader>> regionCP : regionsActiveClassLoaders.entrySet()) {
208 assertTrue("Some CP classloaders for region " + regionCP.getKey() + " are not cached."
209 + " ClassLoader Cache:" + externalClassLoaders
210 + " Region ClassLoaders:" + regionCP.getValue(),
211 externalClassLoaders.containsAll(regionCP.getValue()));
212 }
213 }
214
215 private String getLocalPath(File file) {
216 return new Path(file.toURI()).toString();
217 }
218
219 @Test
220
221 public void testClassLoadingFromLocalFS() throws Exception {
222 File jarFile = buildCoprocessorJar(cpName3);
223
224
225 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(cpName3));
226 htd.addFamily(new HColumnDescriptor("test"));
227 htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName3 + "|" +
228 Coprocessor.PRIORITY_USER);
229 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
230 admin.createTable(htd);
231 waitForTable(htd.getTableName());
232
233
234 boolean found = false;
235 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
236 for (HRegion region:
237 hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
238 if (region.getRegionNameAsString().startsWith(cpName3)) {
239 found = (region.getCoprocessorHost().findCoprocessor(cpName3) != null);
240 }
241 }
242 assertTrue("Class " + cpName3 + " was missing on a region", found);
243 }
244
245 @Test
246
247 public void testPrivateClassLoader() throws Exception {
248 File jarFile = buildCoprocessorJar(cpName4);
249
250
251 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(cpName4));
252 htd.addFamily(new HColumnDescriptor("test"));
253 htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName4 + "|" +
254 Coprocessor.PRIORITY_USER);
255 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
256 admin.createTable(htd);
257 waitForTable(htd.getTableName());
258
259
260 boolean found = false;
261 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
262 for (HRegion region:
263 hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
264 if (region.getRegionNameAsString().startsWith(cpName4)) {
265 Coprocessor cp = region.getCoprocessorHost().findCoprocessor(cpName4);
266 if (cp != null) {
267 found = true;
268 assertEquals("Class " + cpName4 + " was not loaded by CoprocessorClassLoader",
269 cp.getClass().getClassLoader().getClass(), CoprocessorClassLoader.class);
270 }
271 }
272 }
273 assertTrue("Class " + cpName4 + " was missing on a region", found);
274 }
275
276 @Test
277
278
279 public void testHBase3810() throws Exception {
280
281
282 File jarFile1 = buildCoprocessorJar(cpName1);
283 File jarFile2 = buildCoprocessorJar(cpName2);
284 File jarFile5 = buildCoprocessorJar(cpName5);
285 File jarFile6 = buildCoprocessorJar(cpName6);
286
287 String cpKey1 = "COPROCESSOR$1";
288 String cpKey2 = " Coprocessor$2 ";
289 String cpKey3 = " coprocessor$03 ";
290
291 String cpValue1 = getLocalPath(jarFile1) + "|" + cpName1 + "|" +
292 Coprocessor.PRIORITY_USER;
293 String cpValue2 = getLocalPath(jarFile2) + " | " + cpName2 + " | ";
294
295 String cpValue3 =
296 " | org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver | | k=v ";
297
298
299 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
300 htd.addFamily(new HColumnDescriptor("test"));
301
302
303 htd.setValue(cpKey1, cpValue1);
304 htd.setValue(cpKey2, cpValue2);
305 htd.setValue(cpKey3, cpValue3);
306
307
308 htd.addCoprocessor(cpName5, new Path(getLocalPath(jarFile5)),
309 Coprocessor.PRIORITY_USER, null);
310 Map<String, String> kvs = new HashMap<String, String>();
311 kvs.put("k1", "v1");
312 kvs.put("k2", "v2");
313 kvs.put("k3", "v3");
314 htd.addCoprocessor(cpName6, new Path(getLocalPath(jarFile6)),
315 Coprocessor.PRIORITY_USER, kvs);
316
317 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
318 if (admin.tableExists(tableName)) {
319 if (admin.isTableEnabled(tableName)) {
320 admin.disableTable(tableName);
321 }
322 admin.deleteTable(tableName);
323 }
324 admin.createTable(htd);
325 waitForTable(htd.getTableName());
326
327
328 boolean found_2 = false, found_1 = false, found_3 = false,
329 found_5 = false, found_6 = false;
330 boolean found6_k1 = false, found6_k2 = false, found6_k3 = false,
331 found6_k4 = false;
332
333 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
334 for (HRegion region:
335 hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
336 if (region.getRegionNameAsString().startsWith(tableName)) {
337 found_1 = found_1 ||
338 (region.getCoprocessorHost().findCoprocessor(cpName1) != null);
339 found_2 = found_2 ||
340 (region.getCoprocessorHost().findCoprocessor(cpName2) != null);
341 found_3 = found_3 ||
342 (region.getCoprocessorHost().findCoprocessor("SimpleRegionObserver")
343 != null);
344 found_5 = found_5 ||
345 (region.getCoprocessorHost().findCoprocessor(cpName5) != null);
346
347 CoprocessorEnvironment env =
348 region.getCoprocessorHost().findCoprocessorEnvironment(cpName6);
349 if (env != null) {
350 found_6 = true;
351 Configuration conf = env.getConfiguration();
352 found6_k1 = conf.get("k1") != null;
353 found6_k2 = conf.get("k2") != null;
354 found6_k3 = conf.get("k3") != null;
355 }
356 }
357 }
358
359 assertTrue("Class " + cpName1 + " was missing on a region", found_1);
360 assertTrue("Class " + cpName2 + " was missing on a region", found_2);
361 assertTrue("Class SimpleRegionObserver was missing on a region", found_3);
362 assertTrue("Class " + cpName5 + " was missing on a region", found_5);
363 assertTrue("Class " + cpName6 + " was missing on a region", found_6);
364
365 assertTrue("Configuration key 'k1' was missing on a region", found6_k1);
366 assertTrue("Configuration key 'k2' was missing on a region", found6_k2);
367 assertTrue("Configuration key 'k3' was missing on a region", found6_k3);
368 assertFalse("Configuration key 'k4' wasn't configured", found6_k4);
369 }
370
371 @Test
372 public void testClassLoadingFromLibDirInJar() throws Exception {
373 loadingClassFromLibDirInJar("/lib/");
374 }
375
376 @Test
377 public void testClassLoadingFromRelativeLibDirInJar() throws Exception {
378 loadingClassFromLibDirInJar("lib/");
379 }
380
381 void loadingClassFromLibDirInJar(String libPrefix) throws Exception {
382 FileSystem fs = cluster.getFileSystem();
383
384 File innerJarFile1 = buildCoprocessorJar(cpName1);
385 File innerJarFile2 = buildCoprocessorJar(cpName2);
386 File outerJarFile = new File(TEST_UTIL.getDataTestDir().toString(), "outer.jar");
387
388 ClassLoaderTestHelper.addJarFilesToJar(
389 outerJarFile, libPrefix, innerJarFile1, innerJarFile2);
390
391
392 fs.copyFromLocalFile(new Path(outerJarFile.getPath()),
393 new Path(fs.getUri().toString() + Path.SEPARATOR));
394 String jarFileOnHDFS = fs.getUri().toString() + Path.SEPARATOR +
395 outerJarFile.getName();
396 assertTrue("Copy jar file to HDFS failed.",
397 fs.exists(new Path(jarFileOnHDFS)));
398 LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS);
399
400
401 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
402 htd.addFamily(new HColumnDescriptor("test"));
403
404 htd.setValue("COPROCESSOR$1", jarFileOnHDFS.toString() + "|" + cpName1 +
405 "|" + Coprocessor.PRIORITY_USER);
406
407 htd.setValue("COPROCESSOR$2", jarFileOnHDFS.toString() + "|" + cpName2 +
408 "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
409 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
410 if (admin.tableExists(tableName)) {
411 if (admin.isTableEnabled(tableName)) {
412 admin.disableTable(tableName);
413 }
414 admin.deleteTable(tableName);
415 }
416 admin.createTable(htd);
417 waitForTable(htd.getTableName());
418
419
420 boolean found1 = false, found2 = false, found2_k1 = false,
421 found2_k2 = false, found2_k3 = false;
422 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
423 for (HRegion region:
424 hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
425 if (region.getRegionNameAsString().startsWith(tableName)) {
426 CoprocessorEnvironment env;
427 env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
428 if (env != null) {
429 found1 = true;
430 }
431 env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
432 if (env != null) {
433 found2 = true;
434 Configuration conf = env.getConfiguration();
435 found2_k1 = conf.get("k1") != null;
436 found2_k2 = conf.get("k2") != null;
437 found2_k3 = conf.get("k3") != null;
438 }
439 }
440 }
441 assertTrue("Class " + cpName1 + " was missing on a region", found1);
442 assertTrue("Class " + cpName2 + " was missing on a region", found2);
443 assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
444 assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
445 assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
446 }
447
448 @Test
449 public void testRegionServerCoprocessorsReported() throws Exception {
450
451
452
453 assertAllRegionServers(regionServerSystemCoprocessors,null);
454 }
455
456
457
458
459
460
461
462
463
464
465 Map<ServerName, ServerLoad> serversForTable(String tableName) {
466 Map<ServerName, ServerLoad> serverLoadHashMap =
467 new HashMap<ServerName, ServerLoad>();
468 for(Map.Entry<ServerName,ServerLoad> server:
469 TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager().
470 getOnlineServers().entrySet()) {
471 for( Map.Entry<byte[], RegionLoad> region:
472 server.getValue().getRegionsLoad().entrySet()) {
473 if (region.getValue().getNameAsString().equals(tableName)) {
474
475 serverLoadHashMap.put(server.getKey(),server.getValue());
476
477 break;
478 }
479 }
480 }
481 return serverLoadHashMap;
482 }
483
484 void assertAllRegionServers(String[] expectedCoprocessors, String tableName)
485 throws InterruptedException {
486 Map<ServerName, ServerLoad> servers;
487 String[] actualCoprocessors = null;
488 boolean success = false;
489 for(int i = 0; i < 5; i++) {
490 if (tableName == null) {
491
492 servers =
493 TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager().
494 getOnlineServers();
495 } else {
496 servers = serversForTable(tableName);
497 }
498 boolean any_failed = false;
499 for(Map.Entry<ServerName,ServerLoad> server: servers.entrySet()) {
500 actualCoprocessors = server.getValue().getRsCoprocessors();
501 if (!Arrays.equals(actualCoprocessors, expectedCoprocessors)) {
502 LOG.debug("failed comparison: actual: " +
503 Arrays.toString(actualCoprocessors) +
504 " ; expected: " + Arrays.toString(expectedCoprocessors));
505 any_failed = true;
506 break;
507 }
508 }
509 if (any_failed == false) {
510 success = true;
511 break;
512 }
513 LOG.debug("retrying after failed comparison: " + i);
514 Thread.sleep(1000);
515 }
516 assertTrue(success);
517 }
518
519 @Test
520 public void testMasterCoprocessorsReported() {
521
522
523
524 final String loadedMasterCoprocessorsVerify =
525 "[" + masterCoprocessor.getSimpleName() + "]";
526 String loadedMasterCoprocessors =
527 java.util.Arrays.toString(
528 TEST_UTIL.getHBaseCluster().getMaster().getCoprocessors());
529 assertEquals(loadedMasterCoprocessorsVerify, loadedMasterCoprocessors);
530 }
531
532 private void waitForTable(TableName name) throws InterruptedException, IOException {
533
534 TEST_UTIL.waitTableEnabled(name.getName());
535
536 Thread.sleep(1000);
537 }
538
539 }
540