View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.coprocessor;
21  
22  import static org.junit.Assert.assertArrayEquals;
23  import static org.junit.Assert.assertFalse;
24  import static org.junit.Assert.assertNotNull;
25  import static org.junit.Assert.assertTrue;
26  
27  import java.io.IOException;
28  import java.lang.reflect.Method;
29  import java.util.ArrayList;
30  import java.util.List;
31  
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.fs.FileSystem;
36  import org.apache.hadoop.fs.Path;
37  import org.apache.hadoop.hbase.Cell;
38  import org.apache.hadoop.hbase.CellUtil;
39  import org.apache.hadoop.hbase.Coprocessor;
40  import org.apache.hadoop.hbase.HBaseTestingUtility;
41  import org.apache.hadoop.hbase.HColumnDescriptor;
42  import org.apache.hadoop.hbase.HRegionInfo;
43  import org.apache.hadoop.hbase.HTableDescriptor;
44  import org.apache.hadoop.hbase.KeyValue;
45  import org.apache.hadoop.hbase.testclassification.MediumTests;
46  import org.apache.hadoop.hbase.MiniHBaseCluster;
47  import org.apache.hadoop.hbase.ServerName;
48  import org.apache.hadoop.hbase.TableName;
49  import org.apache.hadoop.hbase.client.Admin;
50  import org.apache.hadoop.hbase.client.Append;
51  import org.apache.hadoop.hbase.client.Delete;
52  import org.apache.hadoop.hbase.client.Durability;
53  import org.apache.hadoop.hbase.client.Get;
54  import org.apache.hadoop.hbase.client.HTable;
55  import org.apache.hadoop.hbase.client.Increment;
56  import org.apache.hadoop.hbase.client.Put;
57  import org.apache.hadoop.hbase.client.Result;
58  import org.apache.hadoop.hbase.client.ResultScanner;
59  import org.apache.hadoop.hbase.client.RowMutations;
60  import org.apache.hadoop.hbase.client.Scan;
61  import org.apache.hadoop.hbase.client.Table;
62  import org.apache.hadoop.hbase.filter.FilterAllFilter;
63  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
64  import org.apache.hadoop.hbase.io.hfile.HFile;
65  import org.apache.hadoop.hbase.io.hfile.HFileContext;
66  import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
67  import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
68  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
69  import org.apache.hadoop.hbase.regionserver.HRegion;
70  import org.apache.hadoop.hbase.regionserver.InternalScanner;
71  import org.apache.hadoop.hbase.regionserver.NoLimitScannerContext;
72  import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
73  import org.apache.hadoop.hbase.regionserver.ScanType;
74  import org.apache.hadoop.hbase.regionserver.ScannerContext;
75  import org.apache.hadoop.hbase.regionserver.Store;
76  import org.apache.hadoop.hbase.regionserver.StoreFile;
77  import org.apache.hadoop.hbase.util.Bytes;
78  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
79  import org.apache.hadoop.hbase.util.JVMClusterUtil;
80  import org.apache.hadoop.hbase.util.Threads;
81  import org.junit.AfterClass;
82  import org.junit.BeforeClass;
83  import org.junit.Ignore;
84  import org.junit.Test;
85  import org.junit.experimental.categories.Category;
86  
87  @Category(MediumTests.class)
88  public class TestRegionObserverInterface {
89    private static final Log LOG = LogFactory.getLog(TestRegionObserverInterface.class);
90  
91    public static final TableName TEST_TABLE = TableName.valueOf("TestTable");
92    public final static byte[] A = Bytes.toBytes("a");
93    public final static byte[] B = Bytes.toBytes("b");
94    public final static byte[] C = Bytes.toBytes("c");
95    public final static byte[] ROW = Bytes.toBytes("testrow");
96  
97    private static HBaseTestingUtility util = new HBaseTestingUtility();
98    private static MiniHBaseCluster cluster = null;
99  
100   @BeforeClass
101   public static void setupBeforeClass() throws Exception {
102     // set configure to indicate which cp should be loaded
103     Configuration conf = util.getConfiguration();
104     conf.setBoolean("hbase.master.distributed.log.replay", true);
105     conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
106         "org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver",
107         "org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver$Legacy");
108 
109     util.startMiniCluster();
110     cluster = util.getMiniHBaseCluster();
111   }
112 
113   @AfterClass
114   public static void tearDownAfterClass() throws Exception {
115     util.shutdownMiniCluster();
116   }
117 
118   @Test (timeout=300000)
119   public void testRegionObserver() throws IOException {
120     TableName tableName = TableName.valueOf(TEST_TABLE.getNameAsString() + ".testRegionObserver");
121     // recreate table every time in order to reset the status of the
122     // coprocessor.
123     Table table = util.createTable(tableName, new byte[][] {A, B, C});
124     try {
125       verifyMethodResult(SimpleRegionObserver.class, new String[] { "hadPreGet", "hadPostGet",
126           "hadPrePut", "hadPostPut", "hadDelete", "hadPostStartRegionOperation",
127           "hadPostCloseRegionOperation", "hadPostBatchMutateIndispensably" }, tableName,
128         new Boolean[] { false, false, false, false, false, false, false, false });
129 
130       Put put = new Put(ROW);
131       put.add(A, A, A);
132       put.add(B, B, B);
133       put.add(C, C, C);
134       table.put(put);
135 
136       verifyMethodResult(SimpleRegionObserver.class, new String[] { "hadPreGet", "hadPostGet",
137           "hadPrePut", "hadPostPut", "hadPreBatchMutate", "hadPostBatchMutate", "hadDelete",
138           "hadPostStartRegionOperation", "hadPostCloseRegionOperation",
139           "hadPostBatchMutateIndispensably" }, TEST_TABLE, new Boolean[] { false, false, true,
140           true, true, true, false, true, true, true });
141 
142       verifyMethodResult(SimpleRegionObserver.class,
143           new String[] {"getCtPreOpen", "getCtPostOpen", "getCtPreClose", "getCtPostClose"},
144           tableName,
145           new Integer[] {1, 1, 0, 0});
146 
147       Get get = new Get(ROW);
148       get.addColumn(A, A);
149       get.addColumn(B, B);
150       get.addColumn(C, C);
151       table.get(get);
152 
153       verifyMethodResult(SimpleRegionObserver.class,
154           new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut",
155       "hadDelete", "hadPrePreparedDeleteTS"},
156       tableName,
157       new Boolean[] {true, true, true, true, false, false}
158           );
159 
160       Delete delete = new Delete(ROW);
161       delete.deleteColumn(A, A);
162       delete.deleteColumn(B, B);
163       delete.deleteColumn(C, C);
164       table.delete(delete);
165 
166       verifyMethodResult(SimpleRegionObserver.class,
167           new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut",
168         "hadPreBatchMutate", "hadPostBatchMutate", "hadDelete", "hadPrePreparedDeleteTS"},
169         tableName,
170         new Boolean[] {true, true, true, true, true, true, true, true}
171           );
172     } finally {
173       util.deleteTable(tableName);
174       table.close();
175     }
176     verifyMethodResult(SimpleRegionObserver.class,
177         new String[] {"getCtPreOpen", "getCtPostOpen", "getCtPreClose", "getCtPostClose"},
178         tableName,
179         new Integer[] {1, 1, 1, 1});
180   }
181 
182   @Test (timeout=300000)
183   public void testRowMutation() throws IOException {
184     TableName tableName = TableName.valueOf(TEST_TABLE.getNameAsString() + ".testRowMutation");
185     Table table = util.createTable(tableName, new byte[][] {A, B, C});
186     try {
187       verifyMethodResult(SimpleRegionObserver.class,
188         new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut",
189             "hadDeleted"},
190         tableName,
191         new Boolean[] {false, false, false, false, false});
192       Put put = new Put(ROW);
193       put.add(A, A, A);
194       put.add(B, B, B);
195       put.add(C, C, C);
196 
197       Delete delete = new Delete(ROW);
198       delete.deleteColumn(A, A);
199       delete.deleteColumn(B, B);
200       delete.deleteColumn(C, C);
201 
202       RowMutations arm = new RowMutations(ROW);
203       arm.add(put);
204       arm.add(delete);
205       table.mutateRow(arm);
206 
207       verifyMethodResult(SimpleRegionObserver.class,
208           new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut",
209       "hadDeleted"},
210       tableName,
211       new Boolean[] {false, false, true, true, true}
212           );
213     } finally {
214       util.deleteTable(tableName);
215       table.close();
216     }
217   }
218 
219   @Test (timeout=300000)
220   public void testIncrementHook() throws IOException {
221     TableName tableName = TableName.valueOf(TEST_TABLE.getNameAsString() + ".testIncrementHook");
222     Table table = util.createTable(tableName, new byte[][] {A, B, C});
223     try {
224       Increment inc = new Increment(Bytes.toBytes(0));
225       inc.addColumn(A, A, 1);
226 
227       verifyMethodResult(SimpleRegionObserver.class,
228           new String[] {"hadPreIncrement", "hadPostIncrement", "hadPreIncrementAfterRowLock"},
229           tableName,
230           new Boolean[] {false, false, false}
231           );
232 
233       table.increment(inc);
234 
235       verifyMethodResult(SimpleRegionObserver.class,
236           new String[] {"hadPreIncrement", "hadPostIncrement", "hadPreIncrementAfterRowLock"},
237           tableName,
238           new Boolean[] {true, true, true}
239           );
240     } finally {
241       util.deleteTable(tableName);
242       table.close();
243     }
244   }
245 
246   @Test (timeout=300000)
247   public void testCheckAndPutHooks() throws IOException {
248     TableName tableName =
249         TableName.valueOf(TEST_TABLE.getNameAsString() + ".testCheckAndPutHooks");
250     try (Table table = util.createTable(tableName, new byte[][] {A, B, C})) {
251       Put p = new Put(Bytes.toBytes(0));
252       p.add(A, A, A);
253       table.put(p);
254       p = new Put(Bytes.toBytes(0));
255       p.add(A, A, A);
256       verifyMethodResult(SimpleRegionObserver.class,
257           new String[] {"hadPreCheckAndPut",
258               "hadPreCheckAndPutAfterRowLock", "hadPostCheckAndPut"},
259           tableName,
260           new Boolean[] {false, false, false}
261           );
262       table.checkAndPut(Bytes.toBytes(0), A, A, A, p);
263       verifyMethodResult(SimpleRegionObserver.class,
264           new String[] {"hadPreCheckAndPut",
265               "hadPreCheckAndPutAfterRowLock", "hadPostCheckAndPut"},
266           tableName,
267           new Boolean[] {true, true, true}
268           );
269     } finally {
270       util.deleteTable(tableName);
271     }
272   }
273 
274   @Test (timeout=300000)
275   public void testCheckAndDeleteHooks() throws IOException {
276     TableName tableName =
277         TableName.valueOf(TEST_TABLE.getNameAsString() + ".testCheckAndDeleteHooks");
278     Table table = util.createTable(tableName, new byte[][] {A, B, C});
279     try {
280       Put p = new Put(Bytes.toBytes(0));
281       p.add(A, A, A);
282       table.put(p);
283       Delete d = new Delete(Bytes.toBytes(0));
284       table.delete(d);
285       verifyMethodResult(SimpleRegionObserver.class,
286           new String[] {"hadPreCheckAndDelete",
287               "hadPreCheckAndDeleteAfterRowLock", "hadPostCheckAndDelete"},
288           tableName,
289           new Boolean[] {false, false, false}
290           );
291       table.checkAndDelete(Bytes.toBytes(0), A, A, A, d);
292       verifyMethodResult(SimpleRegionObserver.class,
293           new String[] {"hadPreCheckAndDelete",
294               "hadPreCheckAndDeleteAfterRowLock", "hadPostCheckAndDelete"},
295           tableName,
296           new Boolean[] {true, true, true}
297           );
298     } finally {
299       util.deleteTable(tableName);
300       table.close();
301     }
302   }
303 
304   @Test (timeout=300000)
305   public void testAppendHook() throws IOException {
306     TableName tableName = TableName.valueOf(TEST_TABLE.getNameAsString() + ".testAppendHook");
307     Table table = util.createTable(tableName, new byte[][] {A, B, C});
308     try {
309       Append app = new Append(Bytes.toBytes(0));
310       app.add(A, A, A);
311 
312       verifyMethodResult(SimpleRegionObserver.class,
313           new String[] {"hadPreAppend", "hadPostAppend", "hadPreAppendAfterRowLock"},
314           tableName,
315           new Boolean[] {false, false, false}
316           );
317 
318       table.append(app);
319 
320       verifyMethodResult(SimpleRegionObserver.class,
321           new String[] {"hadPreAppend", "hadPostAppend", "hadPreAppendAfterRowLock"},
322           tableName,
323           new Boolean[] {true, true, true}
324           );
325     } finally {
326       util.deleteTable(tableName);
327       table.close();
328     }
329   }
330 
331   @Test (timeout=300000)
332   // HBase-3583
333   public void testHBase3583() throws IOException {
334     TableName tableName =
335         TableName.valueOf("testHBase3583");
336     util.createTable(tableName, new byte[][] {A, B, C});
337     util.waitUntilAllRegionsAssigned(tableName);
338 
339     verifyMethodResult(SimpleRegionObserver.class,
340         new String[] {"hadPreGet", "hadPostGet", "wasScannerNextCalled",
341             "wasScannerCloseCalled"},
342         tableName,
343         new Boolean[] {false, false, false, false}
344     );
345 
346     Table table = new HTable(util.getConfiguration(), tableName);
347     Put put = new Put(ROW);
348     put.add(A, A, A);
349     table.put(put);
350 
351     Get get = new Get(ROW);
352     get.addColumn(A, A);
353     table.get(get);
354 
355     // verify that scannerNext and scannerClose upcalls won't be invoked
356     // when we perform get().
357     verifyMethodResult(SimpleRegionObserver.class,
358         new String[] {"hadPreGet", "hadPostGet", "wasScannerNextCalled",
359             "wasScannerCloseCalled"},
360         tableName,
361         new Boolean[] {true, true, false, false}
362     );
363 
364     Scan s = new Scan();
365     ResultScanner scanner = table.getScanner(s);
366     try {
367       for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
368       }
369     } finally {
370       scanner.close();
371     }
372 
373     // now scanner hooks should be invoked.
374     verifyMethodResult(SimpleRegionObserver.class,
375         new String[] {"wasScannerNextCalled", "wasScannerCloseCalled"},
376         tableName,
377         new Boolean[] {true, true}
378     );
379     util.deleteTable(tableName);
380     table.close();
381   }
382 
383   @Test(timeout = 300000)
384   public void testHBASE14489() throws IOException {
385     TableName tableName = TableName.valueOf("testHBASE14489");
386     HTable table = util.createTable(tableName, new byte[][] { A });
387     Put put = new Put(ROW);
388     put.addColumn(A, A, A);
389     table.put(put);
390 
391     Scan s = new Scan();
392     s.setFilter(new FilterAllFilter());
393     ResultScanner scanner = table.getScanner(s);
394     try {
395       for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
396       }
397     } finally {
398       scanner.close();
399     }
400     verifyMethodResult(SimpleRegionObserver.class, new String[] { "wasScannerFilterRowCalled" },
401       tableName, new Boolean[] { true });
402     util.deleteTable(tableName);
403     table.close();
404   }
405 
406   @Test (timeout=300000)
407   // HBase-3758
408   public void testHBase3758() throws IOException {
409     TableName tableName =
410         TableName.valueOf("testHBase3758");
411     util.createTable(tableName, new byte[][] {A, B, C});
412 
413     verifyMethodResult(SimpleRegionObserver.class,
414         new String[] {"hadDeleted", "wasScannerOpenCalled"},
415         tableName,
416         new Boolean[] {false, false}
417     );
418 
419     Table table = new HTable(util.getConfiguration(), tableName);
420     Put put = new Put(ROW);
421     put.add(A, A, A);
422     table.put(put);
423 
424     Delete delete = new Delete(ROW);
425     table.delete(delete);
426 
427     verifyMethodResult(SimpleRegionObserver.class,
428         new String[] {"hadDeleted", "wasScannerOpenCalled"},
429         tableName,
430         new Boolean[] {true, false}
431     );
432 
433     Scan s = new Scan();
434     ResultScanner scanner = table.getScanner(s);
435     try {
436       for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
437       }
438     } finally {
439       scanner.close();
440     }
441 
442     // now scanner hooks should be invoked.
443     verifyMethodResult(SimpleRegionObserver.class,
444         new String[] {"wasScannerOpenCalled"},
445         tableName,
446         new Boolean[] {true}
447     );
448     util.deleteTable(tableName);
449     table.close();
450   }
451 
452   /* Overrides compaction to only output rows with keys that are even numbers */
453   public static class EvenOnlyCompactor extends BaseRegionObserver {
454     long lastCompaction;
455     long lastFlush;
456 
457     @Override
458     public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e,
459         Store store, final InternalScanner scanner, final ScanType scanType) {
460       return new InternalScanner() {
461         @Override
462         public boolean next(List<Cell> results) throws IOException {
463           return next(results, NoLimitScannerContext.getInstance());
464         }
465 
466         @Override
467         public boolean next(List<Cell> results, ScannerContext scannerContext)
468             throws IOException {
469           List<Cell> internalResults = new ArrayList<Cell>();
470           boolean hasMore;
471           do {
472             hasMore = scanner.next(internalResults, scannerContext);
473             if (!internalResults.isEmpty()) {
474               long row = Bytes.toLong(CellUtil.cloneValue(internalResults.get(0)));
475               if (row % 2 == 0) {
476                 // return this row
477                 break;
478               }
479               // clear and continue
480               internalResults.clear();
481             }
482           } while (hasMore);
483 
484           if (!internalResults.isEmpty()) {
485             results.addAll(internalResults);
486           }
487           return hasMore;
488         }
489 
490         @Override
491         public void close() throws IOException {
492           scanner.close();
493         }
494       };
495     }
496 
497     @Override
498     public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e,
499         Store store, StoreFile resultFile) {
500       lastCompaction = EnvironmentEdgeManager.currentTime();
501     }
502 
503     @Override
504     public void postFlush(ObserverContext<RegionCoprocessorEnvironment> e) {
505       lastFlush = EnvironmentEdgeManager.currentTime();
506     }
507   }
508   /**
509    * Tests overriding compaction handling via coprocessor hooks
510    * @throws Exception
511    */
512   @Test (timeout=300000)
513   public void testCompactionOverride() throws Exception {
514     TableName compactTable = TableName.valueOf("TestCompactionOverride");
515     Admin admin = util.getHBaseAdmin();
516     if (admin.tableExists(compactTable)) {
517       admin.disableTable(compactTable);
518       admin.deleteTable(compactTable);
519     }
520 
521     HTableDescriptor htd = new HTableDescriptor(compactTable);
522     htd.addFamily(new HColumnDescriptor(A));
523     htd.addCoprocessor(EvenOnlyCompactor.class.getName());
524     admin.createTable(htd);
525 
526     Table table = new HTable(util.getConfiguration(), compactTable);
527     for (long i=1; i<=10; i++) {
528       byte[] iBytes = Bytes.toBytes(i);
529       Put put = new Put(iBytes);
530       put.setDurability(Durability.SKIP_WAL);
531       put.add(A, A, iBytes);
532       table.put(put);
533     }
534 
535     HRegion firstRegion = cluster.getRegions(compactTable).get(0);
536     Coprocessor cp = firstRegion.getCoprocessorHost().findCoprocessor(
537         EvenOnlyCompactor.class.getName());
538     assertNotNull("EvenOnlyCompactor coprocessor should be loaded", cp);
539     EvenOnlyCompactor compactor = (EvenOnlyCompactor)cp;
540 
541     // force a compaction
542     long ts = System.currentTimeMillis();
543     admin.flush(compactTable);
544     // wait for flush
545     for (int i=0; i<10; i++) {
546       if (compactor.lastFlush >= ts) {
547         break;
548       }
549       Thread.sleep(1000);
550     }
551     assertTrue("Flush didn't complete", compactor.lastFlush >= ts);
552     LOG.debug("Flush complete");
553 
554     ts = compactor.lastFlush;
555     admin.majorCompact(compactTable);
556     // wait for compaction
557     for (int i=0; i<30; i++) {
558       if (compactor.lastCompaction >= ts) {
559         break;
560       }
561       Thread.sleep(1000);
562     }
563     LOG.debug("Last compaction was at "+compactor.lastCompaction);
564     assertTrue("Compaction didn't complete", compactor.lastCompaction >= ts);
565 
566     // only even rows should remain
567     ResultScanner scanner = table.getScanner(new Scan());
568     try {
569       for (long i=2; i<=10; i+=2) {
570         Result r = scanner.next();
571         assertNotNull(r);
572         assertFalse(r.isEmpty());
573         byte[] iBytes = Bytes.toBytes(i);
574         assertArrayEquals("Row should be "+i, r.getRow(), iBytes);
575         assertArrayEquals("Value should be "+i, r.getValue(A, A), iBytes);
576       }
577     } finally {
578       scanner.close();
579     }
580     table.close();
581   }
582 
583   @Test (timeout=300000)
584   public void bulkLoadHFileTest() throws Exception {
585     String testName = TestRegionObserverInterface.class.getName()+".bulkLoadHFileTest";
586     TableName tableName = TableName.valueOf(TEST_TABLE.getNameAsString() + ".bulkLoadHFileTest");
587     Configuration conf = util.getConfiguration();
588     HTable table = util.createTable(tableName, new byte[][] {A, B, C});
589     try {
590       verifyMethodResult(SimpleRegionObserver.class,
591           new String[] {"hadPreBulkLoadHFile", "hadPostBulkLoadHFile"},
592           tableName,
593           new Boolean[] {false, false}
594           );
595 
596       FileSystem fs = util.getTestFileSystem();
597       final Path dir = util.getDataTestDirOnTestFS(testName).makeQualified(fs);
598       Path familyDir = new Path(dir, Bytes.toString(A));
599 
600       createHFile(util.getConfiguration(), fs, new Path(familyDir,Bytes.toString(A)), A, A);
601 
602       // Bulk load
603       new LoadIncrementalHFiles(conf).doBulkLoad(dir, table);
604 
605       verifyMethodResult(SimpleRegionObserver.class,
606           new String[] {"hadPreBulkLoadHFile", "hadPostBulkLoadHFile"},
607           tableName,
608           new Boolean[] {true, true}
609           );
610     } finally {
611       util.deleteTable(tableName);
612       table.close();
613     }
614   }
615 
616   @Ignore // TODO: HBASE-13391 to fix flaky test
617   @Test (timeout=300000)
618   public void testRecovery() throws Exception {
619     LOG.info(TestRegionObserverInterface.class.getName() +".testRecovery");
620     TableName tableName = TableName.valueOf(TEST_TABLE.getNameAsString() + ".testRecovery");
621     HTable table = util.createTable(tableName, new byte[][] {A, B, C});
622     try {
623       JVMClusterUtil.RegionServerThread rs1 = cluster.startRegionServer();
624       ServerName sn2 = rs1.getRegionServer().getServerName();
625       String regEN = table.getRegionLocations().firstEntry().getKey().getEncodedName();
626 
627       util.getHBaseAdmin().move(regEN.getBytes(), sn2.getServerName().getBytes());
628       while (!sn2.equals(table.getRegionLocations().firstEntry().getValue() )){
629         Thread.sleep(100);
630       }
631 
632       Put put = new Put(ROW);
633       put.add(A, A, A);
634       put.add(B, B, B);
635       put.add(C, C, C);
636       table.put(put);
637 
638       verifyMethodResult(SimpleRegionObserver.class,
639           new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut",
640         "hadPreBatchMutate", "hadPostBatchMutate", "hadDelete"},
641         tableName,
642         new Boolean[] {false, false, true, true, true, true, false}
643           );
644 
645       verifyMethodResult(SimpleRegionObserver.class,
646           new String[] {"getCtPreWALRestore", "getCtPostWALRestore", "getCtPrePut", "getCtPostPut",
647               "getCtPreWALRestoreDeprecated", "getCtPostWALRestoreDeprecated"},
648           tableName,
649           new Integer[] {0, 0, 1, 1, 0, 0});
650 
651       cluster.killRegionServer(rs1.getRegionServer().getServerName());
652       Threads.sleep(1000); // Let the kill soak in.
653       util.waitUntilAllRegionsAssigned(tableName);
654       LOG.info("All regions assigned");
655 
656       verifyMethodResult(SimpleRegionObserver.class,
657           new String[] {"getCtPreWALRestore", "getCtPostWALRestore", "getCtPrePut", "getCtPostPut",
658               "getCtPreWALRestoreDeprecated", "getCtPostWALRestoreDeprecated"},
659           tableName,
660           new Integer[]{1, 1, 0, 0, 0, 0});
661     } finally {
662       util.deleteTable(tableName);
663       table.close();
664     }
665   }
666 
667   @Ignore // TODO: HBASE-13391 to fix flaky test
668   @Test (timeout=300000)
669   public void testLegacyRecovery() throws Exception {
670     LOG.info(TestRegionObserverInterface.class.getName() +".testLegacyRecovery");
671     TableName tableName = TableName.valueOf(TEST_TABLE.getNameAsString() + ".testLegacyRecovery");
672     HTable table = util.createTable(tableName, new byte[][] {A, B, C});
673     try {
674       JVMClusterUtil.RegionServerThread rs1 = cluster.startRegionServer();
675       ServerName sn2 = rs1.getRegionServer().getServerName();
676       String regEN = table.getRegionLocations().firstEntry().getKey().getEncodedName();
677 
678       util.getHBaseAdmin().move(regEN.getBytes(), sn2.getServerName().getBytes());
679       while (!sn2.equals(table.getRegionLocations().firstEntry().getValue() )){
680         Thread.sleep(100);
681       }
682 
683       Put put = new Put(ROW);
684       put.add(A, A, A);
685       put.add(B, B, B);
686       put.add(C, C, C);
687       table.put(put);
688 
689       verifyMethodResult(SimpleRegionObserver.Legacy.class,
690           new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut",
691         "hadPreBatchMutate", "hadPostBatchMutate", "hadDelete"},
692         tableName,
693         new Boolean[] {false, false, true, true, true, true, false}
694           );
695 
696       verifyMethodResult(SimpleRegionObserver.Legacy.class,
697           new String[] {"getCtPreWALRestore", "getCtPostWALRestore", "getCtPrePut", "getCtPostPut",
698               "getCtPreWALRestoreDeprecated", "getCtPostWALRestoreDeprecated"},
699           tableName,
700           new Integer[] {0, 0, 1, 1, 0, 0});
701 
702       cluster.killRegionServer(rs1.getRegionServer().getServerName());
703       Threads.sleep(1000); // Let the kill soak in.
704       util.waitUntilAllRegionsAssigned(tableName);
705       LOG.info("All regions assigned");
706 
707       verifyMethodResult(SimpleRegionObserver.Legacy.class,
708           new String[] {"getCtPreWALRestore", "getCtPostWALRestore", "getCtPrePut", "getCtPostPut",
709               "getCtPreWALRestoreDeprecated", "getCtPostWALRestoreDeprecated"},
710           tableName,
711           new Integer[]{1, 1, 0, 0, 1, 1});
712     } finally {
713       util.deleteTable(tableName);
714       table.close();
715     }
716   }
717 
718   @Test (timeout=300000)
719   public void testPreWALRestoreSkip() throws Exception {
720     LOG.info(TestRegionObserverInterface.class.getName() + ".testPreWALRestoreSkip");
721     TableName tableName = TableName.valueOf(SimpleRegionObserver.TABLE_SKIPPED);
722     HTable table = util.createTable(tableName, new byte[][] { A, B, C });
723 
724     JVMClusterUtil.RegionServerThread rs1 = cluster.startRegionServer();
725     ServerName sn2 = rs1.getRegionServer().getServerName();
726     String regEN = table.getRegionLocations().firstEntry().getKey().getEncodedName();
727 
728     util.getHBaseAdmin().move(regEN.getBytes(), sn2.getServerName().getBytes());
729     while (!sn2.equals(table.getRegionLocations().firstEntry().getValue())) {
730       Thread.sleep(100);
731     }
732 
733     Put put = new Put(ROW);
734     put.add(A, A, A);
735     put.add(B, B, B);
736     put.add(C, C, C);
737     table.put(put);
738     table.flushCommits();
739 
740     cluster.killRegionServer(rs1.getRegionServer().getServerName());
741     Threads.sleep(20000); // just to be sure that the kill has fully started.
742     util.waitUntilAllRegionsAssigned(tableName);
743 
744     verifyMethodResult(SimpleRegionObserver.class, new String[] { "getCtPreWALRestore",
745         "getCtPostWALRestore", "getCtPreWALRestoreDeprecated", "getCtPostWALRestoreDeprecated"},
746         tableName,
747         new Integer[] {0, 0, 0, 0});
748 
749     util.deleteTable(tableName);
750     table.close();
751   }
752 
753   // check each region whether the coprocessor upcalls are called or not.
754   private void verifyMethodResult(Class<?> c, String methodName[], TableName tableName,
755                                   Object value[]) throws IOException {
756     try {
757       for (JVMClusterUtil.RegionServerThread t : cluster.getRegionServerThreads()) {
758         if (!t.isAlive() || t.getRegionServer().isAborted() || t.getRegionServer().isStopping()){
759           continue;
760         }
761         for (HRegionInfo r : ProtobufUtil.getOnlineRegions(t.getRegionServer().getRSRpcServices())) {
762           if (!r.getTable().equals(tableName)) {
763             continue;
764           }
765           RegionCoprocessorHost cph = t.getRegionServer().getOnlineRegion(r.getRegionName()).
766               getCoprocessorHost();
767 
768           Coprocessor cp = cph.findCoprocessor(c.getName());
769           assertNotNull(cp);
770           for (int i = 0; i < methodName.length; ++i) {
771             Method m = c.getMethod(methodName[i]);
772             Object o = m.invoke(cp);
773             assertTrue("Result of " + c.getName() + "." + methodName[i]
774                 + " is expected to be " + value[i].toString()
775                 + ", while we get " + o.toString(), o.equals(value[i]));
776           }
777         }
778       }
779     } catch (Exception e) {
780       throw new IOException(e.toString());
781     }
782   }
783 
784   private static void createHFile(
785       Configuration conf,
786       FileSystem fs, Path path,
787       byte[] family, byte[] qualifier) throws IOException {
788     HFileContext context = new HFileContextBuilder().build();
789     HFile.Writer writer = HFile.getWriterFactory(conf, new CacheConfig(conf))
790         .withPath(fs, path)
791         .withFileContext(context)
792         .create();
793     long now = System.currentTimeMillis();
794     try {
795       for (int i =1;i<=9;i++) {
796         KeyValue kv = new KeyValue(Bytes.toBytes(i+""), family, qualifier, now, Bytes.toBytes(i+""));
797         writer.append(kv);
798       }
799     } finally {
800       writer.close();
801     }
802   }
803 }