View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   * http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.coprocessor;
21  
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertNotNull;
24  import static org.junit.Assert.assertNull;
25  import static org.junit.Assert.assertTrue;
26  
27  import java.io.IOException;
28  import java.util.Collection;
29  import java.util.List;
30  import java.util.Map;
31  import java.util.NavigableMap;
32  import java.util.concurrent.CountDownLatch;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.hbase.*;
38  import org.apache.hadoop.hbase.client.HBaseAdmin;
39  import org.apache.hadoop.hbase.client.HTable;
40  import org.apache.hadoop.hbase.master.AssignmentManager;
41  import org.apache.hadoop.hbase.master.HMaster;
42  import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
43  import org.apache.hadoop.hbase.master.RegionPlan;
44  import org.apache.hadoop.hbase.master.RegionState;
45  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
46  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsRequest;
47  import org.apache.hadoop.hbase.protobuf.RequestConverter;
48  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
49  import org.apache.hadoop.hbase.regionserver.HRegionServer;
50  import org.apache.hadoop.hbase.util.Bytes;
51  import org.apache.hadoop.hbase.util.Threads;
52  import org.junit.AfterClass;
53  import org.junit.BeforeClass;
54  import org.junit.Test;
55  import org.junit.experimental.categories.Category;
56  
57  /**
58   * Tests invocation of the {@link org.apache.hadoop.hbase.coprocessor.MasterObserver}
59   * interface hooks at all appropriate times during normal HMaster operations.
60   */
61  @Category(MediumTests.class)
62  public class TestMasterObserver {
63    private static final Log LOG = LogFactory.getLog(TestMasterObserver.class);
64  
65    public static CountDownLatch tableCreationLatch = new CountDownLatch(1);
66  
67    public static class CPMasterObserver implements MasterObserver {
68  
69      private boolean bypass = false;
70      private boolean preCreateTableCalled;
71      private boolean postCreateTableCalled;
72      private boolean preDeleteTableCalled;
73      private boolean postDeleteTableCalled;
74      private boolean preModifyTableCalled;
75      private boolean postModifyTableCalled;
76      private boolean preCreateNamespaceCalled;
77      private boolean postCreateNamespaceCalled;
78      private boolean preDeleteNamespaceCalled;
79      private boolean postDeleteNamespaceCalled;
80      private boolean preModifyNamespaceCalled;
81      private boolean postModifyNamespaceCalled;
82      private boolean preAddColumnCalled;
83      private boolean postAddColumnCalled;
84      private boolean preModifyColumnCalled;
85      private boolean postModifyColumnCalled;
86      private boolean preDeleteColumnCalled;
87      private boolean postDeleteColumnCalled;
88      private boolean preEnableTableCalled;
89      private boolean postEnableTableCalled;
90      private boolean preDisableTableCalled;
91      private boolean postDisableTableCalled;
92      private boolean preMoveCalled;
93      private boolean postMoveCalled;
94      private boolean preAssignCalled;
95      private boolean postAssignCalled;
96      private boolean preUnassignCalled;
97      private boolean postUnassignCalled;
98      private boolean preRegionOfflineCalled;
99      private boolean postRegionOfflineCalled;
100     private boolean preBalanceCalled;
101     private boolean postBalanceCalled;
102     private boolean preBalanceSwitchCalled;
103     private boolean postBalanceSwitchCalled;
104     private boolean preShutdownCalled;
105     private boolean preStopMasterCalled;
106     private boolean preMasterInitializationCalled;
107     private boolean postStartMasterCalled;
108     private boolean startCalled;
109     private boolean stopCalled;
110     private boolean preSnapshotCalled;
111     private boolean postSnapshotCalled;
112     private boolean preCloneSnapshotCalled;
113     private boolean postCloneSnapshotCalled;
114     private boolean preRestoreSnapshotCalled;
115     private boolean postRestoreSnapshotCalled;
116     private boolean preDeleteSnapshotCalled;
117     private boolean postDeleteSnapshotCalled;
118     private boolean preCreateTableHandlerCalled;
119     private boolean postCreateTableHandlerCalled;
120     private boolean preDeleteTableHandlerCalled;
121     private boolean postDeleteTableHandlerCalled;
122     private boolean preAddColumnHandlerCalled;
123     private boolean postAddColumnHandlerCalled;
124     private boolean preModifyColumnHandlerCalled;
125     private boolean postModifyColumnHandlerCalled;
126     private boolean preDeleteColumnHandlerCalled;
127     private boolean postDeleteColumnHandlerCalled;
128     private boolean preEnableTableHandlerCalled;
129     private boolean postEnableTableHandlerCalled;
130     private boolean preDisableTableHandlerCalled;
131     private boolean postDisableTableHandlerCalled;
132     private boolean preModifyTableHandlerCalled;
133     private boolean postModifyTableHandlerCalled;
134     private boolean preGetTableDescriptorsCalled;
135     private boolean postGetTableDescriptorsCalled;
136 
137     public void enableBypass(boolean bypass) {
138       this.bypass = bypass;
139     }
140 
141     public void resetStates() {
142       preCreateTableCalled = false;
143       postCreateTableCalled = false;
144       preDeleteTableCalled = false;
145       postDeleteTableCalled = false;
146       preModifyTableCalled = false;
147       postModifyTableCalled = false;
148       preCreateNamespaceCalled = false;
149       postCreateNamespaceCalled = false;
150       preDeleteNamespaceCalled = false;
151       postDeleteNamespaceCalled = false;
152       preModifyNamespaceCalled = false;
153       postModifyNamespaceCalled = false;
154       preAddColumnCalled = false;
155       postAddColumnCalled = false;
156       preModifyColumnCalled = false;
157       postModifyColumnCalled = false;
158       preDeleteColumnCalled = false;
159       postDeleteColumnCalled = false;
160       preEnableTableCalled = false;
161       postEnableTableCalled = false;
162       preDisableTableCalled = false;
163       postDisableTableCalled = false;
164       preMoveCalled= false;
165       postMoveCalled = false;
166       preAssignCalled = false;
167       postAssignCalled = false;
168       preUnassignCalled = false;
169       postUnassignCalled = false;
170       preRegionOfflineCalled = false;
171       postRegionOfflineCalled = false;
172       preBalanceCalled = false;
173       postBalanceCalled = false;
174       preBalanceSwitchCalled = false;
175       postBalanceSwitchCalled = false;
176       preSnapshotCalled = false;
177       postSnapshotCalled = false;
178       preCloneSnapshotCalled = false;
179       postCloneSnapshotCalled = false;
180       preRestoreSnapshotCalled = false;
181       postRestoreSnapshotCalled = false;
182       preDeleteSnapshotCalled = false;
183       postDeleteSnapshotCalled = false;
184       preCreateTableHandlerCalled = false;
185       postCreateTableHandlerCalled = false;
186       preDeleteTableHandlerCalled = false;
187       postDeleteTableHandlerCalled = false;
188       preModifyTableHandlerCalled = false;
189       postModifyTableHandlerCalled = false;
190       preAddColumnHandlerCalled = false;
191       postAddColumnHandlerCalled = false;
192       preModifyColumnHandlerCalled = false;
193       postModifyColumnHandlerCalled = false;
194       preDeleteColumnHandlerCalled = false;
195       postDeleteColumnHandlerCalled = false;
196       preEnableTableHandlerCalled = false;
197       postEnableTableHandlerCalled = false;
198       preDisableTableHandlerCalled = false;
199       postDisableTableHandlerCalled = false;
200       preModifyTableHandlerCalled = false;
201       postModifyTableHandlerCalled = false;
202       preGetTableDescriptorsCalled = false;
203       postGetTableDescriptorsCalled = false;
204     }
205 
206     @Override
207     public void preCreateTable(ObserverContext<MasterCoprocessorEnvironment> env,
208         HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
209       if (bypass) {
210         env.bypass();
211       }
212       preCreateTableCalled = true;
213     }
214 
215     @Override
216     public void postCreateTable(ObserverContext<MasterCoprocessorEnvironment> env,
217         HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
218       postCreateTableCalled = true;
219     }
220 
221     public boolean wasCreateTableCalled() {
222       return preCreateTableCalled && postCreateTableCalled;
223     }
224 
225     public boolean preCreateTableCalledOnly() {
226       return preCreateTableCalled && !postCreateTableCalled;
227     }
228 
229     @Override
230     public void preDeleteTable(ObserverContext<MasterCoprocessorEnvironment> env,
231         TableName tableName) throws IOException {
232       if (bypass) {
233         env.bypass();
234       }
235       preDeleteTableCalled = true;
236     }
237 
238     @Override
239     public void postDeleteTable(ObserverContext<MasterCoprocessorEnvironment> env,
240         TableName tableName) throws IOException {
241       postDeleteTableCalled = true;
242     }
243 
244     public boolean wasDeleteTableCalled() {
245       return preDeleteTableCalled && postDeleteTableCalled;
246     }
247 
248     public boolean preDeleteTableCalledOnly() {
249       return preDeleteTableCalled && !postDeleteTableCalled;
250     }
251 
252     @Override
253     public void preModifyTable(ObserverContext<MasterCoprocessorEnvironment> env,
254         TableName tableName, HTableDescriptor htd) throws IOException {
255       if (bypass) {
256         env.bypass();
257       }else{
258         env.shouldBypass();
259       }
260       preModifyTableCalled = true;
261     }
262 
263     @Override
264     public void postModifyTable(ObserverContext<MasterCoprocessorEnvironment> env,
265         TableName tableName, HTableDescriptor htd) throws IOException {
266       postModifyTableCalled = true;
267     }
268 
269     public boolean wasModifyTableCalled() {
270       return preModifyTableCalled && postModifyTableCalled;
271     }
272 
273     public boolean preModifyTableCalledOnly() {
274       return preModifyTableCalled && !postModifyTableCalled;
275     }
276 
277     @Override
278     public void preCreateNamespace(ObserverContext<MasterCoprocessorEnvironment> env,
279         NamespaceDescriptor ns) throws IOException {
280       if (bypass) {
281         env.bypass();
282       }
283       preCreateNamespaceCalled = true;
284     }
285 
286     @Override
287     public void postCreateNamespace(ObserverContext<MasterCoprocessorEnvironment> env,
288         NamespaceDescriptor ns) throws IOException {
289       postCreateNamespaceCalled = true;
290     }
291 
292     public boolean wasCreateNamespaceCalled() {
293       return preCreateNamespaceCalled && postCreateNamespaceCalled;
294     }
295 
296     public boolean preCreateNamespaceCalledOnly() {
297       return preCreateNamespaceCalled && !postCreateNamespaceCalled;
298     }
299 
300     @Override
301     public void preDeleteNamespace(ObserverContext<MasterCoprocessorEnvironment> env,
302         String name) throws IOException {
303       if (bypass) {
304         env.bypass();
305       }
306       preDeleteNamespaceCalled = true;
307     }
308 
309     @Override
310     public void postDeleteNamespace(ObserverContext<MasterCoprocessorEnvironment> env,
311         String name) throws IOException {
312       postDeleteNamespaceCalled = true;
313     }
314 
315     public boolean wasDeleteNamespaceCalled() {
316       return preDeleteNamespaceCalled && postDeleteNamespaceCalled;
317     }
318 
319     public boolean preDeleteNamespaceCalledOnly() {
320       return preDeleteNamespaceCalled && !postDeleteNamespaceCalled;
321     }
322 
323     @Override
324     public void preModifyNamespace(ObserverContext<MasterCoprocessorEnvironment> env,
325         NamespaceDescriptor ns) throws IOException {
326       if (bypass) {
327         env.bypass();
328       }
329       preModifyNamespaceCalled = true;
330     }
331 
332     @Override
333     public void postModifyNamespace(ObserverContext<MasterCoprocessorEnvironment> env,
334         NamespaceDescriptor ns) throws IOException {
335       postModifyNamespaceCalled = true;
336     }
337 
338     public boolean wasModifyNamespaceCalled() {
339       return preModifyNamespaceCalled && postModifyNamespaceCalled;
340     }
341 
342     public boolean preModifyNamespaceCalledOnly() {
343       return preModifyNamespaceCalled && !postModifyNamespaceCalled;
344     }
345 
346     @Override
347     public void preAddColumn(ObserverContext<MasterCoprocessorEnvironment> env,
348         TableName tableName, HColumnDescriptor column) throws IOException {
349       if (bypass) {
350         env.bypass();
351       }else{
352         env.shouldBypass();
353       }
354 
355       preAddColumnCalled = true;
356     }
357 
358     @Override
359     public void postAddColumn(ObserverContext<MasterCoprocessorEnvironment> env,
360         TableName tableName, HColumnDescriptor column) throws IOException {
361       postAddColumnCalled = true;
362     }
363 
364     public boolean wasAddColumnCalled() {
365       return preAddColumnCalled && postAddColumnCalled;
366     }
367 
368     public boolean preAddColumnCalledOnly() {
369       return preAddColumnCalled && !postAddColumnCalled;
370     }
371 
372     @Override
373     public void preModifyColumn(ObserverContext<MasterCoprocessorEnvironment> env,
374         TableName tableName, HColumnDescriptor descriptor) throws IOException {
375       if (bypass) {
376         env.bypass();
377       }
378       preModifyColumnCalled = true;
379     }
380 
381     @Override
382     public void postModifyColumn(ObserverContext<MasterCoprocessorEnvironment> env,
383         TableName tableName, HColumnDescriptor descriptor) throws IOException {
384       postModifyColumnCalled = true;
385     }
386 
387     public boolean wasModifyColumnCalled() {
388       return preModifyColumnCalled && postModifyColumnCalled;
389     }
390 
391     public boolean preModifyColumnCalledOnly() {
392       return preModifyColumnCalled && !postModifyColumnCalled;
393     }
394 
395     @Override
396     public void preDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> env,
397         TableName tableName, byte[] c) throws IOException {
398       if (bypass) {
399         env.bypass();
400       }
401       preDeleteColumnCalled = true;
402     }
403 
404     @Override
405     public void postDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> env,
406         TableName tableName, byte[] c) throws IOException {
407       postDeleteColumnCalled = true;
408     }
409 
410     public boolean wasDeleteColumnCalled() {
411       return preDeleteColumnCalled && postDeleteColumnCalled;
412     }
413 
414     public boolean preDeleteColumnCalledOnly() {
415       return preDeleteColumnCalled && !postDeleteColumnCalled;
416     }
417 
418     @Override
419     public void preEnableTable(ObserverContext<MasterCoprocessorEnvironment> env,
420         TableName tableName) throws IOException {
421       if (bypass) {
422         env.bypass();
423       }
424       preEnableTableCalled = true;
425     }
426 
427     @Override
428     public void postEnableTable(ObserverContext<MasterCoprocessorEnvironment> env,
429         TableName tableName) throws IOException {
430       postEnableTableCalled = true;
431     }
432 
433     public boolean wasEnableTableCalled() {
434       return preEnableTableCalled && postEnableTableCalled;
435     }
436 
437     public boolean preEnableTableCalledOnly() {
438       return preEnableTableCalled && !postEnableTableCalled;
439     }
440 
441     @Override
442     public void preDisableTable(ObserverContext<MasterCoprocessorEnvironment> env,
443         TableName tableName) throws IOException {
444       if (bypass) {
445         env.bypass();
446       }
447       preDisableTableCalled = true;
448     }
449 
450     @Override
451     public void postDisableTable(ObserverContext<MasterCoprocessorEnvironment> env,
452         TableName tableName) throws IOException {
453       postDisableTableCalled = true;
454     }
455 
456     public boolean wasDisableTableCalled() {
457       return preDisableTableCalled && postDisableTableCalled;
458     }
459 
460     public boolean preDisableTableCalledOnly() {
461       return preDisableTableCalled && !postDisableTableCalled;
462     }
463 
464     @Override
465     public void preMove(ObserverContext<MasterCoprocessorEnvironment> env,
466         HRegionInfo region, ServerName srcServer, ServerName destServer)
467     throws IOException {
468       if (bypass) {
469         env.bypass();
470       }
471       preMoveCalled = true;
472     }
473 
474     @Override
475     public void postMove(ObserverContext<MasterCoprocessorEnvironment> env, HRegionInfo region,
476         ServerName srcServer, ServerName destServer)
477     throws IOException {
478       postMoveCalled = true;
479     }
480 
481     public boolean wasMoveCalled() {
482       return preMoveCalled && postMoveCalled;
483     }
484 
485     public boolean preMoveCalledOnly() {
486       return preMoveCalled && !postMoveCalled;
487     }
488 
489     @Override
490     public void preAssign(ObserverContext<MasterCoprocessorEnvironment> env,
491         final HRegionInfo regionInfo) throws IOException {
492       if (bypass) {
493         env.bypass();
494       }
495       preAssignCalled = true;
496     }
497 
498     @Override
499     public void postAssign(ObserverContext<MasterCoprocessorEnvironment> env,
500         final HRegionInfo regionInfo) throws IOException {
501       postAssignCalled = true;
502     }
503 
504     public boolean wasAssignCalled() {
505       return preAssignCalled && postAssignCalled;
506     }
507 
508     public boolean preAssignCalledOnly() {
509       return preAssignCalled && !postAssignCalled;
510     }
511 
512     @Override
513     public void preUnassign(ObserverContext<MasterCoprocessorEnvironment> env,
514         final HRegionInfo regionInfo, final boolean force) throws IOException {
515       if (bypass) {
516         env.bypass();
517       }
518       preUnassignCalled = true;
519     }
520 
521     @Override
522     public void postUnassign(ObserverContext<MasterCoprocessorEnvironment> env,
523         final HRegionInfo regionInfo, final boolean force) throws IOException {
524       postUnassignCalled = true;
525     }
526 
527     public boolean wasUnassignCalled() {
528       return preUnassignCalled && postUnassignCalled;
529     }
530 
531     public boolean preUnassignCalledOnly() {
532       return preUnassignCalled && !postUnassignCalled;
533     }
534 
535     @Override
536     public void preRegionOffline(ObserverContext<MasterCoprocessorEnvironment> env,
537         final HRegionInfo regionInfo) throws IOException {
538       preRegionOfflineCalled = true;
539     }
540 
541     @Override
542     public void postRegionOffline(ObserverContext<MasterCoprocessorEnvironment> env,
543         final HRegionInfo regionInfo) throws IOException {
544       postRegionOfflineCalled = true;
545     }
546 
547     public boolean wasRegionOfflineCalled() {
548       return preRegionOfflineCalled && postRegionOfflineCalled;
549     }
550 
551     public boolean preRegionOfflineCalledOnly() {
552       return preRegionOfflineCalled && !postRegionOfflineCalled;
553     }
554 
555     @Override
556     public void preBalance(ObserverContext<MasterCoprocessorEnvironment> env)
557         throws IOException {
558       if (bypass) {
559         env.bypass();
560       }
561       preBalanceCalled = true;
562     }
563 
564     @Override
565     public void postBalance(ObserverContext<MasterCoprocessorEnvironment> env,
566         List<RegionPlan> plans) throws IOException {
567       postBalanceCalled = true;
568     }
569 
570     public boolean wasBalanceCalled() {
571       return preBalanceCalled && postBalanceCalled;
572     }
573 
574     public boolean preBalanceCalledOnly() {
575       return preBalanceCalled && !postBalanceCalled;
576     }
577 
578     @Override
579     public boolean preBalanceSwitch(ObserverContext<MasterCoprocessorEnvironment> env, boolean b)
580         throws IOException {
581       if (bypass) {
582         env.bypass();
583       }
584       preBalanceSwitchCalled = true;
585       return b;
586     }
587 
588     @Override
589     public void postBalanceSwitch(ObserverContext<MasterCoprocessorEnvironment> env,
590         boolean oldValue, boolean newValue) throws IOException {
591       postBalanceSwitchCalled = true;
592     }
593 
594     public boolean wasBalanceSwitchCalled() {
595       return preBalanceSwitchCalled && postBalanceSwitchCalled;
596     }
597 
598     public boolean preBalanceSwitchCalledOnly() {
599       return preBalanceSwitchCalled && !postBalanceSwitchCalled;
600     }
601 
602     @Override
603     public void preShutdown(ObserverContext<MasterCoprocessorEnvironment> env)
604         throws IOException {
605       preShutdownCalled = true;
606     }
607 
608     @Override
609     public void preStopMaster(ObserverContext<MasterCoprocessorEnvironment> env)
610         throws IOException {
611       preStopMasterCalled = true;
612     }
613 
614     @Override
615     public void preMasterInitialization(
616         ObserverContext<MasterCoprocessorEnvironment> ctx) throws IOException {
617       preMasterInitializationCalled = true;
618     }
619     
620     public boolean wasMasterInitializationCalled(){
621       return preMasterInitializationCalled;
622     }
623     
624     @Override
625     public void postStartMaster(ObserverContext<MasterCoprocessorEnvironment> ctx)
626         throws IOException {
627       postStartMasterCalled = true;
628     }
629 
630     public boolean wasStartMasterCalled() {
631       return postStartMasterCalled;
632     }
633 
634     @Override
635     public void start(CoprocessorEnvironment env) throws IOException {
636       startCalled = true;
637     }
638 
639     @Override
640     public void stop(CoprocessorEnvironment env) throws IOException {
641       stopCalled = true;
642     }
643 
644     public boolean wasStarted() { return startCalled; }
645 
646     public boolean wasStopped() { return stopCalled; }
647 
648     @Override
649     public void preSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
650         final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
651         throws IOException {
652       preSnapshotCalled = true;
653     }
654 
655     @Override
656     public void postSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
657         final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
658         throws IOException {
659       postSnapshotCalled = true;
660     }
661 
662     public boolean wasSnapshotCalled() {
663       return preSnapshotCalled && postSnapshotCalled;
664     }
665 
666     @Override
667     public void preCloneSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
668         final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
669         throws IOException {
670       preCloneSnapshotCalled = true;
671     }
672 
673     @Override
674     public void postCloneSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
675         final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
676         throws IOException {
677       postCloneSnapshotCalled = true;
678     }
679 
680     public boolean wasCloneSnapshotCalled() {
681       return preCloneSnapshotCalled && postCloneSnapshotCalled;
682     }
683 
684     @Override
685     public void preRestoreSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
686         final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
687         throws IOException {
688       preRestoreSnapshotCalled = true;
689     }
690 
691     @Override
692     public void postRestoreSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
693         final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
694         throws IOException {
695       postRestoreSnapshotCalled = true;
696     }
697 
698     public boolean wasRestoreSnapshotCalled() {
699       return preRestoreSnapshotCalled && postRestoreSnapshotCalled;
700     }
701 
702     @Override
703     public void preDeleteSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
704         final SnapshotDescription snapshot) throws IOException {
705       preDeleteSnapshotCalled = true;
706     }
707 
708     @Override
709     public void postDeleteSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
710         final SnapshotDescription snapshot) throws IOException {
711       postDeleteSnapshotCalled = true;
712     }
713 
714     public boolean wasDeleteSnapshotCalled() {
715       return preDeleteSnapshotCalled && postDeleteSnapshotCalled;
716     }
717 
718     @Override
719     public void preCreateTableHandler(
720         ObserverContext<MasterCoprocessorEnvironment> env,
721         HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
722       if (bypass) {
723         env.bypass();
724       }
725       preCreateTableHandlerCalled = true;
726     }
727 
728     @Override
729     public void postCreateTableHandler(
730         ObserverContext<MasterCoprocessorEnvironment> ctx,
731         HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
732       postCreateTableHandlerCalled = true;
733       tableCreationLatch.countDown();
734     }
735 
736     public boolean wasPreCreateTableHandlerCalled(){
737       return preCreateTableHandlerCalled;
738     }
739     public boolean wasCreateTableHandlerCalled() {
740       return preCreateTableHandlerCalled && postCreateTableHandlerCalled;
741     }
742 
743     public boolean wasCreateTableHandlerCalledOnly() {
744       return preCreateTableHandlerCalled && !postCreateTableHandlerCalled;
745     }
746 
747     @Override
748     public void preDeleteTableHandler(
749         ObserverContext<MasterCoprocessorEnvironment> env, TableName tableName)
750         throws IOException {
751       if (bypass) {
752         env.bypass();
753       }
754       preDeleteTableHandlerCalled = true;
755     }
756 
757     @Override
758     public void postDeleteTableHandler(
759         ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
760         throws IOException {
761       postDeleteTableHandlerCalled = true;
762     }
763 
764     public boolean wasDeleteTableHandlerCalled() {
765       return preDeleteTableHandlerCalled && postDeleteTableHandlerCalled;
766     }
767 
768     public boolean wasDeleteTableHandlerCalledOnly() {
769       return preDeleteTableHandlerCalled && !postDeleteTableHandlerCalled;
770     }
771     @Override
772     public void preModifyTableHandler(
773         ObserverContext<MasterCoprocessorEnvironment> env, TableName tableName,
774         HTableDescriptor htd) throws IOException {
775       if (bypass) {
776         env.bypass();
777       }
778       preModifyTableHandlerCalled = true;
779     }
780 
781     @Override
782     public void postModifyTableHandler(
783         ObserverContext<MasterCoprocessorEnvironment> env, TableName tableName,
784         HTableDescriptor htd) throws IOException {
785       postModifyTableHandlerCalled = true;
786     }
787 
788     public boolean wasModifyTableHandlerCalled() {
789       return preModifyColumnHandlerCalled && postModifyColumnHandlerCalled;
790     }
791 
792     public boolean wasModifyTableHandlerCalledOnly() {
793       return preModifyColumnHandlerCalled && !postModifyColumnHandlerCalled;
794     }
795 
796     @Override
797     public void preAddColumnHandler(
798         ObserverContext<MasterCoprocessorEnvironment> env, TableName tableName,
799         HColumnDescriptor column) throws IOException {
800       if (bypass) {
801         env.bypass();
802       }
803       preAddColumnHandlerCalled = true;
804     }
805 
806     @Override
807     public void postAddColumnHandler(
808         ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
809         HColumnDescriptor column) throws IOException {
810       postAddColumnHandlerCalled = true;
811     }
812     public boolean wasAddColumnHandlerCalled() {
813       return preAddColumnHandlerCalled && postAddColumnHandlerCalled;
814     }
815 
816     public boolean preAddColumnHandlerCalledOnly() {
817       return preAddColumnHandlerCalled && !postAddColumnHandlerCalled;
818     }
819 
820     @Override
821     public void preModifyColumnHandler(
822         ObserverContext<MasterCoprocessorEnvironment> env, TableName tableName,
823         HColumnDescriptor descriptor) throws IOException {
824       if (bypass) {
825         env.bypass();
826       }
827       preModifyColumnHandlerCalled = true;
828     }
829 
830     @Override
831     public void postModifyColumnHandler(
832         ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
833         HColumnDescriptor descriptor) throws IOException {
834       postModifyColumnHandlerCalled = true;
835     }
836 
837     public boolean wasModifyColumnHandlerCalled() {
838       return preModifyColumnHandlerCalled && postModifyColumnHandlerCalled;
839     }
840 
841     public boolean preModifyColumnHandlerCalledOnly() {
842       return preModifyColumnHandlerCalled && !postModifyColumnHandlerCalled;
843     }
844     @Override
845     public void preDeleteColumnHandler(
846         ObserverContext<MasterCoprocessorEnvironment> env, TableName tableName,
847         byte[] c) throws IOException {
848       if (bypass) {
849         env.bypass();
850       }
851       preDeleteColumnHandlerCalled = true;
852     }
853 
854     @Override
855     public void postDeleteColumnHandler(
856         ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
857         byte[] c) throws IOException {
858       postDeleteColumnHandlerCalled = true;
859     }
860 
861     public boolean wasDeleteColumnHandlerCalled() {
862       return preDeleteColumnHandlerCalled && postDeleteColumnHandlerCalled;
863     }
864 
865     public boolean preDeleteColumnHandlerCalledOnly() {
866       return preDeleteColumnHandlerCalled && !postDeleteColumnHandlerCalled;
867     }
868 
869     @Override
870     public void preEnableTableHandler(
871         ObserverContext<MasterCoprocessorEnvironment> env, TableName tableName)
872         throws IOException {
873       if (bypass) {
874         env.bypass();
875       }
876       preEnableTableHandlerCalled = true;
877     }
878 
879     @Override
880     public void postEnableTableHandler(
881         ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
882         throws IOException {
883       postEnableTableHandlerCalled = true;
884     }
885 
886     public boolean wasEnableTableHandlerCalled() {
887       return preEnableTableHandlerCalled && postEnableTableHandlerCalled;
888     }
889 
890     public boolean preEnableTableHandlerCalledOnly() {
891       return preEnableTableHandlerCalled && !postEnableTableHandlerCalled;
892     }
893 
894     @Override
895     public void preDisableTableHandler(
896         ObserverContext<MasterCoprocessorEnvironment> env, TableName tableName)
897         throws IOException {
898       if (bypass) {
899         env.bypass();
900       }
901       preDisableTableHandlerCalled = true;
902     }
903 
904     @Override
905     public void postDisableTableHandler(
906         ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
907         throws IOException {
908       postDisableTableHandlerCalled = true;
909     }
910 
911     public boolean wasDisableTableHandlerCalled() {
912       return preDisableTableHandlerCalled && postDisableTableHandlerCalled;
913     }
914 
915     public boolean preDisableTableHandlerCalledOnly() {
916       return preDisableTableHandlerCalled && !postDisableTableHandlerCalled;
917     }
918 
919     @Override
920     public void preGetTableDescriptors(ObserverContext<MasterCoprocessorEnvironment> ctx,
921         List<TableName> tableNamesList, List<HTableDescriptor> descriptors)
922         throws IOException {
923       preGetTableDescriptorsCalled = true;
924     }
925 
926     @Override
927     public void postGetTableDescriptors(ObserverContext<MasterCoprocessorEnvironment> ctx,
928         List<HTableDescriptor> descriptors) throws IOException {
929       postGetTableDescriptorsCalled = true;
930     }
931 
932     public boolean wasGetTableDescriptorsCalled() {
933       return preGetTableDescriptorsCalled && postGetTableDescriptorsCalled;
934     }
935   }
936 
937   private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
938   private static byte[] TEST_SNAPSHOT = Bytes.toBytes("observed_snapshot");
939   private static TableName TEST_TABLE =
940       TableName.valueOf("observed_table");
941   private static byte[] TEST_CLONE = Bytes.toBytes("observed_clone");
942   private static byte[] TEST_FAMILY = Bytes.toBytes("fam1");
943   private static byte[] TEST_FAMILY2 = Bytes.toBytes("fam2");
944   private static byte[] TEST_FAMILY3 = Bytes.toBytes("fam3");
945 
946   @BeforeClass
947   public static void setupBeforeClass() throws Exception {
948     Configuration conf = UTIL.getConfiguration();
949     conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
950         CPMasterObserver.class.getName());
951     conf.set("hbase.master.hfilecleaner.plugins",
952       "org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner," +
953       "org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner");
954     conf.set("hbase.master.logcleaner.plugins",
955       "org.apache.hadoop.hbase.master.snapshot.SnapshotLogCleaner");
956     // We need more than one data server on this test
957     UTIL.startMiniCluster(2);
958   }
959 
960   @AfterClass
961   public static void tearDownAfterClass() throws Exception {
962     UTIL.shutdownMiniCluster();
963   }
964 
965   @Test
966   public void testStarted() throws Exception {
967     MiniHBaseCluster cluster = UTIL.getHBaseCluster();
968 
969     HMaster master = cluster.getMaster();
970     assertTrue("Master should be active", master.isActiveMaster());
971     MasterCoprocessorHost host = master.getCoprocessorHost();
972     assertNotNull("CoprocessorHost should not be null", host);
973     CPMasterObserver cp = (CPMasterObserver)host.findCoprocessor(
974         CPMasterObserver.class.getName());
975     assertNotNull("CPMasterObserver coprocessor not found or not installed!", cp);
976 
977     // check basic lifecycle
978     assertTrue("MasterObserver should have been started", cp.wasStarted());
979     assertTrue("preMasterInitialization() hook should have been called",
980         cp.wasMasterInitializationCalled());
981     assertTrue("postStartMaster() hook should have been called",
982         cp.wasStartMasterCalled());
983   }
984 
985   @Test
986   public void testTableOperations() throws Exception {
987     MiniHBaseCluster cluster = UTIL.getHBaseCluster();
988 
989     HMaster master = cluster.getMaster();
990     MasterCoprocessorHost host = master.getCoprocessorHost();
991     CPMasterObserver cp = (CPMasterObserver)host.findCoprocessor(
992         CPMasterObserver.class.getName());
993     cp.enableBypass(true);
994     cp.resetStates();
995     assertFalse("No table created yet", cp.wasCreateTableCalled());
996 
997     // create a table
998     HTableDescriptor htd = new HTableDescriptor(TEST_TABLE);
999     htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
1000     HBaseAdmin admin = UTIL.getHBaseAdmin();
1001 
1002     tableCreationLatch = new CountDownLatch(1);
1003     admin.createTable(htd);
1004     // preCreateTable can't bypass default action.
1005     assertTrue("Test table should be created", cp.wasCreateTableCalled());
1006     tableCreationLatch.await();
1007     assertTrue("Table pre create handler called.", cp
1008         .wasPreCreateTableHandlerCalled());
1009     assertTrue("Table create handler should be called.",
1010         cp.wasCreateTableHandlerCalled());
1011 
1012     tableCreationLatch = new CountDownLatch(1);
1013     admin.disableTable(TEST_TABLE);
1014     assertTrue(admin.isTableDisabled(TEST_TABLE));
1015     // preDisableTable can't bypass default action.
1016     assertTrue("Coprocessor should have been called on table disable",
1017       cp.wasDisableTableCalled());
1018     assertTrue("Disable table handler should be called.",
1019         cp.wasDisableTableHandlerCalled());
1020 
1021     // enable
1022     assertFalse(cp.wasEnableTableCalled());
1023     admin.enableTable(TEST_TABLE);
1024     assertTrue(admin.isTableEnabled(TEST_TABLE));
1025     // preEnableTable can't bypass default action.
1026     assertTrue("Coprocessor should have been called on table enable",
1027       cp.wasEnableTableCalled());
1028     assertTrue("Enable table handler should be called.",
1029         cp.wasEnableTableHandlerCalled());
1030 
1031     admin.disableTable(TEST_TABLE);
1032     assertTrue(admin.isTableDisabled(TEST_TABLE));
1033 
1034     // modify table
1035     htd.setMaxFileSize(512 * 1024 * 1024);
1036     modifyTableSync(admin, TEST_TABLE, htd);
1037     // preModifyTable can't bypass default action.
1038     assertTrue("Test table should have been modified",
1039       cp.wasModifyTableCalled());
1040 
1041     // add a column family
1042     admin.addColumn(TEST_TABLE, new HColumnDescriptor(TEST_FAMILY2));
1043     assertTrue("New column family shouldn't have been added to test table",
1044       cp.preAddColumnCalledOnly());
1045 
1046     // modify a column family
1047     HColumnDescriptor hcd1 = new HColumnDescriptor(TEST_FAMILY2);
1048     hcd1.setMaxVersions(25);
1049     admin.modifyColumn(TEST_TABLE, hcd1);
1050     assertTrue("Second column family should be modified",
1051       cp.preModifyColumnCalledOnly());
1052 
1053     // delete table
1054     admin.deleteTable(TEST_TABLE);
1055     assertFalse("Test table should have been deleted",
1056         admin.tableExists(TEST_TABLE));
1057     // preDeleteTable can't bypass default action.
1058     assertTrue("Coprocessor should have been called on table delete",
1059         cp.wasDeleteTableCalled());
1060     assertTrue("Delete table handler should be called.",
1061         cp.wasDeleteTableHandlerCalled());
1062 
1063     // turn off bypass, run the tests again
1064     cp.enableBypass(false);
1065     cp.resetStates();
1066 
1067     admin.createTable(htd);
1068     assertTrue("Test table should be created", cp.wasCreateTableCalled());
1069     tableCreationLatch.await();
1070     assertTrue("Table pre create handler called.", cp
1071         .wasPreCreateTableHandlerCalled());
1072     assertTrue("Table create handler should be called.",
1073         cp.wasCreateTableHandlerCalled());
1074 
1075     // disable
1076     assertFalse(cp.wasDisableTableCalled());
1077     assertFalse(cp.wasDisableTableHandlerCalled());
1078     admin.disableTable(TEST_TABLE);
1079     assertTrue(admin.isTableDisabled(TEST_TABLE));
1080     assertTrue("Coprocessor should have been called on table disable",
1081       cp.wasDisableTableCalled());
1082     assertTrue("Disable table handler should be called.",
1083         cp.wasDisableTableHandlerCalled());
1084 
1085     // modify table
1086     htd.setMaxFileSize(512 * 1024 * 1024);
1087     modifyTableSync(admin, TEST_TABLE, htd);
1088     assertTrue("Test table should have been modified",
1089         cp.wasModifyTableCalled());
1090     // add a column family
1091     admin.addColumn(TEST_TABLE, new HColumnDescriptor(TEST_FAMILY2));
1092     assertTrue("New column family should have been added to test table",
1093         cp.wasAddColumnCalled());
1094     assertTrue("Add column handler should be called.",
1095         cp.wasAddColumnHandlerCalled());
1096 
1097     // modify a column family
1098     HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAMILY2);
1099     hcd.setMaxVersions(25);
1100     admin.modifyColumn(TEST_TABLE, hcd);
1101     assertTrue("Second column family should be modified",
1102         cp.wasModifyColumnCalled());
1103     assertTrue("Modify table handler should be called.",
1104         cp.wasModifyColumnHandlerCalled());
1105 
1106     // enable
1107     assertFalse(cp.wasEnableTableCalled());
1108     assertFalse(cp.wasEnableTableHandlerCalled());
1109     admin.enableTable(TEST_TABLE);
1110     assertTrue(admin.isTableEnabled(TEST_TABLE));
1111     assertTrue("Coprocessor should have been called on table enable",
1112         cp.wasEnableTableCalled());
1113     assertTrue("Enable table handler should be called.",
1114         cp.wasEnableTableHandlerCalled());
1115 
1116     // disable again
1117     admin.disableTable(TEST_TABLE);
1118     assertTrue(admin.isTableDisabled(TEST_TABLE));
1119 
1120     // delete column
1121     assertFalse("No column family deleted yet", cp.wasDeleteColumnCalled());
1122     assertFalse("Delete table column handler should not be called.",
1123         cp.wasDeleteColumnHandlerCalled());
1124     admin.deleteColumn(TEST_TABLE, TEST_FAMILY2);
1125     HTableDescriptor tableDesc = admin.getTableDescriptor(TEST_TABLE);
1126     assertNull("'"+Bytes.toString(TEST_FAMILY2)+"' should have been removed",
1127         tableDesc.getFamily(TEST_FAMILY2));
1128     assertTrue("Coprocessor should have been called on column delete",
1129         cp.wasDeleteColumnCalled());
1130     assertTrue("Delete table column handler should be called.",
1131         cp.wasDeleteColumnHandlerCalled());
1132 
1133     // delete table
1134     assertFalse("No table deleted yet", cp.wasDeleteTableCalled());
1135     assertFalse("Delete table handler should not be called.",
1136         cp.wasDeleteTableHandlerCalled());
1137     admin.deleteTable(TEST_TABLE);
1138     assertFalse("Test table should have been deleted",
1139         admin.tableExists(TEST_TABLE));
1140     assertTrue("Coprocessor should have been called on table delete",
1141         cp.wasDeleteTableCalled());
1142     assertTrue("Delete table handler should be called.",
1143         cp.wasDeleteTableHandlerCalled());
1144   }
1145 
1146   @Test
1147   public void testSnapshotOperations() throws Exception {
1148     MiniHBaseCluster cluster = UTIL.getHBaseCluster();
1149     HMaster master = cluster.getMaster();
1150     MasterCoprocessorHost host = master.getCoprocessorHost();
1151     CPMasterObserver cp = (CPMasterObserver)host.findCoprocessor(
1152         CPMasterObserver.class.getName());
1153     cp.resetStates();
1154 
1155     // create a table
1156     HTableDescriptor htd = new HTableDescriptor(TEST_TABLE);
1157     htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
1158     HBaseAdmin admin = UTIL.getHBaseAdmin();
1159 
1160     tableCreationLatch = new CountDownLatch(1);
1161     admin.createTable(htd);
1162     tableCreationLatch.await();
1163     tableCreationLatch = new CountDownLatch(1);
1164 
1165     admin.disableTable(TEST_TABLE);
1166     assertTrue(admin.isTableDisabled(TEST_TABLE));
1167 
1168     try {
1169       // Test snapshot operation
1170       assertFalse("Coprocessor should not have been called yet",
1171         cp.wasSnapshotCalled());
1172       admin.snapshot(TEST_SNAPSHOT, TEST_TABLE);
1173       assertTrue("Coprocessor should have been called on snapshot",
1174         cp.wasSnapshotCalled());
1175 
1176       // Test clone operation
1177       admin.cloneSnapshot(TEST_SNAPSHOT, TEST_CLONE);
1178       assertTrue("Coprocessor should have been called on snapshot clone",
1179         cp.wasCloneSnapshotCalled());
1180       assertFalse("Coprocessor restore should not have been called on snapshot clone",
1181         cp.wasRestoreSnapshotCalled());
1182       admin.disableTable(TEST_CLONE);
1183       assertTrue(admin.isTableDisabled(TEST_TABLE));
1184       admin.deleteTable(TEST_CLONE);
1185 
1186       // Test restore operation
1187       cp.resetStates();
1188       admin.restoreSnapshot(TEST_SNAPSHOT);
1189       assertTrue("Coprocessor should have been called on snapshot restore",
1190         cp.wasRestoreSnapshotCalled());
1191       assertFalse("Coprocessor clone should not have been called on snapshot restore",
1192         cp.wasCloneSnapshotCalled());
1193 
1194       admin.deleteSnapshot(TEST_SNAPSHOT);
1195       assertTrue("Coprocessor should have been called on snapshot delete",
1196         cp.wasDeleteSnapshotCalled());
1197     } finally {
1198       admin.deleteTable(TEST_TABLE);
1199     }
1200   }
1201 
1202   @Test
1203   public void testNamespaceOperations() throws Exception {
1204     MiniHBaseCluster cluster = UTIL.getHBaseCluster();
1205     String testNamespace = "observed_ns";
1206     HMaster master = cluster.getMaster();
1207     MasterCoprocessorHost host = master.getCoprocessorHost();
1208     CPMasterObserver cp = (CPMasterObserver)host.findCoprocessor(
1209         CPMasterObserver.class.getName());
1210 
1211     cp.enableBypass(false);
1212     cp.resetStates();
1213 
1214 
1215     // create a table
1216     HBaseAdmin admin = UTIL.getHBaseAdmin();
1217     admin.createNamespace(NamespaceDescriptor.create(testNamespace).build());
1218     assertTrue("Test namespace should be created", cp.wasCreateNamespaceCalled());
1219 
1220     assertNotNull(admin.getNamespaceDescriptor(testNamespace));
1221 
1222     // turn off bypass, run the tests again
1223     cp.enableBypass(true);
1224     cp.resetStates();
1225 
1226     admin.modifyNamespace(NamespaceDescriptor.create(testNamespace).build());
1227     assertTrue("Test namespace should not have been modified",
1228         cp.preModifyNamespaceCalledOnly());
1229 
1230     assertNotNull(admin.getNamespaceDescriptor(testNamespace));
1231 
1232     admin.deleteNamespace(testNamespace);
1233     assertTrue("Test namespace should not have been deleted", cp.preDeleteNamespaceCalledOnly());
1234 
1235     assertNotNull(admin.getNamespaceDescriptor(testNamespace));
1236 
1237     cp.enableBypass(false);
1238     cp.resetStates();
1239 
1240     // delete table
1241     admin.modifyNamespace(NamespaceDescriptor.create(testNamespace).build());
1242     assertTrue("Test namespace should have been modified", cp.wasModifyNamespaceCalled());
1243 
1244     admin.deleteNamespace(testNamespace);
1245     assertTrue("Test namespace should have been deleted", cp.wasDeleteNamespaceCalled());
1246 
1247     cp.enableBypass(true);
1248     cp.resetStates();
1249 
1250     admin.createNamespace(NamespaceDescriptor.create(testNamespace).build());
1251     assertTrue("Test namespace should not be created", cp.preCreateNamespaceCalledOnly());
1252   }
1253 
1254   private void modifyTableSync(HBaseAdmin admin, TableName tableName, HTableDescriptor htd)
1255       throws IOException {
1256     admin.modifyTable(tableName, htd);
1257     //wait until modify table finishes
1258     for (int t = 0; t < 100; t++) { //10 sec timeout
1259       HTableDescriptor td = admin.getTableDescriptor(htd.getTableName());
1260       if (td.equals(htd)) {
1261         break;
1262       }
1263       Threads.sleep(100);
1264     }
1265   }
1266 
1267   @Test
1268   public void testRegionTransitionOperations() throws Exception {
1269     MiniHBaseCluster cluster = UTIL.getHBaseCluster();
1270 
1271     HMaster master = cluster.getMaster();
1272     MasterCoprocessorHost host = master.getCoprocessorHost();
1273     CPMasterObserver cp = (CPMasterObserver)host.findCoprocessor(
1274         CPMasterObserver.class.getName());
1275     cp.enableBypass(false);
1276     cp.resetStates();
1277 
1278     HTable table = UTIL.createTable(TEST_TABLE, TEST_FAMILY);
1279 
1280     try {
1281       UTIL.createMultiRegions(table, TEST_FAMILY);
1282       UTIL.waitUntilAllRegionsAssigned(TEST_TABLE);
1283   
1284       NavigableMap<HRegionInfo, ServerName> regions = table.getRegionLocations();
1285       Map.Entry<HRegionInfo, ServerName> firstGoodPair = null;
1286       for (Map.Entry<HRegionInfo, ServerName> e: regions.entrySet()) {
1287         if (e.getValue() != null) {
1288           firstGoodPair = e;
1289           break;
1290         }
1291       }
1292       assertNotNull("Found a non-null entry", firstGoodPair);
1293       LOG.info("Found " + firstGoodPair.toString());
1294       // Try to force a move
1295       Collection<ServerName> servers = master.getClusterStatus().getServers();
1296       String destName = null;
1297       String serverNameForFirstRegion = firstGoodPair.getValue().toString();
1298       LOG.info("serverNameForFirstRegion=" + serverNameForFirstRegion);
1299       boolean found = false;
1300       // Find server that is NOT carrying the first region
1301       for (ServerName info : servers) {
1302         LOG.info("ServerName=" + info);
1303         if (!serverNameForFirstRegion.equals(info.getServerName())) {
1304           destName = info.toString();
1305           found = true;
1306           break;
1307         }
1308       }
1309       assertTrue("Found server", found);
1310       LOG.info("Found " + destName);
1311       master.moveRegion(null,RequestConverter.buildMoveRegionRequest(
1312         firstGoodPair.getKey().getEncodedNameAsBytes(),Bytes.toBytes(destName)));
1313       assertTrue("Coprocessor should have been called on region move",
1314         cp.wasMoveCalled());
1315   
1316       // make sure balancer is on
1317       master.balanceSwitch(true);
1318       assertTrue("Coprocessor should have been called on balance switch",
1319           cp.wasBalanceSwitchCalled());
1320   
1321       // turn balancer off
1322       master.balanceSwitch(false);
1323   
1324       // wait for assignments to finish, if any
1325       AssignmentManager mgr = master.getAssignmentManager();
1326       Collection<RegionState> transRegions =
1327         mgr.getRegionStates().getRegionsInTransition().values();
1328       for (RegionState state : transRegions) {
1329         mgr.getRegionStates().waitOnRegionToClearRegionsInTransition(state.getRegion());
1330       }
1331   
1332       // move half the open regions from RS 0 to RS 1
1333       HRegionServer rs = cluster.getRegionServer(0);
1334       byte[] destRS = Bytes.toBytes(cluster.getRegionServer(1).getServerName().toString());
1335       //Make sure no regions are in transition now
1336       waitForRITtoBeZero(master);
1337       List<HRegionInfo> openRegions = ProtobufUtil.getOnlineRegions(rs);
1338       int moveCnt = openRegions.size()/2;
1339       for (int i=0; i<moveCnt; i++) {
1340         HRegionInfo info = openRegions.get(i);
1341         if (!info.isMetaTable()) {
1342           master.moveRegion(null,RequestConverter.buildMoveRegionRequest(
1343             openRegions.get(i).getEncodedNameAsBytes(), destRS));
1344         }
1345       }
1346       //Make sure no regions are in transition now
1347       waitForRITtoBeZero(master);
1348       // now trigger a balance
1349       master.balanceSwitch(true);
1350       boolean balanceRun = master.balance();
1351       assertTrue("Coprocessor should be called on region rebalancing",
1352           cp.wasBalanceCalled());
1353     } finally {
1354       UTIL.deleteTable(TEST_TABLE);
1355     }
1356   }
1357 
1358   private void waitForRITtoBeZero(HMaster master) throws Exception {
1359     // wait for assignments to finish
1360     AssignmentManager mgr = master.getAssignmentManager();
1361     Collection<RegionState> transRegions =
1362       mgr.getRegionStates().getRegionsInTransition().values();
1363     for (RegionState state : transRegions) {
1364       mgr.getRegionStates().waitOnRegionToClearRegionsInTransition(state.getRegion());
1365     }
1366   }
1367 
1368   @Test
1369   public void testTableDescriptorsEnumeration() throws Exception {
1370     MiniHBaseCluster cluster = UTIL.getHBaseCluster();
1371 
1372     HMaster master = cluster.getMaster();
1373     MasterCoprocessorHost host = master.getCoprocessorHost();
1374     CPMasterObserver cp = (CPMasterObserver)host.findCoprocessor(
1375         CPMasterObserver.class.getName());
1376     cp.resetStates();
1377 
1378     GetTableDescriptorsRequest req =
1379         RequestConverter.buildGetTableDescriptorsRequest((List<TableName>)null);
1380     master.getTableDescriptors(null, req);
1381 
1382     assertTrue("Coprocessor should be called on table descriptors request",
1383       cp.wasGetTableDescriptorsCalled());
1384   }
1385 
1386 }