View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
3    * agreements. See the NOTICE file distributed with this work for additional information regarding
4    * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
5    * "License"); you may not use this file except in compliance with the License. You may obtain a
6    * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable
7    * law or agreed to in writing, software distributed under the License is distributed on an "AS IS"
8    * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License
9    * for the specific language governing permissions and limitations under the License.
10   */
11  package org.apache.hadoop.hbase.namespace;
12  
13  import static org.junit.Assert.assertEquals;
14  import static org.junit.Assert.assertFalse;
15  import static org.junit.Assert.assertNotNull;
16  import static org.junit.Assert.assertNull;
17  import static org.junit.Assert.assertTrue;
18  import static org.junit.Assert.fail;
19  
20  import java.io.IOException;
21  import java.util.Collections;
22  import java.util.List;
23  import java.util.Set;
24  import java.util.concurrent.CountDownLatch;
25  
26  import org.apache.commons.lang.StringUtils;
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.fs.FileSystem;
31  import org.apache.hadoop.fs.Path;
32  import org.apache.hadoop.hbase.CategoryBasedTimeout;
33  import org.apache.hadoop.hbase.Coprocessor;
34  import org.apache.hadoop.hbase.CoprocessorEnvironment;
35  import org.apache.hadoop.hbase.DoNotRetryIOException;
36  import org.apache.hadoop.hbase.HBaseTestingUtility;
37  import org.apache.hadoop.hbase.HColumnDescriptor;
38  import org.apache.hadoop.hbase.HConstants;
39  import org.apache.hadoop.hbase.HRegionInfo;
40  import org.apache.hadoop.hbase.HTableDescriptor;
41  import org.apache.hadoop.hbase.MiniHBaseCluster;
42  import org.apache.hadoop.hbase.NamespaceDescriptor;
43  import org.apache.hadoop.hbase.TableName;
44  import org.apache.hadoop.hbase.Waiter;
45  import org.apache.hadoop.hbase.client.Connection;
46  import org.apache.hadoop.hbase.client.ConnectionFactory;
47  import org.apache.hadoop.hbase.client.HBaseAdmin;
48  import org.apache.hadoop.hbase.client.HTable;
49  import org.apache.hadoop.hbase.client.Mutation;
50  import org.apache.hadoop.hbase.client.RegionLocator;
51  import org.apache.hadoop.hbase.client.Table;
52  import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver;
53  import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
54  import org.apache.hadoop.hbase.coprocessor.BaseRegionServerObserver;
55  import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
56  import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
57  import org.apache.hadoop.hbase.coprocessor.ObserverContext;
58  import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
59  import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment;
60  import org.apache.hadoop.hbase.coprocessor.RegionServerObserver;
61  import org.apache.hadoop.hbase.mapreduce.TableInputFormatBase;
62  import org.apache.hadoop.hbase.master.HMaster;
63  import org.apache.hadoop.hbase.master.RegionState;
64  import org.apache.hadoop.hbase.master.RegionStates;
65  import org.apache.hadoop.hbase.master.TableNamespaceManager;
66  import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
67  import org.apache.hadoop.hbase.quotas.QuotaExceededException;
68  import org.apache.hadoop.hbase.quotas.QuotaUtil;
69  import org.apache.hadoop.hbase.regionserver.HRegion;
70  import org.apache.hadoop.hbase.regionserver.HRegionServer;
71  import org.apache.hadoop.hbase.regionserver.Region;
72  import org.apache.hadoop.hbase.regionserver.RegionServerCoprocessorHost;
73  import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
74  import org.apache.hadoop.hbase.testclassification.MediumTests;
75  import org.apache.hadoop.hbase.util.Bytes;
76  import org.apache.hadoop.hbase.util.FSUtils;
77  import org.apache.zookeeper.KeeperException;
78  import org.junit.After;
79  import org.junit.AfterClass;
80  import org.junit.BeforeClass;
81  import org.junit.Ignore;
82  import org.junit.Rule;
83  import org.junit.Test;
84  import org.junit.experimental.categories.Category;
85  import org.junit.rules.TestRule;
86  
87  import com.google.common.collect.Sets;
88  
89  @Category(MediumTests.class)
90  public class TestNamespaceAuditor {
91    @Rule public final TestRule timeout = CategoryBasedTimeout.builder().
92        withTimeout(this.getClass()).withLookingForStuckThread(true).build();
93    private static final Log LOG = LogFactory.getLog(TestNamespaceAuditor.class);
94    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
95    private static HBaseAdmin ADMIN;
96    private String prefix = "TestNamespaceAuditor";
97  
98    @BeforeClass
99    public static void before() throws Exception {
100     UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true);
101     setupOnce();
102   }
103 
104   public static void setupOnce() throws Exception, IOException {
105     Configuration conf = UTIL.getConfiguration();
106     conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, CustomObserver.class.getName());
107     conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MasterSyncObserver.class.getName());
108     conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
109     conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
110     conf.setClass("hbase.coprocessor.regionserver.classes", CPRegionServerObserver.class,
111       RegionServerObserver.class);
112     UTIL.startMiniCluster(1, 1);
113     waitForQuotaEnabled();
114     ADMIN = UTIL.getHBaseAdmin();
115   }
116 
117   @AfterClass
118   public static void tearDown() throws Exception {
119     UTIL.shutdownMiniCluster();
120   }
121 
122   @After
123   public void cleanup() throws Exception, KeeperException {
124     for (HTableDescriptor table : ADMIN.listTables()) {
125       ADMIN.disableTable(table.getTableName());
126       deleteTable(table.getTableName());
127     }
128     for (NamespaceDescriptor ns : ADMIN.listNamespaceDescriptors()) {
129       if (ns.getName().startsWith(prefix)) {
130         ADMIN.deleteNamespace(ns.getName());
131       }
132     }
133     assertTrue("Quota manager not enabled", UTIL.getHBaseCluster().getMaster()
134         .getMasterQuotaManager().isQuotaEnabled());
135   }
136 
137   @Test
138   public void testTableOperations() throws Exception {
139     String nsp = prefix + "_np2";
140     NamespaceDescriptor nspDesc =
141         NamespaceDescriptor.create(nsp)
142             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "5")
143             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
144     ADMIN.createNamespace(nspDesc);
145     assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
146     assertEquals(ADMIN.listNamespaceDescriptors().length, 3);
147     HTableDescriptor tableDescOne =
148         new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"));
149     HTableDescriptor tableDescTwo =
150         new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"));
151     HTableDescriptor tableDescThree =
152         new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table3"));
153     ADMIN.createTable(tableDescOne);
154     boolean constraintViolated = false;
155     try {
156       ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 5);
157     } catch (Exception exp) {
158       assertTrue(exp instanceof IOException);
159       constraintViolated = true;
160     } finally {
161       assertTrue("Constraint not violated for table " + tableDescTwo.getTableName(),
162         constraintViolated);
163     }
164     ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
165     NamespaceTableAndRegionInfo nspState = getQuotaManager().getState(nsp);
166     assertNotNull(nspState);
167     assertTrue(nspState.getTables().size() == 2);
168     assertTrue(nspState.getRegionCount() == 5);
169     constraintViolated = false;
170     try {
171       ADMIN.createTable(tableDescThree);
172     } catch (Exception exp) {
173       assertTrue(exp instanceof IOException);
174       constraintViolated = true;
175     } finally {
176       assertTrue("Constraint not violated for table " + tableDescThree.getTableName(),
177         constraintViolated);
178     }
179   }
180 
181   @Test
182   public void testValidQuotas() throws Exception {
183     boolean exceptionCaught = false;
184     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
185     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
186     NamespaceDescriptor nspDesc =
187         NamespaceDescriptor.create(prefix + "vq1")
188             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "hihdufh")
189             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
190     try {
191       ADMIN.createNamespace(nspDesc);
192     } catch (Exception exp) {
193       LOG.warn(exp);
194       exceptionCaught = true;
195     } finally {
196       assertTrue(exceptionCaught);
197       assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
198     }
199     nspDesc =
200         NamespaceDescriptor.create(prefix + "vq2")
201             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "-456")
202             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
203     try {
204       ADMIN.createNamespace(nspDesc);
205     } catch (Exception exp) {
206       LOG.warn(exp);
207       exceptionCaught = true;
208     } finally {
209       assertTrue(exceptionCaught);
210       assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
211     }
212     nspDesc =
213         NamespaceDescriptor.create(prefix + "vq3")
214             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10")
215             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "sciigd").build();
216     try {
217       ADMIN.createNamespace(nspDesc);
218     } catch (Exception exp) {
219       LOG.warn(exp);
220       exceptionCaught = true;
221     } finally {
222       assertTrue(exceptionCaught);
223       assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
224     }
225     nspDesc =
226         NamespaceDescriptor.create(prefix + "vq4")
227             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10")
228             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "-1500").build();
229     try {
230       ADMIN.createNamespace(nspDesc);
231     } catch (Exception exp) {
232       LOG.warn(exp);
233       exceptionCaught = true;
234     } finally {
235       assertTrue(exceptionCaught);
236       assertFalse(fs.exists(FSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
237     }
238   }
239 
240   @Test
241   public void testDeleteTable() throws Exception {
242     String namespace = prefix + "_dummy";
243     NamespaceDescriptor nspDesc =
244         NamespaceDescriptor.create(namespace)
245             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "100")
246             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "3").build();
247     ADMIN.createNamespace(nspDesc);
248     assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(namespace));
249     NamespaceTableAndRegionInfo stateInfo = getNamespaceState(nspDesc.getName());
250     assertNotNull("Namespace state found null for " + namespace, stateInfo);
251     HTableDescriptor tableDescOne =
252         new HTableDescriptor(TableName.valueOf(namespace + TableName.NAMESPACE_DELIM + "table1"));
253     HTableDescriptor tableDescTwo =
254         new HTableDescriptor(TableName.valueOf(namespace + TableName.NAMESPACE_DELIM + "table2"));
255     ADMIN.createTable(tableDescOne);
256     ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 5);
257     stateInfo = getNamespaceState(nspDesc.getName());
258     assertNotNull("Namespace state found to be null.", stateInfo);
259     assertEquals(2, stateInfo.getTables().size());
260     assertEquals(5, stateInfo.getRegionCountOfTable(tableDescTwo.getTableName()));
261     assertEquals(6, stateInfo.getRegionCount());
262     ADMIN.disableTable(tableDescOne.getTableName());
263     deleteTable(tableDescOne.getTableName());
264     stateInfo = getNamespaceState(nspDesc.getName());
265     assertNotNull("Namespace state found to be null.", stateInfo);
266     assertEquals(5, stateInfo.getRegionCount());
267     assertEquals(1, stateInfo.getTables().size());
268     ADMIN.disableTable(tableDescTwo.getTableName());
269     deleteTable(tableDescTwo.getTableName());
270     ADMIN.deleteNamespace(namespace);
271     stateInfo = getNamespaceState(namespace);
272     assertNull("Namespace state not found to be null.", stateInfo);
273   }
274 
275   public static class CPRegionServerObserver extends BaseRegionServerObserver {
276     private volatile boolean shouldFailMerge = false;
277 
278     public void failMerge(boolean fail) {
279       shouldFailMerge = fail;
280     }
281 
282     private boolean triggered = false;
283 
284     public synchronized void waitUtilTriggered() throws InterruptedException {
285       while (!triggered) {
286         wait();
287       }
288     }
289 
290     @Override
291     public synchronized void preMerge(ObserverContext<RegionServerCoprocessorEnvironment> ctx,
292         Region regionA, Region regionB) throws IOException {
293       triggered = true;
294       notifyAll();
295       if (shouldFailMerge) {
296         throw new IOException("fail merge");
297       }
298     }
299   }
300 
301   @Test
302   public void testRegionMerge() throws Exception {
303     String nsp1 = prefix + "_regiontest";
304     NamespaceDescriptor nspDesc =
305         NamespaceDescriptor.create(nsp1)
306             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "3")
307             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
308     ADMIN.createNamespace(nspDesc);
309     final TableName tableTwo = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table2");
310     byte[] columnFamily = Bytes.toBytes("info");
311     HTableDescriptor tableDescOne = new HTableDescriptor(tableTwo);
312     tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
313     final int initialRegions = 3;
314     ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("2000"), initialRegions);
315     try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
316         Table table = connection.getTable(tableTwo)) {
317       UTIL.loadNumericRows(table, Bytes.toBytes("info"), 1000, 1999);
318     }
319     ADMIN.flush(tableTwo);
320     List<HRegionInfo> hris = ADMIN.getTableRegions(tableTwo);
321     Collections.sort(hris);
322     // merge the two regions
323     final Set<String> encodedRegionNamesToMerge =
324         Sets.newHashSet(hris.get(0).getEncodedName(), hris.get(1).getEncodedName());
325     ADMIN.mergeRegions(hris.get(0).getEncodedNameAsBytes(), hris.get(1).getEncodedNameAsBytes(),
326       false);
327     waitForMergeToComplete(tableTwo, encodedRegionNamesToMerge);
328     hris = ADMIN.getTableRegions(tableTwo);
329     assertEquals(initialRegions - 1, hris.size());
330     Collections.sort(hris);
331 
332     final HRegionInfo hriToSplit = hris.get(1);
333     ADMIN.split(tableTwo, Bytes.toBytes("500"));
334 
335     UTIL.waitFor(10000, 100, new Waiter.ExplainingPredicate<Exception>() {
336 
337       @Override
338       public boolean evaluate() throws Exception {
339         RegionStates regionStates =
340             UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
341         for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
342           if (hri.getEncodedName().equals(hriToSplit.getEncodedName())) {
343             return false;
344           }
345           if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
346             return false;
347           }
348         }
349         return true;
350       }
351 
352       @Override
353       public String explainFailure() throws Exception {
354         RegionStates regionStates =
355             UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
356         for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
357           if (hri.getEncodedName().equals(hriToSplit.getEncodedName())) {
358             return hriToSplit + " which is expected to be split is still online";
359           }
360           if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
361             return hri + " is still in not opened";
362           }
363         }
364         return "Unknown";
365       }
366     });
367     hris = ADMIN.getTableRegions(tableTwo);
368     assertEquals(initialRegions, hris.size());
369     Collections.sort(hris);
370 
371     // fail region merge through Coprocessor hook
372     MiniHBaseCluster cluster = UTIL.getHBaseCluster();
373     HRegionServer regionServer = cluster.getRegionServer(0);
374     RegionServerCoprocessorHost cpHost = regionServer.getRegionServerCoprocessorHost();
375     Coprocessor coprocessor = cpHost.findCoprocessor(CPRegionServerObserver.class.getName());
376     CPRegionServerObserver regionServerObserver = (CPRegionServerObserver) coprocessor;
377     regionServerObserver.failMerge(true);
378     regionServerObserver.triggered = false;
379 
380     ADMIN.mergeRegions(hris.get(1).getEncodedNameAsBytes(), hris.get(2).getEncodedNameAsBytes(),
381       false);
382     regionServerObserver.waitUtilTriggered();
383     hris = ADMIN.getTableRegions(tableTwo);
384     assertEquals(initialRegions, hris.size());
385     Collections.sort(hris);
386     // verify that we cannot split
387     HRegionInfo hriToSplit2 = hris.get(1);
388     ADMIN.split(tableTwo,
389       TableInputFormatBase.getSplitKey(hriToSplit2.getStartKey(), hriToSplit2.getEndKey(), true));
390     waitForMergeToComplete(tableTwo, encodedRegionNamesToMerge);
391     assertEquals(initialRegions, ADMIN.getTableRegions(tableTwo).size());
392   }
393 
394   private void waitForMergeToComplete(final TableName tableTwo,
395       final Set<String> encodedRegionNamesToMerge) throws Exception {
396     UTIL.waitFor(10000, 100, new Waiter.ExplainingPredicate<Exception>() {
397 
398       @Override
399       public boolean evaluate() throws Exception {
400         RegionStates regionStates =
401             UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
402         for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
403           if (encodedRegionNamesToMerge.contains(hri.getEncodedName())) {
404             return false;
405           }
406           if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
407             return false;
408           }
409         }
410         return true;
411       }
412 
413       @Override
414       public String explainFailure() throws Exception {
415         RegionStates regionStates =
416             UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
417         for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
418           if (encodedRegionNamesToMerge.contains(hri.getEncodedName())) {
419             return hri + " which is expected to be merged is still online";
420           }
421           if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
422             return hri + " is still in not opened";
423           }
424         }
425         return "Unknown";
426       }
427     });
428   }
429 
430   @Ignore("Hangs on occasion waiting on countdown latch") @Test
431   public void testRegionOperations() throws Exception {
432     String nsp1 = prefix + "_regiontest";
433     NamespaceDescriptor nspDesc =
434         NamespaceDescriptor.create(nsp1)
435             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "2")
436             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
437     ADMIN.createNamespace(nspDesc);
438     boolean constraintViolated = false;
439     final TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
440     byte[] columnFamily = Bytes.toBytes("info");
441     HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
442     tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
443     NamespaceTableAndRegionInfo stateInfo;
444     try {
445       ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("1000"), 7);
446     } catch (Exception exp) {
447       assertTrue(exp instanceof DoNotRetryIOException);
448       LOG.info(exp);
449       constraintViolated = true;
450     } finally {
451       assertTrue(constraintViolated);
452     }
453     assertFalse(ADMIN.tableExists(tableOne));
454     // This call will pass.
455     ADMIN.createTable(tableDescOne);
456     Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
457     HTable htable = (HTable) connection.getTable(tableOne);
458     UTIL.loadNumericRows(htable, Bytes.toBytes("info"), 1, 1000);
459     ADMIN.flush(tableOne);
460     stateInfo = getNamespaceState(nsp1);
461     assertEquals(1, stateInfo.getTables().size());
462     assertEquals(1, stateInfo.getRegionCount());
463     restartMaster();
464     ADMIN.split(tableOne, Bytes.toBytes("500"));
465     HRegion actualRegion = UTIL.getHBaseCluster().getRegions(tableOne).get(0);
466     CustomObserver observer =
467         (CustomObserver) actualRegion.getCoprocessorHost().findCoprocessor(
468           CustomObserver.class.getName());
469     assertNotNull(observer);
470     observer.postSplit.await();
471     assertEquals(2, ADMIN.getTableRegions(tableOne).size());
472     actualRegion = UTIL.getHBaseCluster().getRegions(tableOne).get(0);
473     observer =
474         (CustomObserver) actualRegion.getCoprocessorHost().findCoprocessor(
475           CustomObserver.class.getName());
476     assertNotNull(observer);
477     ADMIN.split(
478       tableOne,
479       getSplitKey(actualRegion.getRegionInfo().getStartKey(), actualRegion.getRegionInfo()
480           .getEndKey()));
481     observer.postSplit.await();
482     // Make sure no regions have been added.
483     List<HRegionInfo> hris = ADMIN.getTableRegions(tableOne);
484     assertEquals(2, hris.size());
485     assertTrue("split completed", observer.preSplitBeforePONR.getCount() == 1);
486 
487     htable.close();
488   }
489 
490   /*
491    * Create a table and make sure that the table creation fails after adding this table entry into
492    * namespace quota cache. Now correct the failure and recreate the table with same name.
493    * HBASE-13394
494    */
495   @Test
496   public void testRecreateTableWithSameNameAfterFirstTimeFailure() throws Exception {
497     String nsp1 = prefix + "_testRecreateTable";
498     NamespaceDescriptor nspDesc =
499         NamespaceDescriptor.create(nsp1)
500             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20")
501             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1").build();
502     ADMIN.createNamespace(nspDesc);
503     final TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
504     byte[] columnFamily = Bytes.toBytes("info");
505     HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
506     tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
507     MasterSyncObserver.throwExceptionInPreCreateTableHandler = true;
508     try {
509       try {
510         ADMIN.createTable(tableDescOne);
511         fail("Table " + tableOne.toString() + "creation should fail.");
512       } catch (Exception exp) {
513         LOG.error(exp);
514       }
515       assertFalse(ADMIN.tableExists(tableOne));
516 
517       NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp1);
518       assertEquals("First table creation failed in namespace so number of tables in namespace "
519           + "should be 0.", 0, nstate.getTables().size());
520 
521       MasterSyncObserver.throwExceptionInPreCreateTableHandler = false;
522       try {
523         ADMIN.createTable(tableDescOne);
524       } catch (Exception e) {
525         fail("Table " + tableOne.toString() + "creation should succeed.");
526         LOG.error(e);
527       }
528       assertTrue(ADMIN.tableExists(tableOne));
529       nstate = getNamespaceState(nsp1);
530       assertEquals("First table was created successfully so table size in namespace should "
531           + "be one now.", 1, nstate.getTables().size());
532     } finally {
533       MasterSyncObserver.throwExceptionInPreCreateTableHandler = false;
534       if (ADMIN.tableExists(tableOne)) {
535         ADMIN.disableTable(tableOne);
536         deleteTable(tableOne);
537       }
538       ADMIN.deleteNamespace(nsp1);
539     }
540   }
541 
542   private NamespaceTableAndRegionInfo getNamespaceState(String namespace) throws KeeperException,
543       IOException {
544     return getQuotaManager().getState(namespace);
545   }
546 
547   byte[] getSplitKey(byte[] startKey, byte[] endKey) {
548     String skey = Bytes.toString(startKey);
549     int key;
550     if (StringUtils.isBlank(skey)) {
551       key = Integer.parseInt(Bytes.toString(endKey)) / 2;
552     } else {
553       key = (int) (Integer.parseInt(skey) * 1.5);
554     }
555     return Bytes.toBytes("" + key);
556   }
557 
558   public static class CustomObserver extends BaseRegionObserver {
559     volatile CountDownLatch postSplit;
560     volatile CountDownLatch preSplitBeforePONR;
561 
562     @Override
563     public void postCompleteSplit(ObserverContext<RegionCoprocessorEnvironment> ctx)
564         throws IOException {
565       postSplit.countDown();
566     }
567 
568     @Override
569     public void preSplitBeforePONR(ObserverContext<RegionCoprocessorEnvironment> ctx,
570         byte[] splitKey, List<Mutation> metaEntries) throws IOException {
571       preSplitBeforePONR.countDown();
572     }
573 
574     @Override
575     public void start(CoprocessorEnvironment e) throws IOException {
576       postSplit = new CountDownLatch(1);
577       preSplitBeforePONR = new CountDownLatch(1);
578     }
579   }
580 
581   @Test
582   public void testStatePreserve() throws Exception {
583     final String nsp1 = prefix + "_testStatePreserve";
584     NamespaceDescriptor nspDesc =
585         NamespaceDescriptor.create(nsp1)
586             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20")
587             .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "10").build();
588     ADMIN.createNamespace(nspDesc);
589     TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
590     TableName tableTwo = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table2");
591     TableName tableThree = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table3");
592     HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
593     HTableDescriptor tableDescTwo = new HTableDescriptor(tableTwo);
594     HTableDescriptor tableDescThree = new HTableDescriptor(tableThree);
595     ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("1000"), 3);
596     ADMIN.createTable(tableDescTwo, Bytes.toBytes("1"), Bytes.toBytes("1000"), 3);
597     ADMIN.createTable(tableDescThree, Bytes.toBytes("1"), Bytes.toBytes("1000"), 4);
598     ADMIN.disableTable(tableThree);
599     deleteTable(tableThree);
600     // wait for chore to complete
601     UTIL.waitFor(1000, new Waiter.Predicate<Exception>() {
602       @Override
603       public boolean evaluate() throws Exception {
604         return (getNamespaceState(nsp1).getTables().size() == 2);
605       }
606     });
607     NamespaceTableAndRegionInfo before = getNamespaceState(nsp1);
608     restartMaster();
609     NamespaceTableAndRegionInfo after = getNamespaceState(nsp1);
610     assertEquals("Expected: " + before.getTables() + " Found: " + after.getTables(), before
611         .getTables().size(), after.getTables().size());
612   }
613 
614   private static void waitForQuotaEnabled() throws Exception {
615     UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
616       @Override
617       public boolean evaluate() throws Exception {
618         HMaster master = UTIL.getHBaseCluster().getMaster();
619         if (master == null) {
620           return false;
621         }
622         MasterQuotaManager quotaManager = master.getMasterQuotaManager();
623         return quotaManager != null && quotaManager.isQuotaEnabled();
624       }
625     });
626   }
627 
628   private void restartMaster() throws Exception {
629     UTIL.getHBaseCluster().getMaster(0).stop("Stopping to start again");
630     UTIL.getHBaseCluster().waitOnMaster(0);
631     UTIL.getHBaseCluster().startMaster();
632     waitForQuotaEnabled();
633   }
634 
635   private NamespaceAuditor getQuotaManager() {
636     return UTIL.getHBaseCluster().getMaster().getMasterQuotaManager().getNamespaceQuotaManager();
637   }
638 
639   public static class MasterSyncObserver extends BaseMasterObserver {
640     volatile CountDownLatch tableDeletionLatch;
641     static boolean throwExceptionInPreCreateTableHandler;
642 
643     @Override
644     public void preDeleteTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
645         TableName tableName) throws IOException {
646       tableDeletionLatch = new CountDownLatch(1);
647     }
648 
649     @Override
650     public void postDeleteTableHandler(final ObserverContext<MasterCoprocessorEnvironment> ctx,
651         TableName tableName) throws IOException {
652       tableDeletionLatch.countDown();
653     }
654 
655     @Override
656     public void preCreateTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
657         HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
658       if (throwExceptionInPreCreateTableHandler) {
659         throw new IOException("Throw exception as it is demanded.");
660       }
661     }
662   }
663 
664   private void deleteTable(final TableName tableName) throws Exception {
665     // NOTE: We need a latch because admin is not sync,
666     // so the postOp coprocessor method may be called after the admin operation returned.
667     MasterSyncObserver observer =
668         (MasterSyncObserver) UTIL.getHBaseCluster().getMaster().getMasterCoprocessorHost()
669             .findCoprocessor(MasterSyncObserver.class.getName());
670     ADMIN.deleteTable(tableName);
671     observer.tableDeletionLatch.await();
672   }
673 
674   @Test(expected = QuotaExceededException.class)
675   public void testExceedTableQuotaInNamespace() throws Exception {
676     String nsp = prefix + "_testExceedTableQuotaInNamespace";
677     NamespaceDescriptor nspDesc =
678         NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1")
679             .build();
680     ADMIN.createNamespace(nspDesc);
681     assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
682     assertEquals(ADMIN.listNamespaceDescriptors().length, 3);
683     HTableDescriptor tableDescOne =
684         new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"));
685     HTableDescriptor tableDescTwo =
686         new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"));
687     ADMIN.createTable(tableDescOne);
688     ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
689   }
690   
691   @Test(expected = QuotaExceededException.class)
692   public void testCloneSnapshotQuotaExceed() throws Exception {
693     String nsp = prefix + "_testTableQuotaExceedWithCloneSnapshot";
694     NamespaceDescriptor nspDesc =
695         NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1")
696             .build();
697     ADMIN.createNamespace(nspDesc);
698     assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
699     TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
700     TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
701     HTableDescriptor tableDescOne = new HTableDescriptor(tableName);
702     ADMIN.createTable(tableDescOne);
703     String snapshot = "snapshot_testTableQuotaExceedWithCloneSnapshot";
704     ADMIN.snapshot(snapshot, tableName);
705     ADMIN.cloneSnapshot(snapshot, cloneTableName);
706     ADMIN.deleteSnapshot(snapshot);
707   }
708 
709   @Test
710   public void testCloneSnapshot() throws Exception {
711     String nsp = prefix + "_testCloneSnapshot";
712     NamespaceDescriptor nspDesc =
713         NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2")
714             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20").build();
715     ADMIN.createNamespace(nspDesc);
716     assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
717     TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
718     TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
719     HTableDescriptor tableDescOne = new HTableDescriptor(tableName);
720 
721     ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
722     String snapshot = "snapshot_testCloneSnapshot";
723     ADMIN.snapshot(snapshot, tableName);
724     ADMIN.cloneSnapshot(snapshot, cloneTableName);
725 
726     int tableLength;
727     try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(tableName)) {
728       tableLength = locator.getStartKeys().length;
729     }
730     assertEquals(tableName.getNameAsString() + " should have four regions.", 4, tableLength);
731 
732     try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(cloneTableName)) {
733       tableLength = locator.getStartKeys().length;
734     }
735     assertEquals(cloneTableName.getNameAsString() + " should have four regions.", 4, tableLength);
736 
737     NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
738     assertEquals("Total tables count should be 2.", 2, nstate.getTables().size());
739     assertEquals("Total regions count should be.", 8, nstate.getRegionCount());
740 
741     ADMIN.deleteSnapshot(snapshot);
742   }
743 
744   @Test
745   public void testRestoreSnapshot() throws Exception {
746     String nsp = prefix + "_testRestoreSnapshot";
747     NamespaceDescriptor nspDesc =
748         NamespaceDescriptor.create(nsp)
749             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build();
750     ADMIN.createNamespace(nspDesc);
751     assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
752     TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
753     HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
754     ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
755 
756     NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
757     assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
758 
759     String snapshot = "snapshot_testRestoreSnapshot";
760     ADMIN.snapshot(snapshot, tableName1);
761 
762     List<HRegionInfo> regions = ADMIN.getTableRegions(tableName1);
763     Collections.sort(regions);
764 
765     ADMIN.split(tableName1, Bytes.toBytes("JJJ"));
766     Thread.sleep(2000);
767     assertEquals("Total regions count should be 5.", 5, nstate.getRegionCount());
768 
769     ADMIN.disableTable(tableName1);
770     ADMIN.restoreSnapshot(snapshot);
771 
772     assertEquals("Total regions count should be 4 after restore.", 4, nstate.getRegionCount());
773 
774     ADMIN.enableTable(tableName1);
775     ADMIN.deleteSnapshot(snapshot);
776   }
777 
778   @Test
779   public void testRestoreSnapshotQuotaExceed() throws Exception {
780     String nsp = prefix + "_testRestoreSnapshotQuotaExceed";
781     NamespaceDescriptor nspDesc =
782         NamespaceDescriptor.create(nsp)
783             .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build();
784     ADMIN.createNamespace(nspDesc);
785     NamespaceDescriptor ndesc = ADMIN.getNamespaceDescriptor(nsp);
786     assertNotNull("Namespace descriptor found null.", ndesc);
787     TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
788     HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
789     ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
790 
791     NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
792     assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
793 
794     String snapshot = "snapshot_testRestoreSnapshotQuotaExceed";
795     // snapshot has 4 regions
796     ADMIN.snapshot(snapshot, tableName1);
797     // recreate table with 1 region and set max regions to 3 for namespace
798     ADMIN.disableTable(tableName1);
799     ADMIN.deleteTable(tableName1);
800     ADMIN.createTable(tableDescOne);
801     ndesc.setConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "3");
802     ADMIN.modifyNamespace(ndesc);
803 
804     ADMIN.disableTable(tableName1);
805     try {
806       ADMIN.restoreSnapshot(snapshot);
807       fail("Region quota is exceeded so QuotaExceededException should be thrown but HBaseAdmin"
808           + " wraps IOException into RestoreSnapshotException");
809     } catch (RestoreSnapshotException ignore) {
810       assertTrue(ignore.getCause() instanceof QuotaExceededException);
811     }
812     assertEquals(1, getNamespaceState(nsp).getRegionCount());
813     ADMIN.enableTable(tableName1);
814     ADMIN.deleteSnapshot(snapshot);
815   }
816 }