View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertNotNull;
24  import static org.junit.Assert.assertNull;
25  import static org.junit.Assert.assertTrue;
26  import static org.junit.Assert.fail;
27  
28  import java.io.IOException;
29  import java.util.Set;
30  import java.util.concurrent.Callable;
31  
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.fs.FileSystem;
35  import org.apache.hadoop.fs.Path;
36  import org.apache.hadoop.hbase.client.Get;
37  import org.apache.hadoop.hbase.client.HBaseAdmin;
38  import org.apache.hadoop.hbase.client.HTable;
39  import org.apache.hadoop.hbase.client.Put;
40  import org.apache.hadoop.hbase.constraint.ConstraintException;
41  import org.apache.hadoop.hbase.master.HMaster;
42  import org.apache.hadoop.hbase.util.Bytes;
43  import org.apache.hadoop.hbase.util.FSUtils;
44  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
45  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
46  import com.google.common.collect.Sets;
47  import org.junit.AfterClass;
48  import org.junit.Assert;
49  import org.junit.Before;
50  import org.junit.BeforeClass;
51  import org.junit.Ignore;
52  import org.junit.Test;
53  import org.junit.experimental.categories.Category;
54  
55  import java.io.IOException;
56  import java.util.Set;
57  
58  import static org.junit.Assert.*;
59  
60  @Category(MediumTests.class)
61  public class TestNamespace {
62    protected static final Log LOG = LogFactory.getLog(TestNamespace.class);
63    private static HMaster master;
64    protected final static int NUM_SLAVES_BASE = 4;
65    private static HBaseTestingUtility TEST_UTIL;
66    protected static HBaseAdmin admin;
67    protected static HBaseCluster cluster;
68    private static ZKNamespaceManager zkNamespaceManager;
69    private String prefix = "TestNamespace";
70  
71  
72    @BeforeClass
73    public static void setUp() throws Exception {
74      TEST_UTIL = new HBaseTestingUtility();
75      TEST_UTIL.getConfiguration().setInt("hbase.namespacejanitor.interval", 5000);
76      TEST_UTIL.startMiniCluster(NUM_SLAVES_BASE);
77      admin = TEST_UTIL.getHBaseAdmin();
78      cluster = TEST_UTIL.getHBaseCluster();
79      master = ((MiniHBaseCluster)cluster).getMaster();
80      zkNamespaceManager =
81          new ZKNamespaceManager(master.getZooKeeperWatcher());
82      zkNamespaceManager.start();
83      LOG.info("Done initializing cluster");
84    }
85  
86    @AfterClass
87    public static void tearDown() throws Exception {
88      TEST_UTIL.shutdownMiniCluster();
89    }
90  
91    @Before
92    public void beforeMethod() throws IOException {
93      for (HTableDescriptor desc : admin.listTables(prefix+".*")) {
94        admin.disableTable(desc.getTableName());
95        admin.deleteTable(desc.getTableName());
96      }
97      for (NamespaceDescriptor ns : admin.listNamespaceDescriptors()) {
98        if (ns.getName().startsWith(prefix)) {
99          admin.deleteNamespace(ns.getName());
100       }
101     }
102   }
103 
104   @Test
105   public void verifyReservedNS() throws IOException {
106     //verify existence of reserved namespaces
107     NamespaceDescriptor ns =
108         admin.getNamespaceDescriptor(NamespaceDescriptor.DEFAULT_NAMESPACE.getName());
109     assertNotNull(ns);
110     assertEquals(ns.getName(), NamespaceDescriptor.DEFAULT_NAMESPACE.getName());
111     assertNotNull(zkNamespaceManager.get(NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR));
112 
113     ns = admin.getNamespaceDescriptor(NamespaceDescriptor.SYSTEM_NAMESPACE.getName());
114     assertNotNull(ns);
115     assertEquals(ns.getName(), NamespaceDescriptor.SYSTEM_NAMESPACE.getName());
116     assertNotNull(zkNamespaceManager.get(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR));
117 
118     assertEquals(2, admin.listNamespaceDescriptors().length);
119 
120     //verify existence of system tables
121     Set<TableName> systemTables = Sets.newHashSet(
122         TableName.META_TABLE_NAME,
123         TableName.NAMESPACE_TABLE_NAME);
124     HTableDescriptor[] descs =
125         admin.listTableDescriptorsByNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE.getName());
126     assertEquals(systemTables.size(), descs.length);
127     for (HTableDescriptor desc : descs) {
128       assertTrue(systemTables.contains(desc.getTableName()));
129     }
130     //verify system tables aren't listed
131     assertEquals(0, admin.listTables().length);
132 
133     //Try creating default and system namespaces.
134     boolean exceptionCaught = false;
135     try {
136       admin.createNamespace(NamespaceDescriptor.DEFAULT_NAMESPACE);
137     } catch (IOException exp) {
138       LOG.warn(exp);
139       exceptionCaught = true;
140     } finally {
141       assertTrue(exceptionCaught);
142     }
143 
144     exceptionCaught = false;
145     try {
146       admin.createNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE);
147     } catch (IOException exp) {
148       LOG.warn(exp);
149       exceptionCaught = true;
150     } finally {
151       assertTrue(exceptionCaught);
152     }
153   }
154 
155   @Test
156   public void testDeleteReservedNS() throws Exception {
157     boolean exceptionCaught = false;
158     try {
159       admin.deleteNamespace(NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR);
160     } catch (IOException exp) {
161       LOG.warn(exp);
162       exceptionCaught = true;
163     } finally {
164       assertTrue(exceptionCaught);
165     }
166 
167     try {
168       admin.deleteNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR);
169     } catch (IOException exp) {
170       LOG.warn(exp);
171       exceptionCaught = true;
172     } finally {
173       assertTrue(exceptionCaught);
174     }
175   }
176 
177   @Test
178   public void createRemoveTest() throws Exception {
179     String testName = "createRemoveTest";
180     String nsName = prefix+"_"+testName;
181     LOG.info(testName);
182 
183     //create namespace and verify
184     admin.createNamespace(NamespaceDescriptor.create(nsName).build());
185     assertEquals(3, admin.listNamespaceDescriptors().length);
186     TEST_UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
187       @Override
188       public boolean evaluate() throws Exception {
189         return zkNamespaceManager.list().size() == 3;
190       }
191     });
192     assertNotNull(zkNamespaceManager.get(nsName));
193     //remove namespace and verify
194     admin.deleteNamespace(nsName);
195     assertEquals(2, admin.listNamespaceDescriptors().length);
196     assertEquals(2, zkNamespaceManager.list().size());
197     assertNull(zkNamespaceManager.get(nsName));
198   }
199 
200   @Test
201   public void createDoubleTest() throws IOException, InterruptedException {
202     String testName = "createDoubleTest";
203     String nsName = prefix+"_"+testName;
204     LOG.info(testName);
205 
206     byte[] tableName = Bytes.toBytes("my_table");
207     byte[] tableNameFoo = Bytes.toBytes(nsName+":my_table");
208     //create namespace and verify
209     admin.createNamespace(NamespaceDescriptor.create(nsName).build());
210     TEST_UTIL.createTable(tableName, Bytes.toBytes(nsName));
211     TEST_UTIL.createTable(tableNameFoo,Bytes.toBytes(nsName));
212     assertEquals(2, admin.listTables().length);
213     assertNotNull(admin
214         .getTableDescriptor(tableName));
215     assertNotNull(admin
216         .getTableDescriptor(tableNameFoo));
217     //remove namespace and verify
218     admin.disableTable(tableName);
219     admin.deleteTable(tableName);
220     assertEquals(1, admin.listTables().length);
221   }
222 
223   @Test
224   public void createTableTest() throws IOException, InterruptedException {
225     String testName = "createTableTest";
226     String nsName = prefix+"_"+testName;
227     LOG.info(testName);
228 
229     HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(nsName+":my_table"));
230     HColumnDescriptor colDesc = new HColumnDescriptor("my_cf");
231     desc.addFamily(colDesc);
232     try {
233       admin.createTable(desc);
234       fail("Expected no namespace exists exception");
235     } catch (NamespaceNotFoundException ex) {
236     }
237     //create table and in new namespace
238     admin.createNamespace(NamespaceDescriptor.create(nsName).build());
239     admin.createTable(desc);
240     TEST_UTIL.waitTableAvailable(desc.getTableName().getName(), 10000);
241     FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
242     assertTrue(fs.exists(
243         new Path(master.getMasterFileSystem().getRootDir(),
244             new Path(HConstants.BASE_NAMESPACE_DIR,
245                 new Path(nsName, desc.getTableName().getQualifierAsString())))));
246     assertEquals(1, admin.listTables().length);
247 
248     //verify non-empty namespace can't be removed
249     try {
250       admin.deleteNamespace(nsName);
251       fail("Expected non-empty namespace constraint exception");
252     } catch (Exception ex) {
253       LOG.info("Caught expected exception: " + ex);
254     }
255 
256     //sanity check try to write and read from table
257     HTable table = new HTable(TEST_UTIL.getConfiguration(), desc.getTableName());
258     Put p = new Put(Bytes.toBytes("row1"));
259     p.add(Bytes.toBytes("my_cf"),Bytes.toBytes("my_col"),Bytes.toBytes("value1"));
260     table.put(p);
261     //flush and read from disk to make sure directory changes are working
262     admin.flush(desc.getTableName().getName());
263     Get g = new Get(Bytes.toBytes("row1"));
264     assertTrue(table.exists(g));
265 
266     //normal case of removing namespace
267     TEST_UTIL.deleteTable(desc.getTableName());
268     admin.deleteNamespace(nsName);
269   }
270 
271   @Test
272   public void createTableInDefaultNamespace() throws Exception {
273     HTableDescriptor desc = new HTableDescriptor(TableName.valueOf("default_table"));
274     HColumnDescriptor colDesc = new HColumnDescriptor("cf1");
275     desc.addFamily(colDesc);
276     admin.createTable(desc);
277     assertTrue(admin.listTables().length == 1);
278     admin.disableTable(desc.getTableName());
279     admin.deleteTable(desc.getTableName());
280   }
281 
282   @Test
283   public void createTableInSystemNamespace() throws Exception {
284     String tableName = "hbase:createTableInSystemNamespace";
285     HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
286     HColumnDescriptor colDesc = new HColumnDescriptor("cf1");
287     desc.addFamily(colDesc);
288     admin.createTable(desc);
289     assertEquals(0, admin.listTables().length);
290     assertTrue(admin.tableExists(Bytes.toBytes(tableName)));
291     admin.disableTable(desc.getTableName());
292     admin.deleteTable(desc.getTableName());
293   }
294 
295   @Ignore @Test
296   public void testNamespaceJanitor() throws Exception {
297     FileSystem fs = TEST_UTIL.getTestFileSystem();
298 
299     int fsCount = fs.listStatus(new Path(FSUtils.getRootDir(TEST_UTIL.getConfiguration()),
300         HConstants.BASE_NAMESPACE_DIR)).length;
301     Path fakeNSPath =
302         FSUtils.getNamespaceDir(FSUtils.getRootDir(TEST_UTIL.getConfiguration()), "foo");
303     assertTrue(fs.mkdirs(fakeNSPath));
304 
305     String fakeZnode = ZKUtil.joinZNode(ZooKeeperWatcher.namespaceZNode, "foo");
306     int zkCount = ZKUtil.listChildrenNoWatch(TEST_UTIL.getZooKeeperWatcher(),
307         ZooKeeperWatcher.namespaceZNode).size();
308     ZKUtil.createWithParents(TEST_UTIL.getZooKeeperWatcher(), fakeZnode);
309     Thread.sleep(10000);
310 
311     //verify namespace count is the same and orphan is removed
312     assertFalse(fs.exists(fakeNSPath));
313     assertEquals(fsCount, fs.listStatus(new Path(FSUtils.getRootDir(TEST_UTIL.getConfiguration()),
314             HConstants.BASE_NAMESPACE_DIR)).length);
315 
316     assertEquals(-1, ZKUtil.checkExists(TEST_UTIL.getZooKeeperWatcher(), fakeZnode));
317     assertEquals(zkCount,
318         ZKUtil.listChildrenNoWatch(TEST_UTIL.getZooKeeperWatcher(),
319             ZooKeeperWatcher.namespaceZNode).size());
320   }
321 
322   @Test(timeout = 60000)
323   public void testNamespaceOperations() throws IOException {
324     admin.createNamespace(NamespaceDescriptor.create(prefix + "ns1").build());
325     admin.createNamespace(NamespaceDescriptor.create(prefix + "ns2").build());
326 
327     // create namespace that already exists
328     runWithExpectedException(new Callable<Void>() {
329       @Override
330       public Void call() throws Exception {
331         admin.createNamespace(NamespaceDescriptor.create(prefix + "ns1").build());
332         return null;
333       }
334     }, NamespaceExistException.class);
335 
336     // create a table in non-existing namespace
337     runWithExpectedException(new Callable<Void>() {
338       @Override
339       public Void call() throws Exception {
340         HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("non_existing_namespace", "table1"));
341         htd.addFamily(new HColumnDescriptor("family1"));
342         admin.createTable(htd);
343         return null;
344       }
345     }, NamespaceNotFoundException.class);
346 
347     // get descriptor for existing namespace
348     admin.getNamespaceDescriptor(prefix + "ns1");
349 
350     // get descriptor for non-existing namespace
351     runWithExpectedException(new Callable<NamespaceDescriptor>() {
352       @Override
353       public NamespaceDescriptor call() throws Exception {
354         return admin.getNamespaceDescriptor("non_existing_namespace");
355       }
356     }, NamespaceNotFoundException.class);
357 
358     // delete descriptor for existing namespace
359     admin.deleteNamespace(prefix + "ns2");
360 
361     // delete descriptor for non-existing namespace
362     runWithExpectedException(new Callable<Void>() {
363       @Override
364       public Void call() throws Exception {
365         admin.deleteNamespace("non_existing_namespace");
366         return null;
367       }
368     }, NamespaceNotFoundException.class);
369 
370     // modify namespace descriptor for existing namespace
371     NamespaceDescriptor ns1 = admin.getNamespaceDescriptor(prefix + "ns1");
372     ns1.setConfiguration("foo", "bar");
373     admin.modifyNamespace(ns1);
374 
375     // modify namespace descriptor for non-existing namespace
376     runWithExpectedException(new Callable<Void>() {
377       @Override
378       public Void call() throws Exception {
379         admin.modifyNamespace(NamespaceDescriptor.create("non_existing_namespace").build());
380         return null;
381       }
382     }, NamespaceNotFoundException.class);
383 
384     // get table descriptors for existing namespace
385     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(prefix + "ns1", "table1"));
386     htd.addFamily(new HColumnDescriptor("family1"));
387     admin.createTable(htd);
388     HTableDescriptor[] htds = admin.listTableDescriptorsByNamespace(prefix + "ns1");
389     assertNotNull("Should have not returned null", htds);
390     assertEquals("Should have returned non-empty array", 1, htds.length);
391 
392     // get table descriptors for non-existing namespace
393     runWithExpectedException(new Callable<Void>() {
394       @Override
395       public Void call() throws Exception {
396         admin.listTableDescriptorsByNamespace("non_existing_namespace");
397         return null;
398       }
399     }, NamespaceNotFoundException.class);
400 
401     // get table names for existing namespace
402     TableName[] tableNames = admin.listTableNamesByNamespace(prefix + "ns1");
403     assertNotNull("Should have not returned null", tableNames);
404     assertEquals("Should have returned non-empty array", 1, tableNames.length);
405 
406     // get table names for non-existing namespace
407     runWithExpectedException(new Callable<Void>() {
408       @Override
409       public Void call() throws Exception {
410         admin.listTableNamesByNamespace("non_existing_namespace");
411         return null;
412       }
413     }, NamespaceNotFoundException.class);
414 
415   }
416 
417   private static <V, E> void runWithExpectedException(Callable<V> callable, Class<E> exceptionClass) {
418     try {
419       callable.call();
420     } catch(Exception ex) {
421       Assert.assertEquals(exceptionClass, ex.getClass());
422       return;
423     }
424     fail("Should have thrown exception " + exceptionClass);
425   }
426 
427 }