View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.client;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertTrue;
24  import static org.junit.Assert.fail;
25  
26  import java.io.IOException;
27  import java.util.ArrayList;
28  import java.util.HashMap;
29  import java.util.Iterator;
30  import java.util.List;
31  import java.util.Map;
32  import java.util.Set;
33  import java.util.concurrent.atomic.AtomicBoolean;
34  import java.util.concurrent.atomic.AtomicInteger;
35  
36  import org.apache.commons.logging.Log;
37  import org.apache.commons.logging.LogFactory;
38  import org.apache.hadoop.hbase.HBaseTestingUtility;
39  import org.apache.hadoop.hbase.HColumnDescriptor;
40  import org.apache.hadoop.hbase.HConstants;
41  import org.apache.hadoop.hbase.HRegionInfo;
42  import org.apache.hadoop.hbase.HTableDescriptor;
43  import org.apache.hadoop.hbase.InvalidFamilyOperationException;
44  import org.apache.hadoop.hbase.testclassification.LargeTests;
45  import org.apache.hadoop.hbase.MasterNotRunningException;
46  import org.apache.hadoop.hbase.MetaTableAccessor;
47  import org.apache.hadoop.hbase.ServerName;
48  import org.apache.hadoop.hbase.TableName;
49  import org.apache.hadoop.hbase.TableNotDisabledException;
50  import org.apache.hadoop.hbase.TableNotEnabledException;
51  import org.apache.hadoop.hbase.TableNotFoundException;
52  import org.apache.hadoop.hbase.executor.EventHandler;
53  import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
54  import org.apache.hadoop.hbase.util.Bytes;
55  import org.apache.hadoop.hbase.util.FSUtils;
56  import org.apache.hadoop.hbase.zookeeper.ZKTableStateClientSideReader;
57  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
58  import org.apache.hadoop.hbase.exceptions.MergeRegionException;
59  import org.apache.hadoop.hbase.master.HMaster;
60  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
61  import org.apache.hadoop.hbase.protobuf.RequestConverter;
62  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
63  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DispatchMergingRegionsRequest;
64  import org.apache.hadoop.hbase.regionserver.HRegion;
65  import org.apache.hadoop.hbase.regionserver.Store;
66  import org.apache.hadoop.hbase.regionserver.StoreFile;
67  import org.apache.hadoop.hbase.util.Pair;
68  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
69  import org.junit.After;
70  import org.junit.AfterClass;
71  import org.junit.Before;
72  import org.junit.BeforeClass;
73  import org.junit.Test;
74  import org.junit.experimental.categories.Category;
75  
76  import com.google.protobuf.ServiceException;
77  
78  /**
79   * Class to test HBaseAdmin.
80   * Spins up the minicluster once at test start and then takes it down afterward.
81   * Add any testing of HBaseAdmin functionality here.
82   */
83  @Category(LargeTests.class)
84  public class TestAdmin1 {
85    private static final Log LOG = LogFactory.getLog(TestAdmin1.class);
86    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
87    private Admin admin;
88  
89    @BeforeClass
90    public static void setUpBeforeClass() throws Exception {
91      TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true);
92      TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
93      TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
94      TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
95      TEST_UTIL.getConfiguration().setBoolean(
96          "hbase.master.enabletable.roundrobin", true);
97      TEST_UTIL.startMiniCluster(3);
98    }
99  
100   @AfterClass
101   public static void tearDownAfterClass() throws Exception {
102     TEST_UTIL.shutdownMiniCluster();
103   }
104 
105   @Before
106   public void setUp() throws Exception {
107     this.admin = TEST_UTIL.getHBaseAdmin();
108   }
109 
110   @After
111   public void tearDown() throws Exception {
112     for (HTableDescriptor htd : this.admin.listTables()) {
113       TEST_UTIL.deleteTable(htd.getName());
114     }
115   }
116 
117   @Test (timeout=300000)
118   public void testSplitFlushCompactUnknownTable() throws InterruptedException {
119     final TableName unknowntable = TableName.valueOf("fubar");
120     Exception exception = null;
121     try {
122       this.admin.compact(unknowntable);
123     } catch (IOException e) {
124       exception = e;
125     }
126     assertTrue(exception instanceof TableNotFoundException);
127 
128     exception = null;
129     try {
130       this.admin.flush(unknowntable);
131     } catch (IOException e) {
132       exception = e;
133     }
134     assertTrue(exception instanceof TableNotFoundException);
135 
136     exception = null;
137     try {
138       this.admin.split(unknowntable);
139     } catch (IOException e) {
140       exception = e;
141     }
142     assertTrue(exception instanceof TableNotFoundException);
143   }
144 
145   @Test (timeout=300000)
146   public void testDeleteEditUnknownColumnFamilyAndOrTable() throws IOException {
147     // Test we get exception if we try to
148     final TableName nonexistentTable = TableName.valueOf("nonexistent");
149     final byte[] nonexistentColumn = Bytes.toBytes("nonexistent");
150     HColumnDescriptor nonexistentHcd = new HColumnDescriptor(nonexistentColumn);
151     Exception exception = null;
152     try {
153       this.admin.addColumn(nonexistentTable, nonexistentHcd);
154     } catch (IOException e) {
155       exception = e;
156     }
157     assertTrue(exception instanceof TableNotFoundException);
158 
159     exception = null;
160     try {
161       this.admin.deleteTable(nonexistentTable);
162     } catch (IOException e) {
163       exception = e;
164     }
165     assertTrue(exception instanceof TableNotFoundException);
166 
167     exception = null;
168     try {
169       this.admin.deleteColumn(nonexistentTable, nonexistentColumn);
170     } catch (IOException e) {
171       exception = e;
172     }
173     assertTrue(exception instanceof TableNotFoundException);
174 
175     exception = null;
176     try {
177       this.admin.disableTable(nonexistentTable);
178     } catch (IOException e) {
179       exception = e;
180     }
181     assertTrue(exception instanceof TableNotFoundException);
182 
183     exception = null;
184     try {
185       this.admin.enableTable(nonexistentTable);
186     } catch (IOException e) {
187       exception = e;
188     }
189     assertTrue(exception instanceof TableNotFoundException);
190 
191     exception = null;
192     try {
193       this.admin.modifyColumn(nonexistentTable, nonexistentHcd);
194     } catch (IOException e) {
195       exception = e;
196     }
197     assertTrue(exception instanceof TableNotFoundException);
198 
199     exception = null;
200     try {
201       HTableDescriptor htd = new HTableDescriptor(nonexistentTable);
202       htd.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
203       this.admin.modifyTable(htd.getTableName(), htd);
204     } catch (IOException e) {
205       exception = e;
206     }
207     assertTrue(exception instanceof TableNotFoundException);
208 
209     // Now make it so at least the table exists and then do tests against a
210     // nonexistent column family -- see if we get right exceptions.
211     final TableName tableName =
212       TableName.valueOf("testDeleteEditUnknownColumnFamilyAndOrTable" + System.currentTimeMillis());
213     HTableDescriptor htd = new HTableDescriptor(tableName);
214     htd.addFamily(new HColumnDescriptor("cf"));
215     this.admin.createTable(htd);
216     try {
217       exception = null;
218       try {
219         this.admin.deleteColumn(htd.getTableName(), nonexistentHcd.getName());
220       } catch (IOException e) {
221         exception = e;
222       }
223       assertTrue("found=" + exception.getClass().getName(),
224           exception instanceof InvalidFamilyOperationException);
225 
226       exception = null;
227       try {
228         this.admin.modifyColumn(htd.getTableName(), nonexistentHcd);
229       } catch (IOException e) {
230         exception = e;
231       }
232       assertTrue("found=" + exception.getClass().getName(),
233           exception instanceof InvalidFamilyOperationException);
234     } finally {
235       this.admin.disableTable(tableName);
236       this.admin.deleteTable(tableName);
237     }
238   }
239 
240   @Test (timeout=300000)
241   public void testDisableAndEnableTable() throws IOException {
242     final byte [] row = Bytes.toBytes("row");
243     final byte [] qualifier = Bytes.toBytes("qualifier");
244     final byte [] value = Bytes.toBytes("value");
245     final TableName table = TableName.valueOf("testDisableAndEnableTable");
246     Table ht = TEST_UTIL.createTable(table, HConstants.CATALOG_FAMILY);
247     Put put = new Put(row);
248     put.add(HConstants.CATALOG_FAMILY, qualifier, value);
249     ht.put(put);
250     Get get = new Get(row);
251     get.addColumn(HConstants.CATALOG_FAMILY, qualifier);
252     ht.get(get);
253 
254     this.admin.disableTable(ht.getName());
255     assertTrue("Table must be disabled.", TEST_UTIL.getHBaseCluster()
256         .getMaster().getAssignmentManager().getTableStateManager().isTableState(
257         ht.getName(), ZooKeeperProtos.Table.State.DISABLED));
258 
259     // Test that table is disabled
260     get = new Get(row);
261     get.addColumn(HConstants.CATALOG_FAMILY, qualifier);
262     boolean ok = false;
263     try {
264       ht.get(get);
265     } catch (TableNotEnabledException e) {
266       ok = true;
267     }
268     ok = false;
269     // verify that scan encounters correct exception
270     Scan scan = new Scan();
271     try {
272       ResultScanner scanner = ht.getScanner(scan);
273       Result res = null;
274       do {
275         res = scanner.next();
276       } while (res != null);
277     } catch (TableNotEnabledException e) {
278       ok = true;
279     }
280     assertTrue(ok);
281     this.admin.enableTable(table);
282     assertTrue("Table must be enabled.", TEST_UTIL.getHBaseCluster()
283         .getMaster().getAssignmentManager().getTableStateManager().isTableState(
284         ht.getName(), ZooKeeperProtos.Table.State.ENABLED));
285 
286     // Test that table is enabled
287     try {
288       ht.get(get);
289     } catch (RetriesExhaustedException e) {
290       ok = false;
291     }
292     assertTrue(ok);
293     ht.close();
294   }
295 
296   @Test (timeout=300000)
297   public void testDisableAndEnableTables() throws IOException {
298     final byte [] row = Bytes.toBytes("row");
299     final byte [] qualifier = Bytes.toBytes("qualifier");
300     final byte [] value = Bytes.toBytes("value");
301     final TableName table1 = TableName.valueOf("testDisableAndEnableTable1");
302     final TableName table2 = TableName.valueOf("testDisableAndEnableTable2");
303     Table ht1 = TEST_UTIL.createTable(table1, HConstants.CATALOG_FAMILY);
304     Table ht2 = TEST_UTIL.createTable(table2, HConstants.CATALOG_FAMILY);
305     Put put = new Put(row);
306     put.add(HConstants.CATALOG_FAMILY, qualifier, value);
307     ht1.put(put);
308     ht2.put(put);
309     Get get = new Get(row);
310     get.addColumn(HConstants.CATALOG_FAMILY, qualifier);
311     ht1.get(get);
312     ht2.get(get);
313 
314     this.admin.disableTables("testDisableAndEnableTable.*");
315 
316     // Test that tables are disabled
317     get = new Get(row);
318     get.addColumn(HConstants.CATALOG_FAMILY, qualifier);
319     boolean ok = false;
320     try {
321       ht1.get(get);
322       ht2.get(get);
323     } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
324       ok = true;
325     }
326 
327     assertTrue(ok);
328     this.admin.enableTables("testDisableAndEnableTable.*");
329 
330     // Test that tables are enabled
331     try {
332       ht1.get(get);
333     } catch (IOException e) {
334       ok = false;
335     }
336     try {
337       ht2.get(get);
338     } catch (IOException e) {
339       ok = false;
340     }
341     assertTrue(ok);
342 
343     ht1.close();
344     ht2.close();
345   }
346 
347   @Test (timeout=300000)
348   public void testCreateTable() throws IOException {
349     HTableDescriptor [] tables = admin.listTables();
350     int numTables = tables.length;
351     TEST_UTIL.createTable(TableName.valueOf("testCreateTable"), HConstants.CATALOG_FAMILY).close();
352     tables = this.admin.listTables();
353     assertEquals(numTables + 1, tables.length);
354     assertTrue("Table must be enabled.", TEST_UTIL.getHBaseCluster()
355         .getMaster().getAssignmentManager().getTableStateManager().isTableState(
356         TableName.valueOf("testCreateTable"), ZooKeeperProtos.Table.State.ENABLED));
357   }
358 
359   @Test (timeout=300000)
360   public void testTruncateTable() throws IOException {
361     testTruncateTable(TableName.valueOf("testTruncateTable"), false);
362   }
363 
364   @Test (timeout=300000)
365   public void testTruncateTablePreservingSplits() throws IOException {
366     testTruncateTable(TableName.valueOf("testTruncateTablePreservingSplits"), true);
367   }
368 
369   private void testTruncateTable(final TableName tableName, boolean preserveSplits)
370       throws IOException {
371     byte[][] splitKeys = new byte[2][];
372     splitKeys[0] = Bytes.toBytes(4);
373     splitKeys[1] = Bytes.toBytes(8);
374 
375     // Create & Fill the table
376     HTable table = TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY, splitKeys);
377     try {
378       TEST_UTIL.loadNumericRows(table, HConstants.CATALOG_FAMILY, 0, 10);
379       assertEquals(10, TEST_UTIL.countRows(table));
380     } finally {
381       table.close();
382     }
383     assertEquals(3, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());
384 
385     // Truncate & Verify
386     this.admin.disableTable(tableName);
387     this.admin.truncateTable(tableName, preserveSplits);
388     table = new HTable(TEST_UTIL.getConfiguration(), tableName);
389     try {
390       assertEquals(0, TEST_UTIL.countRows(table));
391     } finally {
392       table.close();
393     }
394     if (preserveSplits) {
395       assertEquals(3, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());
396     } else {
397       assertEquals(1, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());
398     }
399   }
400 
401   @Test (timeout=300000)
402   public void testGetTableDescriptor() throws IOException {
403     HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
404     HColumnDescriptor fam2 = new HColumnDescriptor("fam2");
405     HColumnDescriptor fam3 = new HColumnDescriptor("fam3");
406     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("myTestTable"));
407     htd.addFamily(fam1);
408     htd.addFamily(fam2);
409     htd.addFamily(fam3);
410     this.admin.createTable(htd);
411     Table table = new HTable(TEST_UTIL.getConfiguration(), htd.getTableName());
412     HTableDescriptor confirmedHtd = table.getTableDescriptor();
413     assertEquals(htd.compareTo(confirmedHtd), 0);
414     table.close();
415   }
416 
417   @Test (timeout=300000)
418   public void testCompactionTimestamps() throws Exception {
419     HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
420     TableName tableName = TableName.valueOf("testCompactionTimestampsTable");
421     HTableDescriptor htd = new HTableDescriptor(tableName);
422     htd.addFamily(fam1);
423     this.admin.createTable(htd);
424     HTable table = (HTable)TEST_UTIL.getConnection().getTable(htd.getTableName());
425     long ts = this.admin.getLastMajorCompactionTimestamp(tableName);
426     assertEquals(0, ts);
427     Put p = new Put(Bytes.toBytes("row1"));
428     p.add(Bytes.toBytes("fam1"), Bytes.toBytes("fam1"), Bytes.toBytes("fam1"));
429     table.put(p);
430     ts = this.admin.getLastMajorCompactionTimestamp(tableName);
431     // no files written -> no data
432     assertEquals(0, ts);
433 
434     this.admin.flush(tableName);
435     ts = this.admin.getLastMajorCompactionTimestamp(tableName);
436     // still 0, we flushed a file, but no major compaction happened
437     assertEquals(0, ts);
438 
439     byte[] regionName =
440         table.getRegionLocator().getAllRegionLocations().get(0).getRegionInfo().getRegionName();
441     long ts1 = this.admin.getLastMajorCompactionTimestampForRegion(regionName);
442     assertEquals(ts, ts1);
443     p = new Put(Bytes.toBytes("row2"));
444     p.add(Bytes.toBytes("fam1"), Bytes.toBytes("fam1"), Bytes.toBytes("fam1"));
445     table.put(p);
446     this.admin.flush(tableName);
447     ts = this.admin.getLastMajorCompactionTimestamp(tableName);
448     // make sure the region API returns the same value, as the old file is still around
449     assertEquals(ts1, ts);
450 
451     TEST_UTIL.compact(tableName, true);
452     table.put(p);
453     // forces a wait for the compaction
454     this.admin.flush(tableName);
455     ts = this.admin.getLastMajorCompactionTimestamp(tableName);
456     // after a compaction our earliest timestamp will have progressed forward
457     assertTrue(ts > ts1);
458 
459     // region api still the same
460     ts1 = this.admin.getLastMajorCompactionTimestampForRegion(regionName);
461     assertEquals(ts, ts1);
462     table.put(p);
463     this.admin.flush(tableName);
464     ts = this.admin.getLastMajorCompactionTimestamp(tableName);
465     assertEquals(ts, ts1);
466     table.close();
467   }
468 
469   @Test (timeout=300000)
470   public void testHColumnValidName() {
471        boolean exceptionThrown;
472        try {
473          new HColumnDescriptor("\\test\\abc");
474        } catch(IllegalArgumentException iae) {
475            exceptionThrown = true;
476            assertTrue(exceptionThrown);
477        }
478    }
479 
480   /**
481    * Verify schema modification takes.
482    * @throws IOException
483    * @throws InterruptedException
484    */
485   @Test (timeout=300000)
486   public void testOnlineChangeTableSchema() throws IOException, InterruptedException {
487     final TableName tableName =
488         TableName.valueOf("changeTableSchemaOnline");
489     TEST_UTIL.getMiniHBaseCluster().getMaster().getConfiguration().setBoolean(
490         "hbase.online.schema.update.enable", true);
491     HTableDescriptor [] tables = admin.listTables();
492     int numTables = tables.length;
493     TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close();
494     tables = this.admin.listTables();
495     assertEquals(numTables + 1, tables.length);
496 
497     // FIRST, do htabledescriptor changes.
498     HTableDescriptor htd = this.admin.getTableDescriptor(tableName);
499     // Make a copy and assert copy is good.
500     HTableDescriptor copy = new HTableDescriptor(htd);
501     assertTrue(htd.equals(copy));
502     // Now amend the copy. Introduce differences.
503     long newFlushSize = htd.getMemStoreFlushSize() / 2;
504     if (newFlushSize <=0) {
505       newFlushSize = HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE / 2;
506     }
507     copy.setMemStoreFlushSize(newFlushSize);
508     final String key = "anyoldkey";
509     assertTrue(htd.getValue(key) == null);
510     copy.setValue(key, key);
511     boolean expectedException = false;
512     try {
513       admin.modifyTable(tableName, copy);
514     } catch (TableNotDisabledException re) {
515       expectedException = true;
516     }
517     assertFalse(expectedException);
518     HTableDescriptor modifiedHtd = this.admin.getTableDescriptor(tableName);
519     assertFalse(htd.equals(modifiedHtd));
520     assertTrue(copy.equals(modifiedHtd));
521     assertEquals(newFlushSize, modifiedHtd.getMemStoreFlushSize());
522     assertEquals(key, modifiedHtd.getValue(key));
523 
524     // Now work on column family changes.
525     int countOfFamilies = modifiedHtd.getFamilies().size();
526     assertTrue(countOfFamilies > 0);
527     HColumnDescriptor hcd = modifiedHtd.getFamilies().iterator().next();
528     int maxversions = hcd.getMaxVersions();
529     final int newMaxVersions = maxversions + 1;
530     hcd.setMaxVersions(newMaxVersions);
531     final byte [] hcdName = hcd.getName();
532     expectedException = false;
533     try {
534       this.admin.modifyColumn(tableName, hcd);
535     } catch (TableNotDisabledException re) {
536       expectedException = true;
537     }
538     assertFalse(expectedException);
539     modifiedHtd = this.admin.getTableDescriptor(tableName);
540     HColumnDescriptor modifiedHcd = modifiedHtd.getFamily(hcdName);
541     assertEquals(newMaxVersions, modifiedHcd.getMaxVersions());
542 
543     // Try adding a column
544     assertFalse(this.admin.isTableDisabled(tableName));
545     final String xtracolName = "xtracol";
546     HColumnDescriptor xtracol = new HColumnDescriptor(xtracolName);
547     xtracol.setValue(xtracolName, xtracolName);
548     expectedException = false;
549     try {
550       this.admin.addColumn(tableName, xtracol);
551     } catch (TableNotDisabledException re) {
552       expectedException = true;
553     }
554     // Add column should work even if the table is enabled
555     assertFalse(expectedException);
556     modifiedHtd = this.admin.getTableDescriptor(tableName);
557     hcd = modifiedHtd.getFamily(xtracol.getName());
558     assertTrue(hcd != null);
559     assertTrue(hcd.getValue(xtracolName).equals(xtracolName));
560 
561     // Delete the just-added column.
562     this.admin.deleteColumn(tableName, xtracol.getName());
563     modifiedHtd = this.admin.getTableDescriptor(tableName);
564     hcd = modifiedHtd.getFamily(xtracol.getName());
565     assertTrue(hcd == null);
566 
567     // Delete the table
568     this.admin.disableTable(tableName);
569     this.admin.deleteTable(tableName);
570     this.admin.listTables();
571     assertFalse(this.admin.tableExists(tableName));
572   }
573 
574   @Test (timeout=300000)
575   public void testShouldFailOnlineSchemaUpdateIfOnlineSchemaIsNotEnabled()
576       throws Exception {
577     final TableName tableName = TableName.valueOf("changeTableSchemaOnlineFailure");
578     TEST_UTIL.getMiniHBaseCluster().getMaster().getConfiguration().setBoolean(
579         "hbase.online.schema.update.enable", false);
580     HTableDescriptor[] tables = admin.listTables();
581     int numTables = tables.length;
582     TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close();
583     tables = this.admin.listTables();
584     assertEquals(numTables + 1, tables.length);
585 
586     // FIRST, do htabledescriptor changes.
587     HTableDescriptor htd = this.admin.getTableDescriptor(tableName);
588     // Make a copy and assert copy is good.
589     HTableDescriptor copy = new HTableDescriptor(htd);
590     assertTrue(htd.equals(copy));
591     // Now amend the copy. Introduce differences.
592     long newFlushSize = htd.getMemStoreFlushSize() / 2;
593     if (newFlushSize <=0) {
594       newFlushSize = HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE / 2;
595     }
596     copy.setMemStoreFlushSize(newFlushSize);
597     final String key = "anyoldkey";
598     assertTrue(htd.getValue(key) == null);
599     copy.setValue(key, key);
600     boolean expectedException = false;
601     try {
602       admin.modifyTable(tableName, copy);
603     } catch (TableNotDisabledException re) {
604       expectedException = true;
605     }
606     assertTrue("Online schema update should not happen.", expectedException);
607 
608     // Reset the value for the other tests
609     TEST_UTIL.getMiniHBaseCluster().getMaster().getConfiguration().setBoolean(
610         "hbase.online.schema.update.enable", true);
611   }
612 
613   /**
614    * Listens for when an event is done in Master.
615    */
616   static class DoneListener implements EventHandler.EventHandlerListener {
617     private final AtomicBoolean done;
618 
619     DoneListener(final AtomicBoolean done) {
620       super();
621       this.done = done;
622     }
623 
624     @Override
625     public void afterProcess(EventHandler event) {
626       this.done.set(true);
627       synchronized (this.done) {
628         // Wake anyone waiting on this value to change.
629         this.done.notifyAll();
630       }
631     }
632 
633     @Override
634     public void beforeProcess(EventHandler event) {
635       // continue
636     }
637   }
638 
639   @SuppressWarnings("deprecation")
640   protected void verifyRoundRobinDistribution(HTable ht, int expectedRegions) throws IOException {
641     int numRS = ht.getConnection().getCurrentNrHRS();
642     Map<HRegionInfo, ServerName> regions = ht.getRegionLocations();
643     Map<ServerName, List<HRegionInfo>> server2Regions = new HashMap<ServerName, List<HRegionInfo>>();
644     for (Map.Entry<HRegionInfo, ServerName> entry : regions.entrySet()) {
645       ServerName server = entry.getValue();
646       List<HRegionInfo> regs = server2Regions.get(server);
647       if (regs == null) {
648         regs = new ArrayList<HRegionInfo>();
649         server2Regions.put(server, regs);
650       }
651       regs.add(entry.getKey());
652     }
653     float average = (float) expectedRegions/numRS;
654     int min = (int)Math.floor(average);
655     int max = (int)Math.ceil(average);
656     for (List<HRegionInfo> regionList : server2Regions.values()) {
657       assertTrue(regionList.size() == min || regionList.size() == max);
658     }
659   }
660 
661   @Test (timeout=300000)
662   public void testCreateTableNumberOfRegions() throws IOException, InterruptedException {
663     TableName tableName = TableName.valueOf("testCreateTableNumberOfRegions");
664     HTableDescriptor desc = new HTableDescriptor(tableName);
665     desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
666     admin.createTable(desc);
667     HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
668     Map<HRegionInfo, ServerName> regions = ht.getRegionLocations();
669     assertEquals("Table should have only 1 region", 1, regions.size());
670     ht.close();
671 
672     TableName TABLE_2 = TableName.valueOf(tableName.getNameAsString() + "_2");
673     desc = new HTableDescriptor(TABLE_2);
674     desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
675     admin.createTable(desc, new byte[][]{new byte[]{42}});
676     HTable ht2 = new HTable(TEST_UTIL.getConfiguration(), TABLE_2);
677     regions = ht2.getRegionLocations();
678     assertEquals("Table should have only 2 region", 2, regions.size());
679     ht2.close();
680 
681     TableName TABLE_3 = TableName.valueOf(tableName.getNameAsString() + "_3");
682     desc = new HTableDescriptor(TABLE_3);
683     desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
684     admin.createTable(desc, "a".getBytes(), "z".getBytes(), 3);
685     HTable ht3 = new HTable(TEST_UTIL.getConfiguration(), TABLE_3);
686     regions = ht3.getRegionLocations();
687     assertEquals("Table should have only 3 region", 3, regions.size());
688     ht3.close();
689 
690     TableName TABLE_4 = TableName.valueOf(tableName.getNameAsString() + "_4");
691     desc = new HTableDescriptor(TABLE_4);
692     desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
693     try {
694       admin.createTable(desc, "a".getBytes(), "z".getBytes(), 2);
695       fail("Should not be able to create a table with only 2 regions using this API.");
696     } catch (IllegalArgumentException eae) {
697     // Expected
698     }
699 
700     TableName TABLE_5 = TableName.valueOf(tableName.getNameAsString() + "_5");
701     desc = new HTableDescriptor(TABLE_5);
702     desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
703     admin.createTable(desc, new byte[] {1}, new byte[] {127}, 16);
704     HTable ht5 = new HTable(TEST_UTIL.getConfiguration(), TABLE_5);
705     regions = ht5.getRegionLocations();
706     assertEquals("Table should have 16 region", 16, regions.size());
707     ht5.close();
708   }
709 
710   @Test (timeout=300000)
711   public void testCreateTableWithRegions() throws IOException, InterruptedException {
712 
713     TableName tableName = TableName.valueOf("testCreateTableWithRegions");
714 
715     byte [][] splitKeys = {
716         new byte [] { 1, 1, 1 },
717         new byte [] { 2, 2, 2 },
718         new byte [] { 3, 3, 3 },
719         new byte [] { 4, 4, 4 },
720         new byte [] { 5, 5, 5 },
721         new byte [] { 6, 6, 6 },
722         new byte [] { 7, 7, 7 },
723         new byte [] { 8, 8, 8 },
724         new byte [] { 9, 9, 9 },
725     };
726     int expectedRegions = splitKeys.length + 1;
727 
728     HTableDescriptor desc = new HTableDescriptor(tableName);
729     desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
730     admin.createTable(desc, splitKeys);
731 
732     boolean tableAvailable = admin.isTableAvailable(tableName, splitKeys);
733     assertTrue("Table should be created with splitKyes + 1 rows in META", tableAvailable);
734 
735     HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
736     Map<HRegionInfo, ServerName> regions = ht.getRegionLocations();
737     assertEquals("Tried to create " + expectedRegions + " regions " +
738         "but only found " + regions.size(),
739         expectedRegions, regions.size());
740     System.err.println("Found " + regions.size() + " regions");
741 
742     Iterator<HRegionInfo> hris = regions.keySet().iterator();
743     HRegionInfo hri = hris.next();
744     assertTrue(hri.getStartKey() == null || hri.getStartKey().length == 0);
745     assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[0]));
746     hri = hris.next();
747     assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[0]));
748     assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[1]));
749     hri = hris.next();
750     assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[1]));
751     assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[2]));
752     hri = hris.next();
753     assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[2]));
754     assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[3]));
755     hri = hris.next();
756     assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[3]));
757     assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[4]));
758     hri = hris.next();
759     assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[4]));
760     assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[5]));
761     hri = hris.next();
762     assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[5]));
763     assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[6]));
764     hri = hris.next();
765     assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[6]));
766     assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[7]));
767     hri = hris.next();
768     assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[7]));
769     assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[8]));
770     hri = hris.next();
771     assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[8]));
772     assertTrue(hri.getEndKey() == null || hri.getEndKey().length == 0);
773 
774     verifyRoundRobinDistribution(ht, expectedRegions);
775     ht.close();
776 
777     // Now test using start/end with a number of regions
778 
779     // Use 80 bit numbers to make sure we aren't limited
780     byte [] startKey = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 };
781     byte [] endKey =   { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 };
782 
783     // Splitting into 10 regions, we expect (null,1) ... (9, null)
784     // with (1,2) (2,3) (3,4) (4,5) (5,6) (6,7) (7,8) (8,9) in the middle
785 
786     expectedRegions = 10;
787 
788     TableName TABLE_2 = TableName.valueOf(tableName.getNameAsString() + "_2");
789 
790     desc = new HTableDescriptor(TABLE_2);
791     desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
792     admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
793     admin.createTable(desc, startKey, endKey, expectedRegions);
794 
795     HTable ht2 = new HTable(TEST_UTIL.getConfiguration(), TABLE_2);
796     regions = ht2.getRegionLocations();
797     assertEquals("Tried to create " + expectedRegions + " regions " +
798         "but only found " + regions.size(),
799         expectedRegions, regions.size());
800     System.err.println("Found " + regions.size() + " regions");
801 
802     hris = regions.keySet().iterator();
803     hri = hris.next();
804     assertTrue(hri.getStartKey() == null || hri.getStartKey().length == 0);
805     assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {1,1,1,1,1,1,1,1,1,1}));
806     hri = hris.next();
807     assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {1,1,1,1,1,1,1,1,1,1}));
808     assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {2,2,2,2,2,2,2,2,2,2}));
809     hri = hris.next();
810     assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {2,2,2,2,2,2,2,2,2,2}));
811     assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {3,3,3,3,3,3,3,3,3,3}));
812     hri = hris.next();
813     assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {3,3,3,3,3,3,3,3,3,3}));
814     assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {4,4,4,4,4,4,4,4,4,4}));
815     hri = hris.next();
816     assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {4,4,4,4,4,4,4,4,4,4}));
817     assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {5,5,5,5,5,5,5,5,5,5}));
818     hri = hris.next();
819     assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {5,5,5,5,5,5,5,5,5,5}));
820     assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {6,6,6,6,6,6,6,6,6,6}));
821     hri = hris.next();
822     assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {6,6,6,6,6,6,6,6,6,6}));
823     assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {7,7,7,7,7,7,7,7,7,7}));
824     hri = hris.next();
825     assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {7,7,7,7,7,7,7,7,7,7}));
826     assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {8,8,8,8,8,8,8,8,8,8}));
827     hri = hris.next();
828     assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {8,8,8,8,8,8,8,8,8,8}));
829     assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {9,9,9,9,9,9,9,9,9,9}));
830     hri = hris.next();
831     assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {9,9,9,9,9,9,9,9,9,9}));
832     assertTrue(hri.getEndKey() == null || hri.getEndKey().length == 0);
833 
834     verifyRoundRobinDistribution(ht2, expectedRegions);
835     ht2.close();
836 
837     // Try once more with something that divides into something infinite
838 
839     startKey = new byte [] { 0, 0, 0, 0, 0, 0 };
840     endKey = new byte [] { 1, 0, 0, 0, 0, 0 };
841 
842     expectedRegions = 5;
843 
844     TableName TABLE_3 = TableName.valueOf(tableName.getNameAsString() + "_3");
845 
846     desc = new HTableDescriptor(TABLE_3);
847     desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
848     admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
849     admin.createTable(desc, startKey, endKey, expectedRegions);
850 
851 
852     HTable ht3 = new HTable(TEST_UTIL.getConfiguration(), TABLE_3);
853     regions = ht3.getRegionLocations();
854     assertEquals("Tried to create " + expectedRegions + " regions " +
855         "but only found " + regions.size(),
856         expectedRegions, regions.size());
857     System.err.println("Found " + regions.size() + " regions");
858 
859     verifyRoundRobinDistribution(ht3, expectedRegions);
860     ht3.close();
861 
862 
863     // Try an invalid case where there are duplicate split keys
864     splitKeys = new byte [][] {
865         new byte [] { 1, 1, 1 },
866         new byte [] { 2, 2, 2 },
867         new byte [] { 3, 3, 3 },
868         new byte [] { 2, 2, 2 }
869     };
870 
871     TableName TABLE_4 = TableName.valueOf(tableName.getNameAsString() + "_4");
872     desc = new HTableDescriptor(TABLE_4);
873     desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
874     Admin ladmin = new HBaseAdmin(TEST_UTIL.getConfiguration());
875     try {
876       ladmin.createTable(desc, splitKeys);
877       assertTrue("Should not be able to create this table because of " +
878           "duplicate split keys", false);
879     } catch(IllegalArgumentException iae) {
880       // Expected
881     }
882     ladmin.close();
883   }
884 
885   @Test (timeout=300000)
886   public void testTableAvailableWithRandomSplitKeys() throws Exception {
887     TableName tableName = TableName.valueOf("testTableAvailableWithRandomSplitKeys");
888     HTableDescriptor desc = new HTableDescriptor(tableName);
889     desc.addFamily(new HColumnDescriptor("col"));
890     byte[][] splitKeys = new byte[1][];
891     splitKeys = new byte [][] {
892         new byte [] { 1, 1, 1 },
893         new byte [] { 2, 2, 2 }
894     };
895     admin.createTable(desc);
896     boolean tableAvailable = admin.isTableAvailable(tableName, splitKeys);
897     assertFalse("Table should be created with 1 row in META", tableAvailable);
898   }
899 
900   @Test (timeout=300000)
901   public void testCreateTableWithOnlyEmptyStartRow() throws IOException {
902     byte[] tableName = Bytes.toBytes("testCreateTableWithOnlyEmptyStartRow");
903     byte[][] splitKeys = new byte[1][];
904     splitKeys[0] = HConstants.EMPTY_BYTE_ARRAY;
905     HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
906     desc.addFamily(new HColumnDescriptor("col"));
907     try {
908       admin.createTable(desc, splitKeys);
909       fail("Test case should fail as empty split key is passed.");
910     } catch (IllegalArgumentException e) {
911     }
912   }
913 
914   @Test (timeout=300000)
915   public void testCreateTableWithEmptyRowInTheSplitKeys() throws IOException{
916     byte[] tableName = Bytes.toBytes("testCreateTableWithEmptyRowInTheSplitKeys");
917     byte[][] splitKeys = new byte[3][];
918     splitKeys[0] = "region1".getBytes();
919     splitKeys[1] = HConstants.EMPTY_BYTE_ARRAY;
920     splitKeys[2] = "region2".getBytes();
921     HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
922     desc.addFamily(new HColumnDescriptor("col"));
923     try {
924       admin.createTable(desc, splitKeys);
925       fail("Test case should fail as empty split key is passed.");
926     } catch (IllegalArgumentException e) {
927       LOG.info("Expected ", e);
928     }
929   }
930 
931   @Test (timeout=120000)
932   public void testTableExist() throws IOException {
933     final TableName table = TableName.valueOf("testTableExist");
934     boolean exist;
935     exist = this.admin.tableExists(table);
936     assertEquals(false, exist);
937     TEST_UTIL.createTable(table, HConstants.CATALOG_FAMILY);
938     exist = this.admin.tableExists(table);
939     assertEquals(true, exist);
940   }
941 
942   /**
943    * Tests forcing split from client and having scanners successfully ride over split.
944    * @throws Exception
945    * @throws IOException
946    */
947   @Test (timeout=400000)
948   public void testForceSplit() throws Exception {
949     byte[][] familyNames = new byte[][] { Bytes.toBytes("cf") };
950     int[] rowCounts = new int[] { 6000 };
951     int numVersions = HColumnDescriptor.DEFAULT_VERSIONS;
952     int blockSize = 256;
953     splitTest(null, familyNames, rowCounts, numVersions, blockSize);
954 
955     byte[] splitKey = Bytes.toBytes(3500);
956     splitTest(splitKey, familyNames, rowCounts, numVersions, blockSize);
957   }
958 
959   /**
960    * Test retain assignment on enableTable.
961    *
962    * @throws IOException
963    */
964   @Test (timeout=300000)
965   public void testEnableTableRetainAssignment() throws IOException {
966     final TableName tableName = TableName.valueOf("testEnableTableAssignment");
967     byte[][] splitKeys = { new byte[] { 1, 1, 1 }, new byte[] { 2, 2, 2 },
968         new byte[] { 3, 3, 3 }, new byte[] { 4, 4, 4 }, new byte[] { 5, 5, 5 },
969         new byte[] { 6, 6, 6 }, new byte[] { 7, 7, 7 }, new byte[] { 8, 8, 8 },
970         new byte[] { 9, 9, 9 } };
971     int expectedRegions = splitKeys.length + 1;
972     HTableDescriptor desc = new HTableDescriptor(tableName);
973     desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
974     admin.createTable(desc, splitKeys);
975     HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
976     Map<HRegionInfo, ServerName> regions = ht.getRegionLocations();
977     assertEquals("Tried to create " + expectedRegions + " regions "
978         + "but only found " + regions.size(), expectedRegions, regions.size());
979     // Disable table.
980     admin.disableTable(tableName);
981     // Enable table, use retain assignment to assign regions.
982     admin.enableTable(tableName);
983     Map<HRegionInfo, ServerName> regions2 = ht.getRegionLocations();
984 
985     // Check the assignment.
986     assertEquals(regions.size(), regions2.size());
987     for (Map.Entry<HRegionInfo, ServerName> entry : regions.entrySet()) {
988       assertEquals(regions2.get(entry.getKey()), entry.getValue());
989     }
990   }
991 
992   /**
993    * Multi-family scenario. Tests forcing split from client and
994    * having scanners successfully ride over split.
995    * @throws Exception
996    * @throws IOException
997    */
998   @Test (timeout=800000)
999   public void testForceSplitMultiFamily() throws Exception {
1000     int numVersions = HColumnDescriptor.DEFAULT_VERSIONS;
1001 
1002     // use small HFile block size so that we can have lots of blocks in HFile
1003     // Otherwise, if there is only one block,
1004     // HFileBlockIndex.midKey()'s value == startKey
1005     int blockSize = 256;
1006     byte[][] familyNames = new byte[][] { Bytes.toBytes("cf1"),
1007       Bytes.toBytes("cf2") };
1008 
1009     // one of the column families isn't splittable
1010     int[] rowCounts = new int[] { 6000, 1 };
1011     splitTest(null, familyNames, rowCounts, numVersions, blockSize);
1012 
1013     rowCounts = new int[] { 1, 6000 };
1014     splitTest(null, familyNames, rowCounts, numVersions, blockSize);
1015 
1016     // one column family has much smaller data than the other
1017     // the split key should be based on the largest column family
1018     rowCounts = new int[] { 6000, 300 };
1019     splitTest(null, familyNames, rowCounts, numVersions, blockSize);
1020 
1021     rowCounts = new int[] { 300, 6000 };
1022     splitTest(null, familyNames, rowCounts, numVersions, blockSize);
1023 
1024   }
1025 
1026   void splitTest(byte[] splitPoint, byte[][] familyNames, int[] rowCounts,
1027     int numVersions, int blockSize) throws Exception {
1028     TableName tableName = TableName.valueOf("testForceSplit");
1029     StringBuilder sb = new StringBuilder();
1030     // Add tail to String so can see better in logs where a test is running.
1031     for (int i = 0; i < rowCounts.length; i++) {
1032       sb.append("_").append(Integer.toString(rowCounts[i]));
1033     }
1034     assertFalse(admin.tableExists(tableName));
1035     final HTable table = TEST_UTIL.createTable(tableName, familyNames,
1036       numVersions, blockSize);
1037 
1038     int rowCount = 0;
1039     byte[] q = new byte[0];
1040 
1041     // insert rows into column families. The number of rows that have values
1042     // in a specific column family is decided by rowCounts[familyIndex]
1043     for (int index = 0; index < familyNames.length; index++) {
1044       ArrayList<Put> puts = new ArrayList<Put>(rowCounts[index]);
1045       for (int i = 0; i < rowCounts[index]; i++) {
1046         byte[] k = Bytes.toBytes(i);
1047         Put put = new Put(k);
1048         put.add(familyNames[index], q, k);
1049         puts.add(put);
1050       }
1051       table.put(puts);
1052 
1053       if ( rowCount < rowCounts[index] ) {
1054         rowCount = rowCounts[index];
1055       }
1056     }
1057 
1058     // get the initial layout (should just be one region)
1059     Map<HRegionInfo, ServerName> m = table.getRegionLocations();
1060     LOG.info("Initial regions (" + m.size() + "): " + m);
1061     assertTrue(m.size() == 1);
1062 
1063     // Verify row count
1064     Scan scan = new Scan();
1065     ResultScanner scanner = table.getScanner(scan);
1066     int rows = 0;
1067     for(@SuppressWarnings("unused") Result result : scanner) {
1068       rows++;
1069     }
1070     scanner.close();
1071     assertEquals(rowCount, rows);
1072 
1073     // Have an outstanding scan going on to make sure we can scan over splits.
1074     scan = new Scan();
1075     scanner = table.getScanner(scan);
1076     // Scan first row so we are into first region before split happens.
1077     scanner.next();
1078 
1079     // Split the table
1080     this.admin.split(tableName, splitPoint);
1081 
1082     final AtomicInteger count = new AtomicInteger(0);
1083     Thread t = new Thread("CheckForSplit") {
1084       @Override
1085       public void run() {
1086         for (int i = 0; i < 45; i++) {
1087           try {
1088             sleep(1000);
1089           } catch (InterruptedException e) {
1090             continue;
1091           }
1092           // check again    table = new HTable(conf, tableName);
1093           Map<HRegionInfo, ServerName> regions = null;
1094           try {
1095             regions = table.getRegionLocations();
1096           } catch (IOException e) {
1097             e.printStackTrace();
1098           }
1099           if (regions == null) continue;
1100           count.set(regions.size());
1101           if (count.get() >= 2) {
1102             LOG.info("Found: " + regions);
1103             break;
1104           }
1105           LOG.debug("Cycle waiting on split");
1106         }
1107         LOG.debug("CheckForSplit thread exited, current region count: " + count.get());
1108       }
1109     };
1110     t.setPriority(Thread.NORM_PRIORITY - 2);
1111     t.start();
1112     t.join();
1113 
1114     // Verify row count
1115     rows = 1; // We counted one row above.
1116     for (@SuppressWarnings("unused") Result result : scanner) {
1117       rows++;
1118       if (rows > rowCount) {
1119         scanner.close();
1120         assertTrue("Scanned more than expected (" + rowCount + ")", false);
1121       }
1122     }
1123     scanner.close();
1124     assertEquals(rowCount, rows);
1125 
1126     Map<HRegionInfo, ServerName> regions = null;
1127     try {
1128       regions = table.getRegionLocations();
1129     } catch (IOException e) {
1130       e.printStackTrace();
1131     }
1132     assertEquals(2, regions.size());
1133     Set<HRegionInfo> hRegionInfos = regions.keySet();
1134     HRegionInfo[] r = hRegionInfos.toArray(new HRegionInfo[hRegionInfos.size()]);
1135     if (splitPoint != null) {
1136       // make sure the split point matches our explicit configuration
1137       assertEquals(Bytes.toString(splitPoint),
1138           Bytes.toString(r[0].getEndKey()));
1139       assertEquals(Bytes.toString(splitPoint),
1140           Bytes.toString(r[1].getStartKey()));
1141       LOG.debug("Properly split on " + Bytes.toString(splitPoint));
1142     } else {
1143       if (familyNames.length > 1) {
1144         int splitKey = Bytes.toInt(r[0].getEndKey());
1145         // check if splitKey is based on the largest column family
1146         // in terms of it store size
1147         int deltaForLargestFamily = Math.abs(rowCount/2 - splitKey);
1148         LOG.debug("SplitKey=" + splitKey + "&deltaForLargestFamily=" + deltaForLargestFamily +
1149           ", r=" + r[0]);
1150         for (int index = 0; index < familyNames.length; index++) {
1151           int delta = Math.abs(rowCounts[index]/2 - splitKey);
1152           if (delta < deltaForLargestFamily) {
1153             assertTrue("Delta " + delta + " for family " + index
1154               + " should be at least deltaForLargestFamily " + deltaForLargestFamily,
1155               false);
1156           }
1157         }
1158       }
1159     }
1160     TEST_UTIL.deleteTable(tableName);
1161     table.close();
1162   }
1163 
1164   @Test
1165   public void testSplitAndMergeWithReplicaTable() throws Exception {
1166     // The test tries to directly split replica regions and directly merge replica regions. These
1167     // are not allowed. The test validates that. Then the test does a valid split/merge of allowed
1168     // regions.
1169     // Set up a table with 3 regions and replication set to 3
1170     TableName tableName = TableName.valueOf("testSplitAndMergeWithReplicaTable");
1171     HTableDescriptor desc = new HTableDescriptor(tableName);
1172     desc.setRegionReplication(3);
1173     byte[] cf = "f".getBytes();
1174     HColumnDescriptor hcd = new HColumnDescriptor(cf);
1175     desc.addFamily(hcd);
1176     byte[][] splitRows = new byte[2][];
1177     splitRows[0] = new byte[]{(byte)'4'};
1178     splitRows[1] = new byte[]{(byte)'7'};
1179     TEST_UTIL.getHBaseAdmin().createTable(desc, splitRows);
1180     List<HRegion> oldRegions;
1181     do {
1182       oldRegions = TEST_UTIL.getHBaseCluster().getRegions(tableName);
1183       Thread.sleep(10);
1184     } while (oldRegions.size() != 9); //3 regions * 3 replicas
1185     // write some data to the table
1186     HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
1187     List<Put> puts = new ArrayList<Put>();
1188     byte[] qualifier = "c".getBytes();
1189     Put put = new Put(new byte[]{(byte)'1'});
1190     put.add(cf, qualifier, "100".getBytes());
1191     puts.add(put);
1192     put = new Put(new byte[]{(byte)'6'});
1193     put.add(cf, qualifier, "100".getBytes());
1194     puts.add(put);
1195     put = new Put(new byte[]{(byte)'8'});
1196     put.add(cf, qualifier, "100".getBytes());
1197     puts.add(put);
1198     ht.put(puts);
1199     ht.flushCommits();
1200     ht.close();
1201     List<Pair<HRegionInfo, ServerName>> regions =
1202         MetaTableAccessor.getTableRegionsAndLocations(TEST_UTIL.getZooKeeperWatcher(),
1203                           TEST_UTIL.getConnection(), tableName);
1204     boolean gotException = false;
1205     // the element at index 1 would be a replica (since the metareader gives us ordered
1206     // regions). Try splitting that region via the split API . Should fail
1207     try {
1208       TEST_UTIL.getHBaseAdmin().split(regions.get(1).getFirst().getRegionName());
1209     } catch (IllegalArgumentException ex) {
1210       gotException = true;
1211     }
1212     assertTrue(gotException);
1213     gotException = false;
1214     // the element at index 1 would be a replica (since the metareader gives us ordered
1215     // regions). Try splitting that region via a different split API (the difference is
1216     // this API goes direct to the regionserver skipping any checks in the admin). Should fail
1217     try {
1218       TEST_UTIL.getHBaseAdmin().split(regions.get(1).getSecond(), regions.get(1).getFirst(),
1219           new byte[]{(byte)'1'});
1220     } catch (IOException ex) {
1221       gotException = true;
1222     }
1223     assertTrue(gotException);
1224     gotException = false;
1225     // Try merging a replica with another. Should fail.
1226     try {
1227       TEST_UTIL.getHBaseAdmin().mergeRegions(regions.get(1).getFirst().getEncodedNameAsBytes(),
1228           regions.get(2).getFirst().getEncodedNameAsBytes(), true);
1229     } catch (IllegalArgumentException m) {
1230       gotException = true;
1231     }
1232     assertTrue(gotException);
1233     // Try going to the master directly (that will skip the check in admin)
1234     try {
1235       DispatchMergingRegionsRequest request = RequestConverter
1236           .buildDispatchMergingRegionsRequest(regions.get(1).getFirst().getEncodedNameAsBytes(),
1237               regions.get(2).getFirst().getEncodedNameAsBytes(), true);
1238       TEST_UTIL.getHBaseAdmin().getConnection().getMaster().dispatchMergingRegions(null, request);
1239     } catch (ServiceException m) {
1240       Throwable t = m.getCause();
1241       do {
1242         if (t instanceof MergeRegionException) {
1243           gotException = true;
1244           break;
1245         }
1246         t = t.getCause();
1247       } while (t != null);
1248     }
1249     assertTrue(gotException);
1250     gotException = false;
1251     // Try going to the regionservers directly
1252     // first move the region to the same regionserver
1253     if (!regions.get(2).getSecond().equals(regions.get(1).getSecond())) {
1254       moveRegionAndWait(regions.get(2).getFirst(), regions.get(1).getSecond());
1255     }
1256     try {
1257       AdminService.BlockingInterface admin = TEST_UTIL.getHBaseAdmin().getConnection()
1258           .getAdmin(regions.get(1).getSecond());
1259       ProtobufUtil.mergeRegions(null, admin, regions.get(1).getFirst(), regions.get(2).getFirst(), true);
1260     } catch (MergeRegionException mm) {
1261       gotException = true;
1262     }
1263     assertTrue(gotException);
1264   }
1265 
1266   private void moveRegionAndWait(HRegionInfo destRegion, ServerName destServer)
1267       throws InterruptedException, MasterNotRunningException,
1268       ZooKeeperConnectionException, IOException {
1269     HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
1270     TEST_UTIL.getHBaseAdmin().move(
1271         destRegion.getEncodedNameAsBytes(),
1272         Bytes.toBytes(destServer.getServerName()));
1273     while (true) {
1274       ServerName serverName = master.getAssignmentManager()
1275           .getRegionStates().getRegionServerOfRegion(destRegion);
1276       if (serverName != null && serverName.equals(destServer)) {
1277         TEST_UTIL.assertRegionOnServer(
1278             destRegion, serverName, 200);
1279         break;
1280       }
1281       Thread.sleep(10);
1282     }
1283   }
1284 
1285   /**
1286    * HADOOP-2156
1287    * @throws IOException
1288    */
1289   @SuppressWarnings("deprecation")
1290   @Test (expected=IllegalArgumentException.class, timeout=300000)
1291   public void testEmptyHTableDescriptor() throws IOException {
1292     this.admin.createTable(new HTableDescriptor());
1293   }
1294 
1295   @Test (expected=IllegalArgumentException.class, timeout=300000)
1296   public void testInvalidHColumnDescriptor() throws IOException {
1297      new HColumnDescriptor("/cfamily/name");
1298   }
1299 
1300   @Test (timeout=300000)
1301   public void testEnableDisableAddColumnDeleteColumn() throws Exception {
1302 	ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
1303     TableName tableName = TableName.valueOf("testEnableDisableAddColumnDeleteColumn");
1304     TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close();
1305     while (!ZKTableStateClientSideReader.isEnabledTable(zkw,
1306       TableName.valueOf("testEnableDisableAddColumnDeleteColumn"))) {
1307       Thread.sleep(10);
1308     }
1309     this.admin.disableTable(tableName);
1310     try {
1311       new HTable(TEST_UTIL.getConfiguration(), tableName);
1312     } catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
1313       //expected
1314     }
1315 
1316     this.admin.addColumn(tableName, new HColumnDescriptor("col2"));
1317     this.admin.enableTable(tableName);
1318     try {
1319       this.admin.deleteColumn(tableName, Bytes.toBytes("col2"));
1320     } catch (TableNotDisabledException e) {
1321       LOG.info(e);
1322     }
1323     this.admin.disableTable(tableName);
1324     this.admin.deleteTable(tableName);
1325   }
1326 
1327   @Test (timeout=300000)
1328   public void testDeleteLastColumnFamily() throws Exception {
1329     TableName tableName = TableName.valueOf("testDeleteLastColumnFamily");
1330     TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close();
1331     while (!this.admin.isTableEnabled(TableName.valueOf("testDeleteLastColumnFamily"))) {
1332       Thread.sleep(10);
1333     }
1334 
1335     // test for enabled table
1336     try {
1337       this.admin.deleteColumn(tableName, HConstants.CATALOG_FAMILY);
1338       fail("Should have failed to delete the only column family of a table");
1339     } catch (InvalidFamilyOperationException ex) {
1340       // expected
1341     }
1342 
1343     // test for disabled table
1344     this.admin.disableTable(tableName);
1345 
1346     try {
1347       this.admin.deleteColumn(tableName, HConstants.CATALOG_FAMILY);
1348       fail("Should have failed to delete the only column family of a table");
1349     } catch (InvalidFamilyOperationException ex) {
1350       // expected
1351     }
1352 
1353     this.admin.deleteTable(tableName);
1354   }
1355 
1356   /*
1357    * Test DFS replication for column families, where one CF has default replication(3) and the other
1358    * is set to 1.
1359    */
1360   @Test(timeout = 300000)
1361   public void testHFileReplication() throws Exception {
1362     TableName name = TableName.valueOf("testHFileReplication");
1363     String fn1 = "rep1";
1364     HColumnDescriptor hcd1 = new HColumnDescriptor(fn1);
1365     hcd1.setDFSReplication((short) 1);
1366     String fn = "defaultRep";
1367     HColumnDescriptor hcd = new HColumnDescriptor(fn);
1368     HTableDescriptor htd = new HTableDescriptor(name);
1369     htd.addFamily(hcd);
1370     htd.addFamily(hcd1);
1371     Table table = TEST_UTIL.createTable(htd, null);
1372     TEST_UTIL.waitTableAvailable(name);
1373     Put p = new Put(Bytes.toBytes("defaultRep_rk"));
1374     byte[] q1 = Bytes.toBytes("q1");
1375     byte[] v1 = Bytes.toBytes("v1");
1376     p.addColumn(Bytes.toBytes(fn), q1, v1);
1377     List<Put> puts = new ArrayList<Put>(2);
1378     puts.add(p);
1379     p = new Put(Bytes.toBytes("rep1_rk"));
1380     p.addColumn(Bytes.toBytes(fn1), q1, v1);
1381     puts.add(p);
1382     try {
1383       table.put(puts);
1384       admin.flush(name);
1385 
1386       List<HRegion> regions = TEST_UTIL.getMiniHBaseCluster().getRegions(name);
1387       for (HRegion r : regions) {
1388         Store store = r.getStore(Bytes.toBytes(fn));
1389         for (StoreFile sf : store.getStorefiles()) {
1390           assertTrue(sf.toString().contains(fn));
1391           assertTrue("Column family " + fn + " should have 3 copies",
1392             FSUtils.getDefaultReplication(TEST_UTIL.getTestFileSystem(), sf.getPath()) == (sf
1393                 .getFileInfo().getFileStatus().getReplication()));
1394         }
1395 
1396         store = r.getStore(Bytes.toBytes(fn1));
1397         for (StoreFile sf : store.getStorefiles()) {
1398           assertTrue(sf.toString().contains(fn1));
1399           assertTrue("Column family " + fn1 + " should have only 1 copy", 1 == sf.getFileInfo()
1400               .getFileStatus().getReplication());
1401         }
1402       }
1403     } finally {
1404       if (admin.isTableEnabled(name)) {
1405         this.admin.disableTable(name);
1406         this.admin.deleteTable(name);
1407       }
1408     }
1409   }
1410 
1411   @Test
1412   public void testMergeRegions() throws Exception {
1413     TableName tableName = TableName.valueOf("testMergeWithFullRegionName");
1414     HColumnDescriptor cd = new HColumnDescriptor("d");
1415     HTableDescriptor td = new HTableDescriptor(tableName);
1416     td.addFamily(cd);
1417     byte[][] splitRows = new byte[2][];
1418     splitRows[0] = new byte[]{(byte)'3'};
1419     splitRows[1] = new byte[]{(byte)'6'};
1420     try {
1421       TEST_UTIL.createTable(td, splitRows);
1422       TEST_UTIL.waitTableAvailable(tableName);
1423 
1424       List<HRegionInfo> tableRegions;
1425       HRegionInfo regionA;
1426       HRegionInfo regionB;
1427 
1428       // merge with full name
1429       tableRegions = admin.getTableRegions(tableName);
1430       assertEquals(3, admin.getTableRegions(tableName).size());
1431       regionA = tableRegions.get(0);
1432       regionB = tableRegions.get(1);
1433       admin.mergeRegions(regionA.getRegionName(), regionB.getRegionName(), false);
1434       Thread.sleep(1000);
1435       assertEquals(2, admin.getTableRegions(tableName).size());
1436 
1437       // merge with encoded name
1438       tableRegions = admin.getTableRegions(tableName);
1439       regionA = tableRegions.get(0);
1440       regionB = tableRegions.get(1);
1441       admin.mergeRegions(regionA.getEncodedNameAsBytes(), regionB.getEncodedNameAsBytes(), false);
1442       Thread.sleep(1000);
1443       assertEquals(1, admin.getTableRegions(tableName).size());
1444     } finally {
1445       this.admin.disableTable(tableName);
1446       this.admin.deleteTable(tableName);
1447     }
1448   }
1449 }