View Javadoc

1   /**
2    * Copyright The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.client;
21  
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertNull;
24  import static org.junit.Assert.assertTrue;
25  import static org.junit.Assert.fail;
26  
27  import java.util.ArrayList;
28  import java.util.Arrays;
29  import java.util.List;
30  import java.util.Random;
31  
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.hbase.HBaseTestingUtility;
35  import org.apache.hadoop.hbase.HColumnDescriptor;
36  import org.apache.hadoop.hbase.HRegionLocation;
37  import org.apache.hadoop.hbase.HTableDescriptor;
38  import org.apache.hadoop.hbase.TableName;
39  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
40  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
41  import org.apache.hadoop.hbase.testclassification.LargeTests;
42  import org.apache.hadoop.hbase.util.Bytes;
43  import org.apache.hadoop.hbase.util.Pair;
44  import org.junit.After;
45  import org.junit.AfterClass;
46  import org.junit.Before;
47  import org.junit.BeforeClass;
48  import org.junit.Test;
49  import org.junit.experimental.categories.Category;
50  
51  @Category(LargeTests.class)
52  public class TestFromClientSide3 {
53    private static final Log LOG = LogFactory.getLog(TestFromClientSide3.class);
54    private final static HBaseTestingUtility TEST_UTIL
55      = new HBaseTestingUtility();
56    private static byte[] FAMILY = Bytes.toBytes("testFamily");
57    private static Random random = new Random();
58    private static int SLAVES = 3;
59    private static byte [] ROW = Bytes.toBytes("testRow");
60    private static final byte[] ANOTHERROW = Bytes.toBytes("anotherrow");
61    private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
62    private static byte [] VALUE = Bytes.toBytes("testValue");
63    private final static byte[] COL_QUAL = Bytes.toBytes("f1");
64    private final static byte[] VAL_BYTES = Bytes.toBytes("v1");
65    private final static byte[] ROW_BYTES = Bytes.toBytes("r1");
66  
67    /**
68     * @throws java.lang.Exception
69     */
70    @BeforeClass
71    public static void setUpBeforeClass() throws Exception {
72      TEST_UTIL.getConfiguration().setBoolean(
73          "hbase.online.schema.update.enable", true);
74      TEST_UTIL.startMiniCluster(SLAVES);
75    }
76  
77    /**
78     * @throws java.lang.Exception
79     */
80    @AfterClass
81    public static void tearDownAfterClass() throws Exception {
82      TEST_UTIL.shutdownMiniCluster();
83    }
84  
85    /**
86     * @throws java.lang.Exception
87     */
88    @Before
89    public void setUp() throws Exception {
90      // Nothing to do.
91    }
92  
93    /**
94     * @throws java.lang.Exception
95     */
96    @After
97    public void tearDown() throws Exception {
98      for (HTableDescriptor htd: TEST_UTIL.getHBaseAdmin().listTables()) {
99        LOG.info("Tear down, remove table=" + htd.getTableName());
100       TEST_UTIL.deleteTable(htd.getTableName());
101     }
102   }
103 
104   private void randomCFPuts(Table table, byte[] row, byte[] family, int nPuts)
105       throws Exception {
106     Put put = new Put(row);
107     for (int i = 0; i < nPuts; i++) {
108       byte[] qualifier = Bytes.toBytes(random.nextInt());
109       byte[] value = Bytes.toBytes(random.nextInt());
110       put.add(family, qualifier, value);
111     }
112     table.put(put);
113   }
114 
115   private void performMultiplePutAndFlush(HBaseAdmin admin, HTable table,
116       byte[] row, byte[] family, int nFlushes, int nPuts)
117   throws Exception {
118 
119     // connection needed for poll-wait
120     HRegionLocation loc = table.getRegionLocation(row, true);
121     AdminProtos.AdminService.BlockingInterface server =
122       admin.getConnection().getAdmin(loc.getServerName());
123     byte[] regName = loc.getRegionInfo().getRegionName();
124 
125     for (int i = 0; i < nFlushes; i++) {
126       randomCFPuts(table, row, family, nPuts);
127       List<String> sf = ProtobufUtil.getStoreFiles(server, regName, FAMILY);
128       int sfCount = sf.size();
129 
130       // TODO: replace this api with a synchronous flush after HBASE-2949
131       admin.flush(table.getTableName());
132 
133       // synchronously poll wait for a new storefile to appear (flush happened)
134       while (ProtobufUtil.getStoreFiles(
135           server, regName, FAMILY).size() == sfCount) {
136         Thread.sleep(40);
137       }
138     }
139   }
140 
141   // override the config settings at the CF level and ensure priority
142   @Test(timeout = 60000)
143   public void testAdvancedConfigOverride() throws Exception {
144     /*
145      * Overall idea: (1) create 3 store files and issue a compaction. config's
146      * compaction.min == 3, so should work. (2) Increase the compaction.min
147      * toggle in the HTD to 5 and modify table. If we use the HTD value instead
148      * of the default config value, adding 3 files and issuing a compaction
149      * SHOULD NOT work (3) Decrease the compaction.min toggle in the HCD to 2
150      * and modify table. The CF schema should override the Table schema and now
151      * cause a minor compaction.
152      */
153     TEST_UTIL.getConfiguration().setInt("hbase.hstore.compaction.min", 3);
154 
155     String tableName = "testAdvancedConfigOverride";
156     TableName TABLE = TableName.valueOf(tableName);
157     HTable hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
158     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
159     ClusterConnection connection = (ClusterConnection)TEST_UTIL.getConnection();
160 
161     // Create 3 store files.
162     byte[] row = Bytes.toBytes(random.nextInt());
163     performMultiplePutAndFlush(admin, hTable, row, FAMILY, 3, 100);
164 
165     // Verify we have multiple store files.
166     HRegionLocation loc = hTable.getRegionLocation(row, true);
167     byte[] regionName = loc.getRegionInfo().getRegionName();
168     AdminProtos.AdminService.BlockingInterface server =
169       connection.getAdmin(loc.getServerName());
170     assertTrue(ProtobufUtil.getStoreFiles(
171       server, regionName, FAMILY).size() > 1);
172 
173     // Issue a compaction request
174     admin.compact(TABLE.getName());
175 
176     // poll wait for the compactions to happen
177     for (int i = 0; i < 10 * 1000 / 40; ++i) {
178       // The number of store files after compaction should be lesser.
179       loc = hTable.getRegionLocation(row, true);
180       if (!loc.getRegionInfo().isOffline()) {
181         regionName = loc.getRegionInfo().getRegionName();
182         server = connection.getAdmin(loc.getServerName());
183         if (ProtobufUtil.getStoreFiles(
184             server, regionName, FAMILY).size() <= 1) {
185           break;
186         }
187       }
188       Thread.sleep(40);
189     }
190     // verify the compactions took place and that we didn't just time out
191     assertTrue(ProtobufUtil.getStoreFiles(
192       server, regionName, FAMILY).size() <= 1);
193 
194     // change the compaction.min config option for this table to 5
195     LOG.info("hbase.hstore.compaction.min should now be 5");
196     HTableDescriptor htd = new HTableDescriptor(hTable.getTableDescriptor());
197     htd.setValue("hbase.hstore.compaction.min", String.valueOf(5));
198     admin.modifyTable(TABLE, htd);
199     Pair<Integer, Integer> st;
200     while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
201       LOG.debug(st.getFirst() + " regions left to update");
202       Thread.sleep(40);
203     }
204     LOG.info("alter status finished");
205 
206     // Create 3 more store files.
207     performMultiplePutAndFlush(admin, hTable, row, FAMILY, 3, 10);
208 
209     // Issue a compaction request
210     admin.compact(TABLE.getName());
211 
212     // This time, the compaction request should not happen
213     Thread.sleep(10 * 1000);
214     loc = hTable.getRegionLocation(row, true);
215     regionName = loc.getRegionInfo().getRegionName();
216     server = connection.getAdmin(loc.getServerName());
217     int sfCount = ProtobufUtil.getStoreFiles(
218       server, regionName, FAMILY).size();
219     assertTrue(sfCount > 1);
220 
221     // change an individual CF's config option to 2 & online schema update
222     LOG.info("hbase.hstore.compaction.min should now be 2");
223     HColumnDescriptor hcd = new HColumnDescriptor(htd.getFamily(FAMILY));
224     hcd.setValue("hbase.hstore.compaction.min", String.valueOf(2));
225     htd.modifyFamily(hcd);
226     admin.modifyTable(TABLE, htd);
227     while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
228       LOG.debug(st.getFirst() + " regions left to update");
229       Thread.sleep(40);
230     }
231     LOG.info("alter status finished");
232 
233     // Issue a compaction request
234     admin.compact(TABLE.getName());
235 
236     // poll wait for the compactions to happen
237     for (int i = 0; i < 10 * 1000 / 40; ++i) {
238       loc = hTable.getRegionLocation(row, true);
239       regionName = loc.getRegionInfo().getRegionName();
240       try {
241         server = connection.getAdmin(loc.getServerName());
242         if (ProtobufUtil.getStoreFiles(
243             server, regionName, FAMILY).size() < sfCount) {
244           break;
245         }
246       } catch (Exception e) {
247         LOG.debug("Waiting for region to come online: " + regionName);
248       }
249       Thread.sleep(40);
250     }
251     // verify the compaction took place and that we didn't just time out
252     assertTrue(ProtobufUtil.getStoreFiles(
253       server, regionName, FAMILY).size() < sfCount);
254 
255     // Finally, ensure that we can remove a custom config value after we made it
256     LOG.info("Removing CF config value");
257     LOG.info("hbase.hstore.compaction.min should now be 5");
258     hcd = new HColumnDescriptor(htd.getFamily(FAMILY));
259     hcd.setValue("hbase.hstore.compaction.min", null);
260     htd.modifyFamily(hcd);
261     admin.modifyTable(TABLE, htd);
262     while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
263       LOG.debug(st.getFirst() + " regions left to update");
264       Thread.sleep(40);
265     }
266     LOG.info("alter status finished");
267     assertNull(hTable.getTableDescriptor().getFamily(FAMILY).getValue(
268         "hbase.hstore.compaction.min"));
269   }
270 
271   @Test
272   public void testHTableBatchWithEmptyPut() throws Exception {
273     Table table = TEST_UTIL.createTable(
274       Bytes.toBytes("testHTableBatchWithEmptyPut"), new byte[][] { FAMILY });
275     try {
276       List actions = (List) new ArrayList();
277       Object[] results = new Object[2];
278       // create an empty Put
279       Put put1 = new Put(ROW);
280       actions.add(put1);
281 
282       Put put2 = new Put(ANOTHERROW);
283       put2.add(FAMILY, QUALIFIER, VALUE);
284       actions.add(put2);
285 
286       table.batch(actions, results);
287       fail("Empty Put should have failed the batch call");
288     } catch (IllegalArgumentException iae) {
289 
290     } finally {
291       table.close();
292     }
293   }
294 
295   @Test
296   public void testHTableExistsMethodSingleRegionSingleGet() throws Exception {
297 
298     // Test with a single region table.
299 
300     Table table = TEST_UTIL.createTable(
301       Bytes.toBytes("testHTableExistsMethodSingleRegionSingleGet"), new byte[][] { FAMILY });
302 
303     Put put = new Put(ROW);
304     put.add(FAMILY, QUALIFIER, VALUE);
305 
306     Get get = new Get(ROW);
307 
308     boolean exist = table.exists(get);
309     assertEquals(exist, false);
310 
311     table.put(put);
312 
313     exist = table.exists(get);
314     assertEquals(exist, true);
315   }
316 
317   public void testHTableExistsMethodSingleRegionMultipleGets() throws Exception {
318 
319     HTable table = TEST_UTIL.createTable(
320       Bytes.toBytes("testHTableExistsMethodSingleRegionMultipleGets"), new byte[][] { FAMILY });
321 
322     Put put = new Put(ROW);
323     put.add(FAMILY, QUALIFIER, VALUE);
324     table.put(put);
325 
326     List<Get> gets = new ArrayList<Get>();
327     gets.add(new Get(ROW));
328     gets.add(null);
329     gets.add(new Get(ANOTHERROW));
330 
331     Boolean[] results = table.exists(gets);
332     assertEquals(results[0], true);
333     assertEquals(results[1], false);
334     assertEquals(results[2], false);
335   }
336 
337   @Test
338   public void testHTableExistsBeforeGet() throws Exception {
339     Table table = TEST_UTIL.createTable(
340       Bytes.toBytes("testHTableExistsBeforeGet"), new byte[][] { FAMILY });
341     try {
342       Put put = new Put(ROW);
343       put.add(FAMILY, QUALIFIER, VALUE);
344       table.put(put);
345 
346       Get get = new Get(ROW);
347 
348       boolean exist = table.exists(get);
349       assertEquals(true, exist);
350 
351       Result result = table.get(get);
352       assertEquals(false, result.isEmpty());
353       assertTrue(Bytes.equals(VALUE, result.getValue(FAMILY, QUALIFIER)));
354     } finally {
355       table.close();
356     }
357   }
358 
359   @Test
360   public void testHTableExistsAllBeforeGet() throws Exception {
361     final byte[] ROW2 = Bytes.add(ROW, Bytes.toBytes("2"));
362     Table table = TEST_UTIL.createTable(
363       Bytes.toBytes("testHTableExistsAllBeforeGet"), new byte[][] { FAMILY });
364     try {
365       Put put = new Put(ROW);
366       put.add(FAMILY, QUALIFIER, VALUE);
367       table.put(put);
368       put = new Put(ROW2);
369       put.add(FAMILY, QUALIFIER, VALUE);
370       table.put(put);
371 
372       Get get = new Get(ROW);
373       Get get2 = new Get(ROW2);
374       ArrayList<Get> getList = new ArrayList(2);
375       getList.add(get);
376       getList.add(get2);
377 
378       boolean[] exists = table.existsAll(getList);
379       assertEquals(true, exists[0]);
380       assertEquals(true, exists[1]);
381 
382       Result[] result = table.get(getList);
383       assertEquals(false, result[0].isEmpty());
384       assertTrue(Bytes.equals(VALUE, result[0].getValue(FAMILY, QUALIFIER)));
385       assertEquals(false, result[1].isEmpty());
386       assertTrue(Bytes.equals(VALUE, result[1].getValue(FAMILY, QUALIFIER)));
387     } finally {
388       table.close();
389     }
390   }
391 
392   @Test
393   public void testHTableExistsMethodMultipleRegionsSingleGet() throws Exception {
394 
395     Table table = TEST_UTIL.createTable(
396       TableName.valueOf("testHTableExistsMethodMultipleRegionsSingleGet"), new byte[][] { FAMILY },
397       1, new byte[] { 0x00 }, new byte[] { (byte) 0xff }, 255);
398     Put put = new Put(ROW);
399     put.add(FAMILY, QUALIFIER, VALUE);
400 
401     Get get = new Get(ROW);
402 
403     boolean exist = table.exists(get);
404     assertEquals(exist, false);
405 
406     table.put(put);
407 
408     exist = table.exists(get);
409     assertEquals(exist, true);
410   }
411 
412   @Test
413   public void testHTableExistsMethodMultipleRegionsMultipleGets() throws Exception {
414     HTable table = TEST_UTIL.createTable(
415       TableName.valueOf("testHTableExistsMethodMultipleRegionsMultipleGets"),
416       new byte[][] { FAMILY }, 1, new byte[] { 0x00 }, new byte[] { (byte) 0xff }, 255);
417     Put put = new Put(ROW);
418     put.add(FAMILY, QUALIFIER, VALUE);
419     table.put (put);
420 
421     List<Get> gets = new ArrayList<Get>();
422     gets.add(new Get(ANOTHERROW));
423     gets.add(new Get(Bytes.add(ROW, new byte[] { 0x00 })));
424     gets.add(new Get(ROW));
425     gets.add(new Get(Bytes.add(ANOTHERROW, new byte[] { 0x00 })));
426 
427     LOG.info("Calling exists");
428     Boolean[] results = table.exists(gets);
429     assertEquals(results[0], false);
430     assertEquals(results[1], false);
431     assertEquals(results[2], true);
432     assertEquals(results[3], false);
433 
434     // Test with the first region.
435     put = new Put(new byte[] { 0x00 });
436     put.add(FAMILY, QUALIFIER, VALUE);
437     table.put(put);
438 
439     gets = new ArrayList<Get>();
440     gets.add(new Get(new byte[] { 0x00 }));
441     gets.add(new Get(new byte[] { 0x00, 0x00 }));
442     results = table.exists(gets);
443     assertEquals(results[0], true);
444     assertEquals(results[1], false);
445 
446     // Test with the last region
447     put = new Put(new byte[] { (byte) 0xff, (byte) 0xff });
448     put.add(FAMILY, QUALIFIER, VALUE);
449     table.put(put);
450 
451     gets = new ArrayList<Get>();
452     gets.add(new Get(new byte[] { (byte) 0xff }));
453     gets.add(new Get(new byte[] { (byte) 0xff, (byte) 0xff }));
454     gets.add(new Get(new byte[] { (byte) 0xff, (byte) 0xff, (byte) 0xff }));
455     results = table.exists(gets);
456     assertEquals(results[0], false);
457     assertEquals(results[1], true);
458     assertEquals(results[2], false);
459   }
460 
461   @Test
462   public void testGetEmptyRow() throws Exception {
463     //Create a table and put in 1 row
464     Admin admin = TEST_UTIL.getHBaseAdmin();
465     HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(Bytes.toBytes("test")));
466     desc.addFamily(new HColumnDescriptor(FAMILY));
467     admin.createTable(desc);
468     Table table = new HTable(TEST_UTIL.getConfiguration(), desc.getTableName());
469 
470     Put put = new Put(ROW_BYTES);
471     put.add(FAMILY, COL_QUAL, VAL_BYTES);
472     table.put(put);
473 
474     //Try getting the row with an empty row key
475     Result res = null;
476     try {
477       res = table.get(new Get(new byte[0]));
478       fail();
479     } catch (IllegalArgumentException e) {
480       // Expected.
481     }
482     assertTrue(res == null);
483     res = table.get(new Get(Bytes.toBytes("r1-not-exist")));
484     assertTrue(res.isEmpty() == true);
485     res = table.get(new Get(ROW_BYTES));
486     assertTrue(Arrays.equals(res.getValue(FAMILY, COL_QUAL), VAL_BYTES));
487     table.close();
488   }
489 }