View Javadoc

1   /**
2    * Copyright The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.client;
21  
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertNull;
24  import static org.junit.Assert.assertTrue;
25  import static org.junit.Assert.fail;
26  
27  import java.util.ArrayList;
28  import java.util.Arrays;
29  import java.util.List;
30  import java.util.Random;
31  
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.hbase.TableName;
35  import org.apache.hadoop.hbase.HBaseTestingUtility;
36  import org.apache.hadoop.hbase.HColumnDescriptor;
37  import org.apache.hadoop.hbase.HRegionLocation;
38  import org.apache.hadoop.hbase.HTableDescriptor;
39  import org.apache.hadoop.hbase.LargeTests;
40  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
41  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
42  import org.apache.hadoop.hbase.util.Bytes;
43  import org.apache.hadoop.hbase.util.Pair;
44  import org.junit.After;
45  import org.junit.AfterClass;
46  import org.junit.Before;
47  import org.junit.BeforeClass;
48  import org.junit.Test;
49  import org.junit.experimental.categories.Category;
50  
51  @Category(LargeTests.class)
52  public class TestFromClientSide3 {
53    final Log LOG = LogFactory.getLog(getClass());
54    private final static HBaseTestingUtility TEST_UTIL
55      = new HBaseTestingUtility();
56    private static byte[] FAMILY = Bytes.toBytes("testFamily");
57    private static Random random = new Random();
58    private static int SLAVES = 3;
59    private static byte [] ROW = Bytes.toBytes("testRow");
60    private static final byte[] ANOTHERROW = Bytes.toBytes("anotherrow");
61    private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
62    private static byte [] VALUE = Bytes.toBytes("testValue");
63    private final static byte[] COL_QUAL = Bytes.toBytes("f1");
64    private final static byte[] VAL_BYTES = Bytes.toBytes("v1");
65    private final static byte[] ROW_BYTES = Bytes.toBytes("r1");
66  
67    /**
68     * @throws java.lang.Exception
69     */
70    @BeforeClass
71    public static void setUpBeforeClass() throws Exception {
72      TEST_UTIL.getConfiguration().setBoolean(
73          "hbase.online.schema.update.enable", true);
74      TEST_UTIL.startMiniCluster(SLAVES);
75    }
76  
77    /**
78     * @throws java.lang.Exception
79     */
80    @AfterClass
81    public static void tearDownAfterClass() throws Exception {
82      TEST_UTIL.shutdownMiniCluster();
83    }
84  
85    /**
86     * @throws java.lang.Exception
87     */
88    @Before
89    public void setUp() throws Exception {
90      // Nothing to do.
91    }
92  
93    /**
94     * @throws java.lang.Exception
95     */
96    @After
97    public void tearDown() throws Exception {
98      // Nothing to do.
99    }
100 
101   private void randomCFPuts(HTable table, byte[] row, byte[] family, int nPuts)
102       throws Exception {
103     Put put = new Put(row);
104     for (int i = 0; i < nPuts; i++) {
105       byte[] qualifier = Bytes.toBytes(random.nextInt());
106       byte[] value = Bytes.toBytes(random.nextInt());
107       put.add(family, qualifier, value);
108     }
109     table.put(put);
110   }
111 
112   private void performMultiplePutAndFlush(HBaseAdmin admin, HTable table,
113       byte[] row, byte[] family, int nFlushes, int nPuts) throws Exception {
114 
115     // connection needed for poll-wait
116     HConnection conn = HConnectionManager.getConnection(TEST_UTIL
117         .getConfiguration());
118     HRegionLocation loc = table.getRegionLocation(row, true);
119     AdminProtos.AdminService.BlockingInterface server = conn.getAdmin(loc.getServerName());
120     byte[] regName = loc.getRegionInfo().getRegionName();
121 
122     for (int i = 0; i < nFlushes; i++) {
123       randomCFPuts(table, row, family, nPuts);
124       List<String> sf = ProtobufUtil.getStoreFiles(server, regName, FAMILY);
125       int sfCount = sf.size();
126 
127       // TODO: replace this api with a synchronous flush after HBASE-2949
128       admin.flush(table.getTableName());
129 
130       // synchronously poll wait for a new storefile to appear (flush happened)
131       while (ProtobufUtil.getStoreFiles(
132           server, regName, FAMILY).size() == sfCount) {
133         Thread.sleep(40);
134       }
135     }
136   }
137 
138   // override the config settings at the CF level and ensure priority
139   @Test(timeout = 60000)
140   public void testAdvancedConfigOverride() throws Exception {
141     /*
142      * Overall idea: (1) create 3 store files and issue a compaction. config's
143      * compaction.min == 3, so should work. (2) Increase the compaction.min
144      * toggle in the HTD to 5 and modify table. If we use the HTD value instead
145      * of the default config value, adding 3 files and issuing a compaction
146      * SHOULD NOT work (3) Decrease the compaction.min toggle in the HCD to 2
147      * and modify table. The CF schema should override the Table schema and now
148      * cause a minor compaction.
149      */
150     TEST_UTIL.getConfiguration().setInt("hbase.hstore.compaction.min", 3);
151 
152     String tableName = "testAdvancedConfigOverride";
153     TableName TABLE =
154         TableName.valueOf(tableName);
155     HTable hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
156     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
157     HConnection connection = HConnectionManager.getConnection(TEST_UTIL
158         .getConfiguration());
159 
160     // Create 3 store files.
161     byte[] row = Bytes.toBytes(random.nextInt());
162     performMultiplePutAndFlush(admin, hTable, row, FAMILY, 3, 100);
163 
164     // Verify we have multiple store files.
165     HRegionLocation loc = hTable.getRegionLocation(row, true);
166     byte[] regionName = loc.getRegionInfo().getRegionName();
167     AdminProtos.AdminService.BlockingInterface server =
168       connection.getAdmin(loc.getServerName());
169     assertTrue(ProtobufUtil.getStoreFiles(
170       server, regionName, FAMILY).size() > 1);
171 
172     // Issue a compaction request
173     admin.compact(TABLE.getName());
174 
175     // poll wait for the compactions to happen
176     for (int i = 0; i < 10 * 1000 / 40; ++i) {
177       // The number of store files after compaction should be lesser.
178       loc = hTable.getRegionLocation(row, true);
179       if (!loc.getRegionInfo().isOffline()) {
180         regionName = loc.getRegionInfo().getRegionName();
181         server = connection.getAdmin(loc.getServerName());
182         if (ProtobufUtil.getStoreFiles(
183             server, regionName, FAMILY).size() <= 1) {
184           break;
185         }
186       }
187       Thread.sleep(40);
188     }
189     // verify the compactions took place and that we didn't just time out
190     assertTrue(ProtobufUtil.getStoreFiles(
191       server, regionName, FAMILY).size() <= 1);
192 
193     // change the compaction.min config option for this table to 5
194     LOG.info("hbase.hstore.compaction.min should now be 5");
195     HTableDescriptor htd = new HTableDescriptor(hTable.getTableDescriptor());
196     htd.setValue("hbase.hstore.compaction.min", String.valueOf(5));
197     admin.modifyTable(TABLE, htd);
198     Pair<Integer, Integer> st;
199     while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
200       LOG.debug(st.getFirst() + " regions left to update");
201       Thread.sleep(40);
202     }
203     LOG.info("alter status finished");
204 
205     // Create 3 more store files.
206     performMultiplePutAndFlush(admin, hTable, row, FAMILY, 3, 10);
207 
208     // Issue a compaction request
209     admin.compact(TABLE.getName());
210 
211     // This time, the compaction request should not happen
212     Thread.sleep(10 * 1000);
213     loc = hTable.getRegionLocation(row, true);
214     regionName = loc.getRegionInfo().getRegionName();
215     server = connection.getAdmin(loc.getServerName());
216     int sfCount = ProtobufUtil.getStoreFiles(
217       server, regionName, FAMILY).size();
218     assertTrue(sfCount > 1);
219 
220     // change an individual CF's config option to 2 & online schema update
221     LOG.info("hbase.hstore.compaction.min should now be 2");
222     HColumnDescriptor hcd = new HColumnDescriptor(htd.getFamily(FAMILY));
223     hcd.setValue("hbase.hstore.compaction.min", String.valueOf(2));
224     htd.addFamily(hcd);
225     admin.modifyTable(TABLE, htd);
226     while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
227       LOG.debug(st.getFirst() + " regions left to update");
228       Thread.sleep(40);
229     }
230     LOG.info("alter status finished");
231 
232     // Issue a compaction request
233     admin.compact(TABLE.getName());
234 
235     // poll wait for the compactions to happen
236     for (int i = 0; i < 10 * 1000 / 40; ++i) {
237       loc = hTable.getRegionLocation(row, true);
238       regionName = loc.getRegionInfo().getRegionName();
239       try {
240         server = connection.getAdmin(loc.getServerName());
241         if (ProtobufUtil.getStoreFiles(
242             server, regionName, FAMILY).size() < sfCount) {
243           break;
244         }
245       } catch (Exception e) {
246         LOG.debug("Waiting for region to come online: " + regionName);
247       }
248       Thread.sleep(40);
249     }
250     // verify the compaction took place and that we didn't just time out
251     assertTrue(ProtobufUtil.getStoreFiles(
252       server, regionName, FAMILY).size() < sfCount);
253 
254     // Finally, ensure that we can remove a custom config value after we made it
255     LOG.info("Removing CF config value");
256     LOG.info("hbase.hstore.compaction.min should now be 5");
257     hcd = new HColumnDescriptor(htd.getFamily(FAMILY));
258     hcd.setValue("hbase.hstore.compaction.min", null);
259     htd.addFamily(hcd);
260     admin.modifyTable(TABLE, htd);
261     while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
262       LOG.debug(st.getFirst() + " regions left to update");
263       Thread.sleep(40);
264     }
265     LOG.info("alter status finished");
266     assertNull(hTable.getTableDescriptor().getFamily(FAMILY).getValue(
267         "hbase.hstore.compaction.min"));
268   }
269 
270   @Test
271   public void testHTableBatchWithEmptyPut() throws Exception {
272     HTable table = TEST_UTIL.createTable(
273       Bytes.toBytes("testHTableBatchWithEmptyPut"), new byte[][] { FAMILY });
274     try {
275       List actions = (List) new ArrayList();
276       Object[] results = new Object[2];
277       // create an empty Put
278       Put put1 = new Put(ROW);
279       actions.add(put1);
280       
281       Put put2 = new Put(ANOTHERROW);
282       put2.add(FAMILY, QUALIFIER, VALUE);
283       actions.add(put2);
284       
285       table.batch(actions, results);
286       fail("Empty Put should have failed the batch call");
287     } catch (IllegalArgumentException iae) {
288       
289     } finally {
290       table.close();
291     }
292   }
293   
294   @Test
295   public void testHTableExistsMethodSingleRegionSingleGet() throws Exception {
296 
297     // Test with a single region table.
298 
299     HTable table = TEST_UTIL.createTable(
300       Bytes.toBytes("testHTableExistsMethodSingleRegionSingleGet"), new byte[][] { FAMILY });
301 
302     Put put = new Put(ROW);
303     put.add(FAMILY, QUALIFIER, VALUE);
304 
305     Get get = new Get(ROW);
306 
307     boolean exist = table.exists(get);
308     assertEquals(exist, false);
309 
310     table.put(put);
311 
312     exist = table.exists(get);
313     assertEquals(exist, true);
314   }
315 
316   public void testHTableExistsMethodSingleRegionMultipleGets() throws Exception {
317 
318     HTable table = TEST_UTIL.createTable(
319       Bytes.toBytes("testHTableExistsMethodSingleRegionMultipleGets"), new byte[][] { FAMILY });
320 
321     Put put = new Put(ROW);
322     put.add(FAMILY, QUALIFIER, VALUE);
323     table.put(put);
324 
325     List<Get> gets = new ArrayList<Get>();
326     gets.add(new Get(ROW));
327     gets.add(null);
328     gets.add(new Get(ANOTHERROW));
329 
330     Boolean[] results = table.exists(gets);
331     assertEquals(results[0], true);
332     assertEquals(results[1], false);
333     assertEquals(results[2], false);
334   }
335 
336   @Test
337   public void testHTableExistsMethodMultipleRegionsSingleGet() throws Exception {
338 
339     HTable table = TEST_UTIL.createTable(
340       Bytes.toBytes("testHTableExistsMethodMultipleRegionsSingleGet"), new byte[][] { FAMILY }, 1,
341       new byte[] { 0x00 }, new byte[] { (byte) 0xff }, 255);
342     Put put = new Put(ROW);
343     put.add(FAMILY, QUALIFIER, VALUE);
344 
345     Get get = new Get(ROW);
346 
347     boolean exist = table.exists(get);
348     assertEquals(exist, false);
349 
350     table.put(put);
351 
352     exist = table.exists(get);
353     assertEquals(exist, true);
354   }
355 
356   @Test
357   public void testHTableExistsMethodMultipleRegionsMultipleGets() throws Exception {
358     HTable table = TEST_UTIL.createTable(
359       Bytes.toBytes("testHTableExistsMethodMultipleRegionsMultipleGets"), new byte[][] { FAMILY },
360       1, new byte[] { 0x00 }, new byte[] { (byte) 0xff }, 255);
361     Put put = new Put(ROW);
362     put.add(FAMILY, QUALIFIER, VALUE);
363     table.put (put);
364 
365     List<Get> gets = new ArrayList<Get>();
366     gets.add(new Get(ANOTHERROW));
367     gets.add(new Get(Bytes.add(ROW, new byte[] { 0x00 })));
368     gets.add(new Get(ROW));
369     gets.add(new Get(Bytes.add(ANOTHERROW, new byte[] { 0x00 })));
370 
371     LOG.info("Calling exists");
372     Boolean[] results = table.exists(gets);
373     assertEquals(results[0], false);
374     assertEquals(results[1], false);
375     assertEquals(results[2], true);
376     assertEquals(results[3], false);
377 
378     // Test with the first region.
379     put = new Put(new byte[] { 0x00 });
380     put.add(FAMILY, QUALIFIER, VALUE);
381     table.put(put);
382 
383     gets = new ArrayList<Get>();
384     gets.add(new Get(new byte[] { 0x00 }));
385     gets.add(new Get(new byte[] { 0x00, 0x00 }));
386     results = table.exists(gets);
387     assertEquals(results[0], true);
388     assertEquals(results[1], false);
389 
390     // Test with the last region
391     put = new Put(new byte[] { (byte) 0xff, (byte) 0xff });
392     put.add(FAMILY, QUALIFIER, VALUE);
393     table.put(put);
394 
395     gets = new ArrayList<Get>();
396     gets.add(new Get(new byte[] { (byte) 0xff }));
397     gets.add(new Get(new byte[] { (byte) 0xff, (byte) 0xff }));
398     gets.add(new Get(new byte[] { (byte) 0xff, (byte) 0xff, (byte) 0xff }));
399     results = table.exists(gets);
400     assertEquals(results[0], false);
401     assertEquals(results[1], true);
402     assertEquals(results[2], false);
403   }
404 
405   @Test
406   public void testGetEmptyRow() throws Exception {
407     //Create a table and put in 1 row
408     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
409     HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(Bytes.toBytes("test")));
410     desc.addFamily(new HColumnDescriptor(FAMILY));
411     admin.createTable(desc);
412     HTable table = new HTable(TEST_UTIL.getConfiguration(), "test");
413 
414     Put put = new Put(ROW_BYTES);
415     put.add(FAMILY, COL_QUAL, VAL_BYTES);
416     table.put(put);
417     table.flushCommits();
418 
419     //Try getting the row with an empty row key
420     Result res = null;
421     try {
422       res = table.get(new Get(new byte[0]));
423       fail();
424     } catch (IllegalArgumentException e) {
425       // Expected.
426     }
427     assertTrue(res == null);
428     res = table.get(new Get(Bytes.toBytes("r1-not-exist")));
429     assertTrue(res.isEmpty() == true);
430     res = table.get(new Get(ROW_BYTES));
431     assertTrue(Arrays.equals(res.getValue(FAMILY, COL_QUAL), VAL_BYTES));
432     table.close();
433   }
434 }