View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.client;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.fail;
23  
24  import java.io.IOException;
25  import java.util.Arrays;
26  import java.util.Collection;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.hbase.CategoryBasedTimeout;
32  import org.apache.hadoop.hbase.Cell;
33  import org.apache.hadoop.hbase.DoNotRetryIOException;
34  import org.apache.hadoop.hbase.HBaseTestingUtility;
35  import org.apache.hadoop.hbase.TableName;
36  import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
37  import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
38  import org.apache.hadoop.hbase.testclassification.LargeTests;
39  import org.apache.hadoop.hbase.util.Bytes;
40  import org.junit.AfterClass;
41  import org.junit.BeforeClass;
42  import org.junit.Rule;
43  import org.junit.Test;
44  import org.junit.experimental.categories.Category;
45  import org.junit.rules.TestName;
46  import org.junit.rules.TestRule;
47  
48  /**
49   * Run Increment tests that use the HBase clients; {@link HTable}.
50   *
51   * Test is parameterized to run the slow and fast increment code paths. If fast, in the @before, we
52   * do a rolling restart of the single regionserver so that it can pick up the go fast configuration.
53   * Doing it this way should be faster than starting/stopping a cluster per test.
54   *
55   * Test takes a long time because spin up a cluster between each run -- ugh.
56   */
57  @Category(LargeTests.class)
58  @SuppressWarnings ("deprecation")
59  public class TestIncrementsFromClientSide {
60    final Log LOG = LogFactory.getLog(getClass());
61    protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
62    private static byte [] ROW = Bytes.toBytes("testRow");
63    private static byte [] FAMILY = Bytes.toBytes("testFamily");
64    // This test depends on there being only one slave running at at a time. See the @Before
65    // method where we do rolling restart.
66    protected static int SLAVES = 1;
67    @Rule public TestName name = new TestName();
68    @Rule public final TestRule timeout = CategoryBasedTimeout.builder().withTimeout(this.getClass()).
69        withLookingForStuckThread(true).build();
70    public static Collection<Object []> data() {
71      return Arrays.asList(new Object[] {Boolean.FALSE}, new Object [] {Boolean.TRUE});
72    }
73  
74    @BeforeClass
75    public static void beforeClass() throws Exception {
76      Configuration conf = TEST_UTIL.getConfiguration();
77      conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
78          MultiRowMutationEndpoint.class.getName());
79      conf.setBoolean("hbase.table.sanity.checks", true); // enable for below tests
80      // We need more than one region server in this test
81      TEST_UTIL.startMiniCluster(SLAVES);
82    }
83  
84    /**
85     * @throws java.lang.Exception
86     */
87    @AfterClass
88    public static void afterClass() throws Exception {
89      TEST_UTIL.shutdownMiniCluster();
90    }
91  
92    @Test
93    public void testIncrementWithDeletes() throws Exception {
94      LOG.info("Starting " + this.name.getMethodName());
95      final TableName TABLENAME =
96          TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName()));
97      Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
98      final byte[] COLUMN = Bytes.toBytes("column");
99  
100     ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5);
101     TEST_UTIL.flush(TABLENAME);
102 
103     Delete del = new Delete(ROW);
104     ht.delete(del);
105 
106     ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5);
107 
108     Get get = new Get(ROW);
109     Result r = ht.get(get);
110     assertEquals(1, r.size());
111     assertEquals(5, Bytes.toLong(r.getValue(FAMILY, COLUMN)));
112   }
113 
114   @Test
115   public void testIncrementingInvalidValue() throws Exception {
116     LOG.info("Starting " + this.name.getMethodName());
117     final TableName TABLENAME =
118         TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName()));
119     Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
120     final byte[] COLUMN = Bytes.toBytes("column");
121     Put p = new Put(ROW);
122     // write an integer here (not a Long)
123     p.add(FAMILY, COLUMN, Bytes.toBytes(5));
124     ht.put(p);
125     try {
126       ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5);
127       fail("Should have thrown DoNotRetryIOException");
128     } catch (DoNotRetryIOException iox) {
129       // success
130     }
131     Increment inc = new Increment(ROW);
132     inc.addColumn(FAMILY, COLUMN, 5);
133     try {
134       ht.increment(inc);
135       fail("Should have thrown DoNotRetryIOException");
136     } catch (DoNotRetryIOException iox) {
137       // success
138     }
139   }
140 
141   @Test
142   public void testIncrementInvalidArguments() throws Exception {
143     LOG.info("Starting " + this.name.getMethodName());
144     final TableName TABLENAME =
145         TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName()));
146     Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
147     final byte[] COLUMN = Bytes.toBytes("column");
148     try {
149       // try null row
150       ht.incrementColumnValue(null, FAMILY, COLUMN, 5);
151       fail("Should have thrown IOException");
152     } catch (IOException iox) {
153       // success
154     }
155     try {
156       // try null family
157       ht.incrementColumnValue(ROW, null, COLUMN, 5);
158       fail("Should have thrown IOException");
159     } catch (IOException iox) {
160       // success
161     }
162     try {
163       // try null qualifier
164       ht.incrementColumnValue(ROW, FAMILY, null, 5);
165       fail("Should have thrown IOException");
166     } catch (IOException iox) {
167       // success
168     }
169     // try null row
170     try {
171       Increment incNoRow = new Increment((byte [])null);
172       incNoRow.addColumn(FAMILY, COLUMN, 5);
173       fail("Should have thrown IllegalArgumentException");
174     } catch (IllegalArgumentException iax) {
175       // success
176     } catch (NullPointerException npe) {
177       // success
178     }
179     // try null family
180     try {
181       Increment incNoFamily = new Increment(ROW);
182       incNoFamily.addColumn(null, COLUMN, 5);
183       fail("Should have thrown IllegalArgumentException");
184     } catch (IllegalArgumentException iax) {
185       // success
186     }
187     // try null qualifier
188     try {
189       Increment incNoQualifier = new Increment(ROW);
190       incNoQualifier.addColumn(FAMILY, null, 5);
191       fail("Should have thrown IllegalArgumentException");
192     } catch (IllegalArgumentException iax) {
193       // success
194     }
195   }
196 
197   @Test
198   public void testIncrementOutOfOrder() throws Exception {
199     LOG.info("Starting " + this.name.getMethodName());
200     final TableName TABLENAME =
201         TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName()));
202     Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
203 
204     byte [][] QUALIFIERS = new byte [][] {
205       Bytes.toBytes("B"), Bytes.toBytes("A"), Bytes.toBytes("C")
206     };
207 
208     Increment inc = new Increment(ROW);
209     for (int i=0; i<QUALIFIERS.length; i++) {
210       inc.addColumn(FAMILY, QUALIFIERS[i], 1);
211     }
212     ht.increment(inc);
213 
214     // Verify expected results
215     Get get = new Get(ROW);
216     Result r = ht.get(get);
217     Cell [] kvs = r.rawCells();
218     assertEquals(3, kvs.length);
219     assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[1], 1);
220     assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[0], 1);
221     assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 1);
222 
223     // Now try multiple columns again
224     inc = new Increment(ROW);
225     for (int i=0; i<QUALIFIERS.length; i++) {
226       inc.addColumn(FAMILY, QUALIFIERS[i], 1);
227     }
228     ht.increment(inc);
229 
230     // Verify
231     r = ht.get(get);
232     kvs = r.rawCells();
233     assertEquals(3, kvs.length);
234     assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[1], 2);
235     assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[0], 2);
236     assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 2);
237   }
238 
239   @Test
240   public void testIncrementOnSameColumn() throws Exception {
241     LOG.info("Starting " + this.name.getMethodName());
242     final byte[] TABLENAME = Bytes.toBytes(filterStringSoTableNameSafe(this.name.getMethodName()));
243     HTable ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
244 
245     byte[][] QUALIFIERS =
246         new byte[][] { Bytes.toBytes("A"), Bytes.toBytes("B"), Bytes.toBytes("C") };
247 
248     Increment inc = new Increment(ROW);
249     for (int i = 0; i < QUALIFIERS.length; i++) {
250       inc.addColumn(FAMILY, QUALIFIERS[i], 1);
251       inc.addColumn(FAMILY, QUALIFIERS[i], 1);
252     }
253     ht.increment(inc);
254 
255     // Verify expected results
256     Get get = new Get(ROW);
257     Result r = ht.get(get);
258     Cell[] kvs = r.rawCells();
259     assertEquals(3, kvs.length);
260     assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[0], 1);
261     assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[1], 1);
262     assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 1);
263 
264     // Now try multiple columns again
265     inc = new Increment(ROW);
266     for (int i = 0; i < QUALIFIERS.length; i++) {
267       inc.addColumn(FAMILY, QUALIFIERS[i], 1);
268       inc.addColumn(FAMILY, QUALIFIERS[i], 1);
269     }
270     ht.increment(inc);
271 
272     // Verify
273     r = ht.get(get);
274     kvs = r.rawCells();
275     assertEquals(3, kvs.length);
276     assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[0], 2);
277     assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[1], 2);
278     assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 2);
279 
280     ht.close();
281   }
282 
283   @Test
284   public void testIncrement() throws Exception {
285     LOG.info("Starting " + this.name.getMethodName());
286     final TableName TABLENAME =
287         TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName()));
288     Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
289 
290     byte [][] ROWS = new byte [][] {
291         Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c"),
292         Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"),
293         Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i")
294     };
295     byte [][] QUALIFIERS = new byte [][] {
296         Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c"),
297         Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"),
298         Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i")
299     };
300 
301     // Do some simple single-column increments
302 
303     // First with old API
304     ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[0], 1);
305     ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[1], 2);
306     ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[2], 3);
307     ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[3], 4);
308 
309     // Now increment things incremented with old and do some new
310     Increment inc = new Increment(ROW);
311     inc.addColumn(FAMILY, QUALIFIERS[1], 1);
312     inc.addColumn(FAMILY, QUALIFIERS[3], 1);
313     inc.addColumn(FAMILY, QUALIFIERS[4], 1);
314     ht.increment(inc);
315 
316     // Verify expected results
317     Get get = new Get(ROW);
318     Result r = ht.get(get);
319     Cell [] kvs = r.rawCells();
320     assertEquals(5, kvs.length);
321     assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[0], 1);
322     assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[1], 3);
323     assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 3);
324     assertIncrementKey(kvs[3], ROW, FAMILY, QUALIFIERS[3], 5);
325     assertIncrementKey(kvs[4], ROW, FAMILY, QUALIFIERS[4], 1);
326 
327     // Now try multiple columns by different amounts
328     inc = new Increment(ROWS[0]);
329     for (int i=0;i<QUALIFIERS.length;i++) {
330       inc.addColumn(FAMILY, QUALIFIERS[i], i+1);
331     }
332     ht.increment(inc);
333     // Verify
334     get = new Get(ROWS[0]);
335     r = ht.get(get);
336     kvs = r.rawCells();
337     assertEquals(QUALIFIERS.length, kvs.length);
338     for (int i=0;i<QUALIFIERS.length;i++) {
339       assertIncrementKey(kvs[i], ROWS[0], FAMILY, QUALIFIERS[i], i+1);
340     }
341 
342     // Re-increment them
343     inc = new Increment(ROWS[0]);
344     for (int i=0;i<QUALIFIERS.length;i++) {
345       inc.addColumn(FAMILY, QUALIFIERS[i], i+1);
346     }
347     ht.increment(inc);
348     // Verify
349     r = ht.get(get);
350     kvs = r.rawCells();
351     assertEquals(QUALIFIERS.length, kvs.length);
352     for (int i=0;i<QUALIFIERS.length;i++) {
353       assertIncrementKey(kvs[i], ROWS[0], FAMILY, QUALIFIERS[i], 2*(i+1));
354     }
355 
356     // Verify that an Increment of an amount of zero, returns current count; i.e. same as for above
357     // test, that is: 2 * (i + 1).
358     inc = new Increment(ROWS[0]);
359     for (int i = 0; i < QUALIFIERS.length; i++) {
360       inc.addColumn(FAMILY, QUALIFIERS[i], 0);
361     }
362     ht.increment(inc);
363     r = ht.get(get);
364     kvs = r.rawCells();
365     assertEquals(QUALIFIERS.length, kvs.length);
366     for (int i = 0; i < QUALIFIERS.length; i++) {
367       assertIncrementKey(kvs[i], ROWS[0], FAMILY, QUALIFIERS[i], 2*(i+1));
368     }
369   }
370 
371 
372   /**
373    * Call over to the adjacent class's method of same name.
374    */
375   static void assertIncrementKey(Cell key, byte [] row, byte [] family,
376       byte [] qualifier, long value) throws Exception {
377     TestFromClientSide.assertIncrementKey(key, row, family, qualifier, value);
378   }
379 
380   public static String filterStringSoTableNameSafe(final String str) {
381     return str.replaceAll("\\[fast\\=(.*)\\]", ".FAST.is.$1");
382   }
383 }