View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements. See the NOTICE file distributed with this
4    * work for additional information regarding copyright ownership. The ASF
5    * licenses this file to you under the Apache License, Version 2.0 (the
6    * "License"); you may not use this file except in compliance with the License.
7    * You may obtain a copy of the License at
8    *
9    * http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14   * License for the specific language governing permissions and limitations
15   * under the License.
16   */
17  package org.apache.hadoop.hbase.client;
18  
19  import static org.junit.Assert.assertEquals;
20  import static org.junit.Assert.assertTrue;
21  
22  import java.util.ArrayList;
23  import java.util.List;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.hbase.Cell;
28  import org.apache.hadoop.hbase.HBaseTestingUtility;
29  import org.apache.hadoop.hbase.HRegionInfo;
30  import org.apache.hadoop.hbase.HRegionLocation;
31  import org.apache.hadoop.hbase.HTestConst;
32  import org.apache.hadoop.hbase.KeyValue;
33  import org.apache.hadoop.hbase.MediumTests;
34  import org.apache.hadoop.hbase.MiniHBaseCluster;
35  import org.apache.hadoop.hbase.filter.ColumnPrefixFilter;
36  import org.apache.hadoop.hbase.filter.ColumnRangeFilter;
37  import org.apache.hadoop.hbase.master.HMaster;
38  import org.apache.hadoop.hbase.master.RegionState.State;
39  import org.apache.hadoop.hbase.master.RegionStates;
40  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
41  import org.apache.hadoop.hbase.regionserver.HRegionServer;
42  import org.apache.hadoop.hbase.util.Bytes;
43  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
44  import org.apache.hadoop.hbase.zookeeper.ZKAssign;
45  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
46  import org.junit.After;
47  import org.junit.AfterClass;
48  import org.junit.Before;
49  import org.junit.BeforeClass;
50  import org.junit.Test;
51  import org.junit.experimental.categories.Category;
52  
53  /**
54   * A client-side test, mostly testing scanners with various parameters.
55   */
56  @Category(MediumTests.class)
57  public class TestScannersFromClientSide {
58    private static final Log LOG = LogFactory.getLog(TestScannersFromClientSide.class);
59  
60    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
61    private static byte [] ROW = Bytes.toBytes("testRow");
62    private static byte [] FAMILY = Bytes.toBytes("testFamily");
63    private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
64    private static byte [] VALUE = Bytes.toBytes("testValue");
65  
66    /**
67     * @throws java.lang.Exception
68     */
69    @BeforeClass
70    public static void setUpBeforeClass() throws Exception {
71      TEST_UTIL.startMiniCluster(3);
72    }
73  
74    /**
75     * @throws java.lang.Exception
76     */
77    @AfterClass
78    public static void tearDownAfterClass() throws Exception {
79      TEST_UTIL.shutdownMiniCluster();
80    }
81  
82    /**
83     * @throws java.lang.Exception
84     */
85    @Before
86    public void setUp() throws Exception {
87      // Nothing to do.
88    }
89  
90    /**
91     * @throws java.lang.Exception
92     */
93    @After
94    public void tearDown() throws Exception {
95      // Nothing to do.
96    }
97  
98    /**
99     * Test from client side for batch of scan
100    *
101    * @throws Exception
102    */
103   @Test
104   public void testScanBatch() throws Exception {
105     byte [] TABLE = Bytes.toBytes("testScanBatch");
106     byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 8);
107 
108     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
109 
110     Put put;
111     Scan scan;
112     Delete delete;
113     Result result;
114     ResultScanner scanner;
115     boolean toLog = true;
116     List<Cell> kvListExp;
117 
118     // table: row, family, c0:0, c1:1, ... , c7:7
119     put = new Put(ROW);
120     for (int i=0; i < QUALIFIERS.length; i++) {
121       KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[i], i, VALUE);
122       put.add(kv);
123     }
124     ht.put(put);
125 
126     // table: row, family, c0:0, c1:1, ..., c6:2, c6:6 , c7:7
127     put = new Put(ROW);
128     KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[6], 2, VALUE);
129     put.add(kv);
130     ht.put(put);
131 
132     // delete upto ts: 3
133     delete = new Delete(ROW);
134     delete.deleteFamily(FAMILY, 3);
135     ht.delete(delete);
136 
137     // without batch
138     scan = new Scan(ROW);
139     scan.setMaxVersions();
140     scanner = ht.getScanner(scan);
141 
142     // c4:4, c5:5, c6:6, c7:7
143     kvListExp = new ArrayList<Cell>();
144     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[4], 4, VALUE));
145     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[5], 5, VALUE));
146     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[6], 6, VALUE));
147     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[7], 7, VALUE));
148     result = scanner.next();
149     verifyResult(result, kvListExp, toLog, "Testing first batch of scan");
150 
151     // with batch
152     scan = new Scan(ROW);
153     scan.setMaxVersions();
154     scan.setBatch(2);
155     scanner = ht.getScanner(scan);
156 
157     // First batch: c4:4, c5:5
158     kvListExp = new ArrayList<Cell>();
159     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[4], 4, VALUE));
160     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[5], 5, VALUE));
161     result = scanner.next();
162     verifyResult(result, kvListExp, toLog, "Testing first batch of scan");
163 
164     // Second batch: c6:6, c7:7
165     kvListExp = new ArrayList<Cell>();
166     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[6], 6, VALUE));
167     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[7], 7, VALUE));
168     result = scanner.next();
169     verifyResult(result, kvListExp, toLog, "Testing second batch of scan");
170 
171   }
172 
173   /**
174    * Test from client side for get with maxResultPerCF set
175    *
176    * @throws Exception
177    */
178   @Test
179   public void testGetMaxResults() throws Exception {
180     byte [] TABLE = Bytes.toBytes("testGetMaxResults");
181     byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3);
182     byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 20);
183 
184     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES);
185 
186     Get get;
187     Put put;
188     Result result;
189     boolean toLog = true;
190     List<Cell> kvListExp;
191 
192     kvListExp = new ArrayList<Cell>();
193     // Insert one CF for row[0]
194     put = new Put(ROW);
195     for (int i=0; i < 10; i++) {
196       KeyValue kv = new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE);
197       put.add(kv);
198       kvListExp.add(kv);
199     }
200     ht.put(put);
201 
202     get = new Get(ROW);
203     result = ht.get(get);
204     verifyResult(result, kvListExp, toLog, "Testing without setting maxResults");
205 
206     get = new Get(ROW);
207     get.setMaxResultsPerColumnFamily(2);
208     result = ht.get(get);
209     kvListExp = new ArrayList<Cell>();
210     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[0], 1, VALUE));
211     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[1], 1, VALUE));
212     verifyResult(result, kvListExp, toLog, "Testing basic setMaxResults");
213 
214     // Filters: ColumnRangeFilter
215     get = new Get(ROW);
216     get.setMaxResultsPerColumnFamily(5);
217     get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, QUALIFIERS[5],
218                                         true));
219     result = ht.get(get);
220     kvListExp = new ArrayList<Cell>();
221     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[2], 1, VALUE));
222     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[3], 1, VALUE));
223     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[4], 1, VALUE));
224     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[5], 1, VALUE));
225     verifyResult(result, kvListExp, toLog, "Testing single CF with CRF");
226 
227     // Insert two more CF for row[0]
228     // 20 columns for CF2, 10 columns for CF1
229     put = new Put(ROW);
230     for (int i=0; i < QUALIFIERS.length; i++) {
231       KeyValue kv = new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE);
232       put.add(kv);
233     }
234     ht.put(put);
235 
236     put = new Put(ROW);
237     for (int i=0; i < 10; i++) {
238       KeyValue kv = new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE);
239       put.add(kv);
240     }
241     ht.put(put);
242 
243     get = new Get(ROW);
244     get.setMaxResultsPerColumnFamily(12);
245     get.addFamily(FAMILIES[1]);
246     get.addFamily(FAMILIES[2]);
247     result = ht.get(get);
248     kvListExp = new ArrayList<Cell>();
249     //Exp: CF1:q0, ..., q9, CF2: q0, q1, q10, q11, ..., q19
250     for (int i=0; i < 10; i++) {
251       kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE));
252     }
253     for (int i=0; i < 2; i++) {
254         kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
255       }
256     for (int i=10; i < 20; i++) {
257       kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
258     }
259     verifyResult(result, kvListExp, toLog, "Testing multiple CFs");
260 
261     // Filters: ColumnRangeFilter and ColumnPrefixFilter
262     get = new Get(ROW);
263     get.setMaxResultsPerColumnFamily(3);
264     get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, null, true));
265     result = ht.get(get);
266     kvListExp = new ArrayList<Cell>();
267     for (int i=2; i < 5; i++) {
268       kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE));
269     }
270     for (int i=2; i < 5; i++) {
271       kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE));
272     }
273     for (int i=2; i < 5; i++) {
274       kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
275     }
276     verifyResult(result, kvListExp, toLog, "Testing multiple CFs + CRF");
277 
278     get = new Get(ROW);
279     get.setMaxResultsPerColumnFamily(7);
280     get.setFilter(new ColumnPrefixFilter(QUALIFIERS[1]));
281     result = ht.get(get);
282     kvListExp = new ArrayList<Cell>();
283     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[1], 1, VALUE));
284     kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[1], 1, VALUE));
285     kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[1], 1, VALUE));
286     for (int i=10; i < 16; i++) {
287       kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
288     }
289     verifyResult(result, kvListExp, toLog, "Testing multiple CFs + PFF");
290 
291   }
292 
293   /**
294    * Test from client side for scan with maxResultPerCF set
295    *
296    * @throws Exception
297    */
298   @Test
299   public void testScanMaxResults() throws Exception {
300     byte [] TABLE = Bytes.toBytes("testScanLimit");
301     byte [][] ROWS = HTestConst.makeNAscii(ROW, 2);
302     byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3);
303     byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 10);
304 
305     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES);
306 
307     Put put;
308     Scan scan;
309     Result result;
310     boolean toLog = true;
311     List<Cell> kvListExp, kvListScan;
312 
313     kvListExp = new ArrayList<Cell>();
314 
315     for (int r=0; r < ROWS.length; r++) {
316       put = new Put(ROWS[r]);
317       for (int c=0; c < FAMILIES.length; c++) {
318         for (int q=0; q < QUALIFIERS.length; q++) {
319           KeyValue kv = new KeyValue(ROWS[r], FAMILIES[c], QUALIFIERS[q], 1, VALUE);
320           put.add(kv);
321           if (q < 4) {
322             kvListExp.add(kv);
323           }
324         }
325       }
326       ht.put(put);
327     }
328 
329     scan = new Scan();
330     scan.setMaxResultsPerColumnFamily(4);
331     ResultScanner scanner = ht.getScanner(scan);
332     kvListScan = new ArrayList<Cell>();
333     while ((result = scanner.next()) != null) {
334       for (Cell kv : result.listCells()) {
335         kvListScan.add(kv);
336       }
337     }
338     result = Result.create(kvListScan);
339     verifyResult(result, kvListExp, toLog, "Testing scan with maxResults");
340 
341   }
342 
343   /**
344    * Test from client side for get with rowOffset
345    *
346    * @throws Exception
347    */
348   @Test
349   public void testGetRowOffset() throws Exception {
350     byte [] TABLE = Bytes.toBytes("testGetRowOffset");
351     byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3);
352     byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 20);
353 
354     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES);
355 
356     Get get;
357     Put put;
358     Result result;
359     boolean toLog = true;
360     List<Cell> kvListExp;
361 
362     // Insert one CF for row
363     kvListExp = new ArrayList<Cell>();
364     put = new Put(ROW);
365     for (int i=0; i < 10; i++) {
366       KeyValue kv = new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE);
367       put.add(kv);
368       // skipping first two kvs
369       if (i < 2) continue;
370       kvListExp.add(kv);
371     }
372     ht.put(put);
373 
374     //setting offset to 2
375     get = new Get(ROW);
376     get.setRowOffsetPerColumnFamily(2);
377     result = ht.get(get);
378     verifyResult(result, kvListExp, toLog, "Testing basic setRowOffset");
379 
380     //setting offset to 20
381     get = new Get(ROW);
382     get.setRowOffsetPerColumnFamily(20);
383     result = ht.get(get);
384     kvListExp = new ArrayList<Cell>();
385     verifyResult(result, kvListExp, toLog, "Testing offset > #kvs");
386 
387     //offset + maxResultPerCF
388     get = new Get(ROW);
389     get.setRowOffsetPerColumnFamily(4);
390     get.setMaxResultsPerColumnFamily(5);
391     result = ht.get(get);
392     kvListExp = new ArrayList<Cell>();
393     for (int i=4; i < 9; i++) {
394       kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE));
395     }
396     verifyResult(result, kvListExp, toLog,
397       "Testing offset + setMaxResultsPerCF");
398 
399     // Filters: ColumnRangeFilter
400     get = new Get(ROW);
401     get.setRowOffsetPerColumnFamily(1);
402     get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, QUALIFIERS[5],
403                                         true));
404     result = ht.get(get);
405     kvListExp = new ArrayList<Cell>();
406     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[3], 1, VALUE));
407     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[4], 1, VALUE));
408     kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[5], 1, VALUE));
409     verifyResult(result, kvListExp, toLog, "Testing offset with CRF");
410 
411     // Insert into two more CFs for row
412     // 10 columns for CF2, 10 columns for CF1
413     for(int j=2; j > 0; j--) {
414       put = new Put(ROW);
415       for (int i=0; i < 10; i++) {
416         KeyValue kv = new KeyValue(ROW, FAMILIES[j], QUALIFIERS[i], 1, VALUE);
417         put.add(kv);
418       }
419       ht.put(put);
420     }
421 
422     get = new Get(ROW);
423     get.setRowOffsetPerColumnFamily(4);
424     get.setMaxResultsPerColumnFamily(2);
425     get.addFamily(FAMILIES[1]);
426     get.addFamily(FAMILIES[2]);
427     result = ht.get(get);
428     kvListExp = new ArrayList<Cell>();
429     //Exp: CF1:q4, q5, CF2: q4, q5
430     kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[4], 1, VALUE));
431     kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[5], 1, VALUE));
432     kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[4], 1, VALUE));
433     kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[5], 1, VALUE));
434     verifyResult(result, kvListExp, toLog,
435        "Testing offset + multiple CFs + maxResults");
436   }
437 
438   /**
439    * Test from client side for scan while the region is reopened
440    * on the same region server.
441    *
442    * @throws Exception
443    */
444   @Test
445   public void testScanOnReopenedRegion() throws Exception {
446     byte [] TABLE = Bytes.toBytes("testScanOnReopenedRegion");
447     byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 2);
448 
449     HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
450 
451     Put put;
452     Scan scan;
453     Result result;
454     ResultScanner scanner;
455     boolean toLog = false;
456     List<Cell> kvListExp;
457 
458     // table: row, family, c0:0, c1:1
459     put = new Put(ROW);
460     for (int i=0; i < QUALIFIERS.length; i++) {
461       KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[i], i, VALUE);
462       put.add(kv);
463     }
464     ht.put(put);
465 
466     scan = new Scan(ROW);
467     scanner = ht.getScanner(scan);
468 
469     HRegionLocation loc = ht.getRegionLocation(ROW);
470     HRegionInfo hri = loc.getRegionInfo();
471     MiniHBaseCluster cluster = TEST_UTIL.getMiniHBaseCluster();
472     byte[] regionName = hri.getRegionName();
473     int i = cluster.getServerWith(regionName);
474     HRegionServer rs = cluster.getRegionServer(i);
475     ProtobufUtil.closeRegion(rs, rs.getServerName(), regionName, false);
476     long startTime = EnvironmentEdgeManager.currentTimeMillis();
477     long timeOut = 300000;
478     while (true) {
479       if (rs.getOnlineRegion(regionName) == null) {
480         break;
481       }
482       assertTrue("Timed out in closing the testing region",
483         EnvironmentEdgeManager.currentTimeMillis() < startTime + timeOut);
484       Thread.sleep(500);
485     }
486 
487     // Now open the region again.
488     ZooKeeperWatcher zkw = TEST_UTIL.getZooKeeperWatcher();
489     try {
490       HMaster master = cluster.getMaster();
491       RegionStates states = master.getAssignmentManager().getRegionStates();
492       states.regionOffline(hri);
493       states.updateRegionState(hri, State.OPENING);
494       ZKAssign.createNodeOffline(zkw, hri, loc.getServerName());
495       ProtobufUtil.openRegion(rs, rs.getServerName(), hri);
496       startTime = EnvironmentEdgeManager.currentTimeMillis();
497       while (true) {
498         if (rs.getOnlineRegion(regionName) != null) {
499           break;
500         }
501         assertTrue("Timed out in open the testing region",
502           EnvironmentEdgeManager.currentTimeMillis() < startTime + timeOut);
503         Thread.sleep(500);
504       }
505     } finally {
506       ZKAssign.deleteNodeFailSilent(zkw, hri);
507     }
508 
509     // c0:0, c1:1
510     kvListExp = new ArrayList<Cell>();
511     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[0], 0, VALUE));
512     kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[1], 1, VALUE));
513     result = scanner.next();
514     verifyResult(result, kvListExp, toLog, "Testing scan on re-opened region");
515   }
516 
517   static void verifyResult(Result result, List<Cell> expKvList, boolean toLog,
518       String msg) {
519 
520     LOG.info(msg);
521     LOG.info("Expected count: " + expKvList.size());
522     LOG.info("Actual count: " + result.size());
523     if (expKvList.size() == 0)
524       return;
525 
526     int i = 0;
527     for (Cell kv : result.rawCells()) {
528       if (i >= expKvList.size()) {
529         break;  // we will check the size later
530       }
531 
532       Cell kvExp = expKvList.get(i++);
533       if (toLog) {
534         LOG.info("get kv is: " + kv.toString());
535         LOG.info("exp kv is: " + kvExp.toString());
536       }
537       assertTrue("Not equal", kvExp.equals(kv));
538     }
539 
540     assertEquals(expKvList.size(), result.size());
541   }
542 
543 
544 }