1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.hadoop.hbase.client;
18
19 import static org.junit.Assert.assertEquals;
20 import static org.junit.Assert.assertTrue;
21
22 import java.util.ArrayList;
23 import java.util.List;
24
25 import org.apache.commons.logging.Log;
26 import org.apache.commons.logging.LogFactory;
27 import org.apache.hadoop.hbase.Cell;
28 import org.apache.hadoop.hbase.HBaseTestingUtility;
29 import org.apache.hadoop.hbase.HRegionInfo;
30 import org.apache.hadoop.hbase.HRegionLocation;
31 import org.apache.hadoop.hbase.HTestConst;
32 import org.apache.hadoop.hbase.KeyValue;
33 import org.apache.hadoop.hbase.MediumTests;
34 import org.apache.hadoop.hbase.MiniHBaseCluster;
35 import org.apache.hadoop.hbase.filter.ColumnPrefixFilter;
36 import org.apache.hadoop.hbase.filter.ColumnRangeFilter;
37 import org.apache.hadoop.hbase.master.HMaster;
38 import org.apache.hadoop.hbase.master.RegionState.State;
39 import org.apache.hadoop.hbase.master.RegionStates;
40 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
41 import org.apache.hadoop.hbase.regionserver.HRegionServer;
42 import org.apache.hadoop.hbase.util.Bytes;
43 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
44 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
45 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
46 import org.junit.After;
47 import org.junit.AfterClass;
48 import org.junit.Before;
49 import org.junit.BeforeClass;
50 import org.junit.Test;
51 import org.junit.experimental.categories.Category;
52
53
54
55
56 @Category(MediumTests.class)
57 public class TestScannersFromClientSide {
58 private static final Log LOG = LogFactory.getLog(TestScannersFromClientSide.class);
59
60 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
61 private static byte [] ROW = Bytes.toBytes("testRow");
62 private static byte [] FAMILY = Bytes.toBytes("testFamily");
63 private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
64 private static byte [] VALUE = Bytes.toBytes("testValue");
65
66
67
68
69 @BeforeClass
70 public static void setUpBeforeClass() throws Exception {
71 TEST_UTIL.startMiniCluster(3);
72 }
73
74
75
76
77 @AfterClass
78 public static void tearDownAfterClass() throws Exception {
79 TEST_UTIL.shutdownMiniCluster();
80 }
81
82
83
84
85 @Before
86 public void setUp() throws Exception {
87
88 }
89
90
91
92
93 @After
94 public void tearDown() throws Exception {
95
96 }
97
98
99
100
101
102
103 @Test
104 public void testScanBatch() throws Exception {
105 byte [] TABLE = Bytes.toBytes("testScanBatch");
106 byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 8);
107
108 HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
109
110 Put put;
111 Scan scan;
112 Delete delete;
113 Result result;
114 ResultScanner scanner;
115 boolean toLog = true;
116 List<Cell> kvListExp;
117
118
119 put = new Put(ROW);
120 for (int i=0; i < QUALIFIERS.length; i++) {
121 KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[i], i, VALUE);
122 put.add(kv);
123 }
124 ht.put(put);
125
126
127 put = new Put(ROW);
128 KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[6], 2, VALUE);
129 put.add(kv);
130 ht.put(put);
131
132
133 delete = new Delete(ROW);
134 delete.deleteFamily(FAMILY, 3);
135 ht.delete(delete);
136
137
138 scan = new Scan(ROW);
139 scan.setMaxVersions();
140 scanner = ht.getScanner(scan);
141
142
143 kvListExp = new ArrayList<Cell>();
144 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[4], 4, VALUE));
145 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[5], 5, VALUE));
146 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[6], 6, VALUE));
147 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[7], 7, VALUE));
148 result = scanner.next();
149 verifyResult(result, kvListExp, toLog, "Testing first batch of scan");
150
151
152 scan = new Scan(ROW);
153 scan.setMaxVersions();
154 scan.setBatch(2);
155 scanner = ht.getScanner(scan);
156
157
158 kvListExp = new ArrayList<Cell>();
159 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[4], 4, VALUE));
160 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[5], 5, VALUE));
161 result = scanner.next();
162 verifyResult(result, kvListExp, toLog, "Testing first batch of scan");
163
164
165 kvListExp = new ArrayList<Cell>();
166 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[6], 6, VALUE));
167 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[7], 7, VALUE));
168 result = scanner.next();
169 verifyResult(result, kvListExp, toLog, "Testing second batch of scan");
170
171 }
172
173
174
175
176
177
178 @Test
179 public void testGetMaxResults() throws Exception {
180 byte [] TABLE = Bytes.toBytes("testGetMaxResults");
181 byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3);
182 byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 20);
183
184 HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES);
185
186 Get get;
187 Put put;
188 Result result;
189 boolean toLog = true;
190 List<Cell> kvListExp;
191
192 kvListExp = new ArrayList<Cell>();
193
194 put = new Put(ROW);
195 for (int i=0; i < 10; i++) {
196 KeyValue kv = new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE);
197 put.add(kv);
198 kvListExp.add(kv);
199 }
200 ht.put(put);
201
202 get = new Get(ROW);
203 result = ht.get(get);
204 verifyResult(result, kvListExp, toLog, "Testing without setting maxResults");
205
206 get = new Get(ROW);
207 get.setMaxResultsPerColumnFamily(2);
208 result = ht.get(get);
209 kvListExp = new ArrayList<Cell>();
210 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[0], 1, VALUE));
211 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[1], 1, VALUE));
212 verifyResult(result, kvListExp, toLog, "Testing basic setMaxResults");
213
214
215 get = new Get(ROW);
216 get.setMaxResultsPerColumnFamily(5);
217 get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, QUALIFIERS[5],
218 true));
219 result = ht.get(get);
220 kvListExp = new ArrayList<Cell>();
221 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[2], 1, VALUE));
222 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[3], 1, VALUE));
223 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[4], 1, VALUE));
224 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[5], 1, VALUE));
225 verifyResult(result, kvListExp, toLog, "Testing single CF with CRF");
226
227
228
229 put = new Put(ROW);
230 for (int i=0; i < QUALIFIERS.length; i++) {
231 KeyValue kv = new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE);
232 put.add(kv);
233 }
234 ht.put(put);
235
236 put = new Put(ROW);
237 for (int i=0; i < 10; i++) {
238 KeyValue kv = new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE);
239 put.add(kv);
240 }
241 ht.put(put);
242
243 get = new Get(ROW);
244 get.setMaxResultsPerColumnFamily(12);
245 get.addFamily(FAMILIES[1]);
246 get.addFamily(FAMILIES[2]);
247 result = ht.get(get);
248 kvListExp = new ArrayList<Cell>();
249
250 for (int i=0; i < 10; i++) {
251 kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE));
252 }
253 for (int i=0; i < 2; i++) {
254 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
255 }
256 for (int i=10; i < 20; i++) {
257 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
258 }
259 verifyResult(result, kvListExp, toLog, "Testing multiple CFs");
260
261
262 get = new Get(ROW);
263 get.setMaxResultsPerColumnFamily(3);
264 get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, null, true));
265 result = ht.get(get);
266 kvListExp = new ArrayList<Cell>();
267 for (int i=2; i < 5; i++) {
268 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE));
269 }
270 for (int i=2; i < 5; i++) {
271 kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[i], 1, VALUE));
272 }
273 for (int i=2; i < 5; i++) {
274 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
275 }
276 verifyResult(result, kvListExp, toLog, "Testing multiple CFs + CRF");
277
278 get = new Get(ROW);
279 get.setMaxResultsPerColumnFamily(7);
280 get.setFilter(new ColumnPrefixFilter(QUALIFIERS[1]));
281 result = ht.get(get);
282 kvListExp = new ArrayList<Cell>();
283 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[1], 1, VALUE));
284 kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[1], 1, VALUE));
285 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[1], 1, VALUE));
286 for (int i=10; i < 16; i++) {
287 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[i], 1, VALUE));
288 }
289 verifyResult(result, kvListExp, toLog, "Testing multiple CFs + PFF");
290
291 }
292
293
294
295
296
297
298 @Test
299 public void testScanMaxResults() throws Exception {
300 byte [] TABLE = Bytes.toBytes("testScanLimit");
301 byte [][] ROWS = HTestConst.makeNAscii(ROW, 2);
302 byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3);
303 byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 10);
304
305 HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES);
306
307 Put put;
308 Scan scan;
309 Result result;
310 boolean toLog = true;
311 List<Cell> kvListExp, kvListScan;
312
313 kvListExp = new ArrayList<Cell>();
314
315 for (int r=0; r < ROWS.length; r++) {
316 put = new Put(ROWS[r]);
317 for (int c=0; c < FAMILIES.length; c++) {
318 for (int q=0; q < QUALIFIERS.length; q++) {
319 KeyValue kv = new KeyValue(ROWS[r], FAMILIES[c], QUALIFIERS[q], 1, VALUE);
320 put.add(kv);
321 if (q < 4) {
322 kvListExp.add(kv);
323 }
324 }
325 }
326 ht.put(put);
327 }
328
329 scan = new Scan();
330 scan.setMaxResultsPerColumnFamily(4);
331 ResultScanner scanner = ht.getScanner(scan);
332 kvListScan = new ArrayList<Cell>();
333 while ((result = scanner.next()) != null) {
334 for (Cell kv : result.listCells()) {
335 kvListScan.add(kv);
336 }
337 }
338 result = Result.create(kvListScan);
339 verifyResult(result, kvListExp, toLog, "Testing scan with maxResults");
340
341 }
342
343
344
345
346
347
348 @Test
349 public void testGetRowOffset() throws Exception {
350 byte [] TABLE = Bytes.toBytes("testGetRowOffset");
351 byte [][] FAMILIES = HTestConst.makeNAscii(FAMILY, 3);
352 byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 20);
353
354 HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES);
355
356 Get get;
357 Put put;
358 Result result;
359 boolean toLog = true;
360 List<Cell> kvListExp;
361
362
363 kvListExp = new ArrayList<Cell>();
364 put = new Put(ROW);
365 for (int i=0; i < 10; i++) {
366 KeyValue kv = new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE);
367 put.add(kv);
368
369 if (i < 2) continue;
370 kvListExp.add(kv);
371 }
372 ht.put(put);
373
374
375 get = new Get(ROW);
376 get.setRowOffsetPerColumnFamily(2);
377 result = ht.get(get);
378 verifyResult(result, kvListExp, toLog, "Testing basic setRowOffset");
379
380
381 get = new Get(ROW);
382 get.setRowOffsetPerColumnFamily(20);
383 result = ht.get(get);
384 kvListExp = new ArrayList<Cell>();
385 verifyResult(result, kvListExp, toLog, "Testing offset > #kvs");
386
387
388 get = new Get(ROW);
389 get.setRowOffsetPerColumnFamily(4);
390 get.setMaxResultsPerColumnFamily(5);
391 result = ht.get(get);
392 kvListExp = new ArrayList<Cell>();
393 for (int i=4; i < 9; i++) {
394 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[i], 1, VALUE));
395 }
396 verifyResult(result, kvListExp, toLog,
397 "Testing offset + setMaxResultsPerCF");
398
399
400 get = new Get(ROW);
401 get.setRowOffsetPerColumnFamily(1);
402 get.setFilter(new ColumnRangeFilter(QUALIFIERS[2], true, QUALIFIERS[5],
403 true));
404 result = ht.get(get);
405 kvListExp = new ArrayList<Cell>();
406 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[3], 1, VALUE));
407 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[4], 1, VALUE));
408 kvListExp.add(new KeyValue(ROW, FAMILIES[0], QUALIFIERS[5], 1, VALUE));
409 verifyResult(result, kvListExp, toLog, "Testing offset with CRF");
410
411
412
413 for(int j=2; j > 0; j--) {
414 put = new Put(ROW);
415 for (int i=0; i < 10; i++) {
416 KeyValue kv = new KeyValue(ROW, FAMILIES[j], QUALIFIERS[i], 1, VALUE);
417 put.add(kv);
418 }
419 ht.put(put);
420 }
421
422 get = new Get(ROW);
423 get.setRowOffsetPerColumnFamily(4);
424 get.setMaxResultsPerColumnFamily(2);
425 get.addFamily(FAMILIES[1]);
426 get.addFamily(FAMILIES[2]);
427 result = ht.get(get);
428 kvListExp = new ArrayList<Cell>();
429
430 kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[4], 1, VALUE));
431 kvListExp.add(new KeyValue(ROW, FAMILIES[1], QUALIFIERS[5], 1, VALUE));
432 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[4], 1, VALUE));
433 kvListExp.add(new KeyValue(ROW, FAMILIES[2], QUALIFIERS[5], 1, VALUE));
434 verifyResult(result, kvListExp, toLog,
435 "Testing offset + multiple CFs + maxResults");
436 }
437
438
439
440
441
442
443
444 @Test
445 public void testScanOnReopenedRegion() throws Exception {
446 byte [] TABLE = Bytes.toBytes("testScanOnReopenedRegion");
447 byte [][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, 2);
448
449 HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
450
451 Put put;
452 Scan scan;
453 Result result;
454 ResultScanner scanner;
455 boolean toLog = false;
456 List<Cell> kvListExp;
457
458
459 put = new Put(ROW);
460 for (int i=0; i < QUALIFIERS.length; i++) {
461 KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[i], i, VALUE);
462 put.add(kv);
463 }
464 ht.put(put);
465
466 scan = new Scan(ROW);
467 scanner = ht.getScanner(scan);
468
469 HRegionLocation loc = ht.getRegionLocation(ROW);
470 HRegionInfo hri = loc.getRegionInfo();
471 MiniHBaseCluster cluster = TEST_UTIL.getMiniHBaseCluster();
472 byte[] regionName = hri.getRegionName();
473 int i = cluster.getServerWith(regionName);
474 HRegionServer rs = cluster.getRegionServer(i);
475 ProtobufUtil.closeRegion(rs, rs.getServerName(), regionName, false);
476 long startTime = EnvironmentEdgeManager.currentTimeMillis();
477 long timeOut = 300000;
478 while (true) {
479 if (rs.getOnlineRegion(regionName) == null) {
480 break;
481 }
482 assertTrue("Timed out in closing the testing region",
483 EnvironmentEdgeManager.currentTimeMillis() < startTime + timeOut);
484 Thread.sleep(500);
485 }
486
487
488 ZooKeeperWatcher zkw = TEST_UTIL.getZooKeeperWatcher();
489 try {
490 HMaster master = cluster.getMaster();
491 RegionStates states = master.getAssignmentManager().getRegionStates();
492 states.regionOffline(hri);
493 states.updateRegionState(hri, State.OPENING);
494 ZKAssign.createNodeOffline(zkw, hri, loc.getServerName());
495 ProtobufUtil.openRegion(rs, rs.getServerName(), hri);
496 startTime = EnvironmentEdgeManager.currentTimeMillis();
497 while (true) {
498 if (rs.getOnlineRegion(regionName) != null) {
499 break;
500 }
501 assertTrue("Timed out in open the testing region",
502 EnvironmentEdgeManager.currentTimeMillis() < startTime + timeOut);
503 Thread.sleep(500);
504 }
505 } finally {
506 ZKAssign.deleteNodeFailSilent(zkw, hri);
507 }
508
509
510 kvListExp = new ArrayList<Cell>();
511 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[0], 0, VALUE));
512 kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[1], 1, VALUE));
513 result = scanner.next();
514 verifyResult(result, kvListExp, toLog, "Testing scan on re-opened region");
515 }
516
517 static void verifyResult(Result result, List<Cell> expKvList, boolean toLog,
518 String msg) {
519
520 LOG.info(msg);
521 LOG.info("Expected count: " + expKvList.size());
522 LOG.info("Actual count: " + result.size());
523 if (expKvList.size() == 0)
524 return;
525
526 int i = 0;
527 for (Cell kv : result.rawCells()) {
528 if (i >= expKvList.size()) {
529 break;
530 }
531
532 Cell kvExp = expKvList.get(i++);
533 if (toLog) {
534 LOG.info("get kv is: " + kv.toString());
535 LOG.info("exp kv is: " + kvExp.toString());
536 }
537 assertTrue("Not equal", kvExp.equals(kv));
538 }
539
540 assertEquals(expKvList.size(), result.size());
541 }
542
543
544 }