1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.rest;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertFalse;
22 import static org.junit.Assert.assertNotNull;
23 import static org.junit.Assert.assertTrue;
24
25 import java.io.DataInputStream;
26 import java.io.EOFException;
27 import java.io.IOException;
28 import java.io.InputStream;
29 import java.io.Serializable;
30 import java.util.ArrayList;
31 import java.util.List;
32
33 import javax.ws.rs.core.MediaType;
34 import javax.xml.bind.JAXBContext;
35 import javax.xml.bind.JAXBException;
36 import javax.xml.bind.Unmarshaller;
37 import javax.xml.bind.annotation.XmlAccessType;
38 import javax.xml.bind.annotation.XmlAccessorType;
39 import javax.xml.bind.annotation.XmlElement;
40 import javax.xml.bind.annotation.XmlRootElement;
41 import javax.xml.parsers.SAXParserFactory;
42 import javax.xml.stream.XMLStreamException;
43
44 import org.apache.hadoop.conf.Configuration;
45 import org.apache.hadoop.hbase.HBaseTestingUtility;
46 import org.apache.hadoop.hbase.HColumnDescriptor;
47 import org.apache.hadoop.hbase.HTableDescriptor;
48 import org.apache.hadoop.hbase.MediumTests;
49 import org.apache.hadoop.hbase.TableName;
50 import org.apache.hadoop.hbase.client.HBaseAdmin;
51 import org.apache.hadoop.hbase.rest.client.Client;
52 import org.apache.hadoop.hbase.rest.client.Cluster;
53 import org.apache.hadoop.hbase.rest.client.Response;
54 import org.apache.hadoop.hbase.rest.model.CellModel;
55 import org.apache.hadoop.hbase.rest.model.CellSetModel;
56 import org.apache.hadoop.hbase.rest.model.RowModel;
57 import org.apache.hadoop.hbase.rest.provider.JacksonProvider;
58 import org.apache.hadoop.hbase.util.Bytes;
59 import org.codehaus.jackson.JsonFactory;
60 import org.codehaus.jackson.JsonParser;
61 import org.codehaus.jackson.JsonToken;
62 import org.codehaus.jackson.map.ObjectMapper;
63 import org.junit.AfterClass;
64 import org.junit.BeforeClass;
65 import org.junit.Test;
66 import org.junit.experimental.categories.Category;
67 import org.xml.sax.InputSource;
68 import org.xml.sax.XMLReader;
69
70 @Category(MediumTests.class)
71 public class TestTableScan {
72
73 private static final String TABLE = "TestScanResource";
74 private static final String CFA = "a";
75 private static final String CFB = "b";
76 private static final String COLUMN_1 = CFA + ":1";
77 private static final String COLUMN_2 = CFB + ":2";
78 private static Client client;
79 private static int expectedRows1;
80 private static int expectedRows2;
81 private static Configuration conf;
82
83 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
84 private static final HBaseRESTTestingUtility REST_TEST_UTIL =
85 new HBaseRESTTestingUtility();
86
87 @BeforeClass
88 public static void setUpBeforeClass() throws Exception {
89 conf = TEST_UTIL.getConfiguration();
90 TEST_UTIL.startMiniCluster();
91 REST_TEST_UTIL.startServletContainer(conf);
92 client = new Client(new Cluster().add("localhost",
93 REST_TEST_UTIL.getServletPort()));
94 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
95 if (!admin.tableExists(TABLE)) {
96 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
97 htd.addFamily(new HColumnDescriptor(CFA));
98 htd.addFamily(new HColumnDescriptor(CFB));
99 admin.createTable(htd);
100 expectedRows1 = TestScannerResource.insertData(conf, TABLE, COLUMN_1, 1.0);
101 expectedRows2 = TestScannerResource.insertData(conf, TABLE, COLUMN_2, 0.5);
102 }
103 }
104
105 @AfterClass
106 public static void tearDownAfterClass() throws Exception {
107 TEST_UTIL.getHBaseAdmin().disableTable(TABLE);
108 TEST_UTIL.getHBaseAdmin().deleteTable(TABLE);
109 REST_TEST_UTIL.shutdownServletContainer();
110 TEST_UTIL.shutdownMiniCluster();
111 }
112
113 @Test
114 public void testSimpleScannerXML() throws IOException, JAXBException, XMLStreamException {
115
116 StringBuilder builder = new StringBuilder();
117 builder.append("/*");
118 builder.append("?");
119 builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
120 builder.append("&");
121 builder.append(Constants.SCAN_LIMIT + "=10");
122 Response response = client.get("/" + TABLE + builder.toString(),
123 Constants.MIMETYPE_XML);
124 assertEquals(200, response.getCode());
125 assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
126 JAXBContext ctx = JAXBContext.newInstance(CellSetModel.class);
127 Unmarshaller ush = ctx.createUnmarshaller();
128 CellSetModel model = (CellSetModel) ush.unmarshal(response.getStream());
129 int count = TestScannerResource.countCellSet(model);
130 assertEquals(10, count);
131 checkRowsNotNull(model);
132
133
134 builder = new StringBuilder();
135 builder.append("/*");
136 builder.append("?");
137 builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
138 response = client.get("/" + TABLE + builder.toString(),
139 Constants.MIMETYPE_XML);
140 assertEquals(200, response.getCode());
141 assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
142 model = (CellSetModel) ush.unmarshal(response.getStream());
143 count = TestScannerResource.countCellSet(model);
144 assertEquals(expectedRows1, count);
145 checkRowsNotNull(model);
146
147
148 builder = new StringBuilder();
149 builder.append("/*");
150 builder.append("?");
151 builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
152 builder.append("&");
153 builder.append(Constants.SCAN_START_ROW + "=aaa");
154 builder.append("&");
155 builder.append(Constants.SCAN_END_ROW + "=aay");
156 response = client.get("/" + TABLE + builder.toString(),
157 Constants.MIMETYPE_XML);
158 assertEquals(200, response.getCode());
159 model = (CellSetModel) ush.unmarshal(response.getStream());
160 count = TestScannerResource.countCellSet(model);
161 RowModel startRow = model.getRows().get(0);
162 assertEquals("aaa", Bytes.toString(startRow.getKey()));
163 RowModel endRow = model.getRows().get(model.getRows().size() - 1);
164 assertEquals("aax", Bytes.toString(endRow.getKey()));
165 assertEquals(24, count);
166 checkRowsNotNull(model);
167
168
169 builder = new StringBuilder();
170 builder.append("/*");
171 builder.append("?");
172 builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
173 builder.append("&");
174 builder.append(Constants.SCAN_START_ROW + "=aaa");
175 builder.append("&");
176 builder.append(Constants.SCAN_LIMIT + "=15");
177 response = client.get("/" + TABLE + builder.toString(),
178 Constants.MIMETYPE_XML);
179 assertEquals(200, response.getCode());
180 assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
181 model = (CellSetModel) ush.unmarshal(response.getStream());
182 startRow = model.getRows().get(0);
183 assertEquals("aaa", Bytes.toString(startRow.getKey()));
184 count = TestScannerResource.countCellSet(model);
185 assertEquals(15, count);
186 checkRowsNotNull(model);
187
188 }
189
190 @Test
191 public void testSimpleScannerJson() throws IOException, JAXBException {
192
193 StringBuilder builder = new StringBuilder();
194 builder.append("/*");
195 builder.append("?");
196 builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
197 builder.append("&");
198 builder.append(Constants.SCAN_LIMIT + "=20");
199 Response response = client.get("/" + TABLE + builder.toString(),
200 Constants.MIMETYPE_JSON);
201 assertEquals(200, response.getCode());
202 assertEquals(Constants.MIMETYPE_JSON, response.getHeader("content-type"));
203 ObjectMapper mapper = new JacksonProvider()
204 .locateMapper(CellSetModel.class, MediaType.APPLICATION_JSON_TYPE);
205 CellSetModel model = mapper.readValue(response.getStream(), CellSetModel.class);
206 int count = TestScannerResource.countCellSet(model);
207 assertEquals(20, count);
208 checkRowsNotNull(model);
209
210
211 builder = new StringBuilder();
212 builder.append("/*");
213 builder.append("?");
214 builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_2);
215 response = client.get("/" + TABLE + builder.toString(),
216 Constants.MIMETYPE_JSON);
217 assertEquals(200, response.getCode());
218 assertEquals(Constants.MIMETYPE_JSON, response.getHeader("content-type"));
219 model = mapper.readValue(response.getStream(), CellSetModel.class);
220 count = TestScannerResource.countCellSet(model);
221 assertEquals(expectedRows2, count);
222 checkRowsNotNull(model);
223
224
225 builder = new StringBuilder();
226 builder.append("/*");
227 builder.append("?");
228 builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
229 builder.append("&");
230 builder.append(Constants.SCAN_START_ROW + "=aaa");
231 builder.append("&");
232 builder.append(Constants.SCAN_END_ROW + "=aay");
233 response = client.get("/" + TABLE + builder.toString(),
234 Constants.MIMETYPE_JSON);
235 assertEquals(200, response.getCode());
236 model = mapper.readValue(response.getStream(), CellSetModel.class);
237 RowModel startRow = model.getRows().get(0);
238 assertEquals("aaa", Bytes.toString(startRow.getKey()));
239 RowModel endRow = model.getRows().get(model.getRows().size() - 1);
240 assertEquals("aax", Bytes.toString(endRow.getKey()));
241 count = TestScannerResource.countCellSet(model);
242 assertEquals(24, count);
243 checkRowsNotNull(model);
244 }
245
246
247
248
249
250 @Test
251 public void testScanUsingListenerUnmarshallerXML() throws Exception {
252 StringBuilder builder = new StringBuilder();
253 builder.append("/*");
254 builder.append("?");
255 builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
256 builder.append("&");
257 builder.append(Constants.SCAN_LIMIT + "=10");
258 Response response = client.get("/" + TABLE + builder.toString(),
259 Constants.MIMETYPE_XML);
260 assertEquals(200, response.getCode());
261 assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
262 JAXBContext context = JAXBContext.newInstance(ClientSideCellSetModel.class, RowModel.class,
263 CellModel.class);
264 Unmarshaller unmarshaller = context.createUnmarshaller();
265
266 final ClientSideCellSetModel.Listener listener = new ClientSideCellSetModel.Listener() {
267 @Override
268 public void handleRowModel(ClientSideCellSetModel helper, RowModel row) {
269 assertTrue(row.getKey() != null);
270 assertTrue(row.getCells().size() > 0);
271 }
272 };
273
274
275 unmarshaller.setListener(new Unmarshaller.Listener() {
276 public void beforeUnmarshal(Object target, Object parent) {
277 if (target instanceof ClientSideCellSetModel) {
278 ((ClientSideCellSetModel) target).setCellSetModelListener(listener);
279 }
280 }
281
282 public void afterUnmarshal(Object target, Object parent) {
283 if (target instanceof ClientSideCellSetModel) {
284 ((ClientSideCellSetModel) target).setCellSetModelListener(null);
285 }
286 }
287 });
288
289
290 SAXParserFactory factory = SAXParserFactory.newInstance();
291 factory.setNamespaceAware(true);
292 XMLReader reader = factory.newSAXParser().getXMLReader();
293 reader.setContentHandler(unmarshaller.getUnmarshallerHandler());
294 assertFalse(ClientSideCellSetModel.listenerInvoked);
295 reader.parse(new InputSource(response.getStream()));
296 assertTrue(ClientSideCellSetModel.listenerInvoked);
297
298 }
299
300 @Test
301 public void testStreamingJSON() throws Exception {
302
303 StringBuilder builder = new StringBuilder();
304 builder.append("/*");
305 builder.append("?");
306 builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
307 builder.append("&");
308 builder.append(Constants.SCAN_LIMIT + "=20");
309 Response response = client.get("/" + TABLE + builder.toString(),
310 Constants.MIMETYPE_JSON);
311 assertEquals(200, response.getCode());
312 assertEquals(Constants.MIMETYPE_JSON, response.getHeader("content-type"));
313 ObjectMapper mapper = new JacksonProvider()
314 .locateMapper(CellSetModel.class, MediaType.APPLICATION_JSON_TYPE);
315 CellSetModel model = mapper.readValue(response.getStream(), CellSetModel.class);
316 int count = TestScannerResource.countCellSet(model);
317 assertEquals(20, count);
318 checkRowsNotNull(model);
319
320
321 builder = new StringBuilder();
322 builder.append("/*");
323 builder.append("?");
324 builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_2);
325 response = client.get("/" + TABLE + builder.toString(),
326 Constants.MIMETYPE_JSON);
327 assertEquals(200, response.getCode());
328 assertEquals(Constants.MIMETYPE_JSON, response.getHeader("content-type"));
329 model = mapper.readValue(response.getStream(), CellSetModel.class);
330 count = TestScannerResource.countCellSet(model);
331 assertEquals(expectedRows2, count);
332 checkRowsNotNull(model);
333
334
335 builder = new StringBuilder();
336 builder.append("/*");
337 builder.append("?");
338 builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
339 builder.append("&");
340 builder.append(Constants.SCAN_START_ROW + "=aaa");
341 builder.append("&");
342 builder.append(Constants.SCAN_END_ROW + "=aay");
343 response = client.get("/" + TABLE + builder.toString(),
344 Constants.MIMETYPE_JSON);
345 assertEquals(200, response.getCode());
346
347 count = 0;
348 JsonFactory jfactory = new JsonFactory(mapper);
349 JsonParser jParser = jfactory.createJsonParser(response.getStream());
350 boolean found = false;
351 while (jParser.nextToken() != JsonToken.END_OBJECT) {
352 if(jParser.getCurrentToken() == JsonToken.START_OBJECT && found) {
353 RowModel row = jParser.readValueAs(RowModel.class);
354 assertNotNull(row.getKey());
355 for (int i = 0; i < row.getCells().size(); i++) {
356 if (count == 0) {
357 assertEquals("aaa", Bytes.toString(row.getKey()));
358 }
359 if (count == 23) {
360 assertEquals("aax", Bytes.toString(row.getKey()));
361 }
362 count++;
363 }
364 jParser.skipChildren();
365 } else {
366 found = jParser.getCurrentToken() == JsonToken.START_ARRAY;
367 }
368 }
369 assertEquals(24, count);
370 }
371
372 @Test
373 public void testSimpleScannerProtobuf() throws Exception {
374 StringBuilder builder = new StringBuilder();
375 builder.append("/*");
376 builder.append("?");
377 builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
378 builder.append("&");
379 builder.append(Constants.SCAN_LIMIT + "=15");
380 Response response = client.get("/" + TABLE + builder.toString(),
381 Constants.MIMETYPE_PROTOBUF);
382 assertEquals(200, response.getCode());
383 assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
384 int rowCount = readProtobufStream(response.getStream());
385 assertEquals(15, rowCount);
386
387
388 builder = new StringBuilder();
389 builder.append("/*");
390 builder.append("?");
391 builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
392 builder.append("&");
393 builder.append(Constants.SCAN_START_ROW + "=aaa");
394 builder.append("&");
395 builder.append(Constants.SCAN_END_ROW + "=aay");
396 response = client.get("/" + TABLE + builder.toString(),
397 Constants.MIMETYPE_PROTOBUF);
398 assertEquals(200, response.getCode());
399 assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
400 rowCount = readProtobufStream(response.getStream());
401 assertEquals(24, rowCount);
402 }
403
404 private void checkRowsNotNull(CellSetModel model) {
405 for (RowModel row: model.getRows()) {
406 assertTrue(row.getKey() != null);
407 assertTrue(row.getCells().size() > 0);
408 }
409 }
410
411
412
413
414
415
416
417 public int readProtobufStream(InputStream inputStream) throws IOException{
418 DataInputStream stream = new DataInputStream(inputStream);
419 CellSetModel model = null;
420 int rowCount = 0;
421 try {
422 while (true) {
423 byte[] lengthBytes = new byte[2];
424 int readBytes = stream.read(lengthBytes);
425 if (readBytes == -1) {
426 break;
427 }
428 assertEquals(2, readBytes);
429 int length = Bytes.toShort(lengthBytes);
430 byte[] cellset = new byte[length];
431 stream.read(cellset);
432 model = new CellSetModel();
433 model.getObjectFromMessage(cellset);
434 checkRowsNotNull(model);
435 rowCount = rowCount + TestScannerResource.countCellSet(model);
436 }
437 } catch (EOFException exp) {
438 exp.printStackTrace();
439 } finally {
440 stream.close();
441 }
442 return rowCount;
443 }
444
445 @Test
446 public void testScanningUnknownColumnJson() throws IOException, JAXBException {
447
448 StringBuilder builder = new StringBuilder();
449 builder.append("/*");
450 builder.append("?");
451 builder.append(Constants.SCAN_COLUMN + "=a:test");
452 Response response = client.get("/" + TABLE + builder.toString(),
453 Constants.MIMETYPE_JSON);
454 assertEquals(200, response.getCode());
455 assertEquals(Constants.MIMETYPE_JSON, response.getHeader("content-type"));
456 ObjectMapper mapper = new JacksonProvider().locateMapper(CellSetModel.class,
457 MediaType.APPLICATION_JSON_TYPE);
458 CellSetModel model = mapper.readValue(response.getStream(), CellSetModel.class);
459 int count = TestScannerResource.countCellSet(model);
460 assertEquals(0, count);
461 }
462
463
464
465
466
467 @XmlRootElement(name = "CellSet")
468 @XmlAccessorType(XmlAccessType.FIELD)
469 public static class ClientSideCellSetModel implements Serializable {
470
471 private static final long serialVersionUID = 1L;
472
473
474
475
476
477 @XmlElement(name="Row")
478 private List<RowModel> row;
479
480 static boolean listenerInvoked = false;
481
482
483
484
485
486 public void setCellSetModelListener(final Listener l) {
487 row = (l == null) ? null : new ArrayList<RowModel>() {
488 private static final long serialVersionUID = 1L;
489
490 public boolean add(RowModel o) {
491 l.handleRowModel(ClientSideCellSetModel.this, o);
492 listenerInvoked = true;
493 return false;
494 }
495 };
496 }
497
498
499
500
501 public static interface Listener {
502 void handleRowModel(ClientSideCellSetModel helper, RowModel rowModel);
503 }
504 }
505 }
506
507
508