View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.rest;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertNotNull;
23  import static org.junit.Assert.assertTrue;
24  
25  import java.io.DataInputStream;
26  import java.io.EOFException;
27  import java.io.IOException;
28  import java.io.InputStream;
29  import java.io.Serializable;
30  import java.util.ArrayList;
31  import java.util.List;
32  
33  import javax.ws.rs.core.MediaType;
34  import javax.xml.bind.JAXBContext;
35  import javax.xml.bind.JAXBException;
36  import javax.xml.bind.Unmarshaller;
37  import javax.xml.bind.annotation.XmlAccessType;
38  import javax.xml.bind.annotation.XmlAccessorType;
39  import javax.xml.bind.annotation.XmlElement;
40  import javax.xml.bind.annotation.XmlRootElement;
41  import javax.xml.parsers.SAXParserFactory;
42  import javax.xml.stream.XMLStreamException;
43  
44  import org.apache.hadoop.conf.Configuration;
45  import org.apache.hadoop.hbase.HBaseTestingUtility;
46  import org.apache.hadoop.hbase.HColumnDescriptor;
47  import org.apache.hadoop.hbase.HTableDescriptor;
48  import org.apache.hadoop.hbase.MediumTests;
49  import org.apache.hadoop.hbase.TableName;
50  import org.apache.hadoop.hbase.client.HBaseAdmin;
51  import org.apache.hadoop.hbase.rest.client.Client;
52  import org.apache.hadoop.hbase.rest.client.Cluster;
53  import org.apache.hadoop.hbase.rest.client.Response;
54  import org.apache.hadoop.hbase.rest.model.CellModel;
55  import org.apache.hadoop.hbase.rest.model.CellSetModel;
56  import org.apache.hadoop.hbase.rest.model.RowModel;
57  import org.apache.hadoop.hbase.rest.provider.JacksonProvider;
58  import org.apache.hadoop.hbase.util.Bytes;
59  import org.codehaus.jackson.JsonFactory;
60  import org.codehaus.jackson.JsonParser;
61  import org.codehaus.jackson.JsonToken;
62  import org.codehaus.jackson.map.ObjectMapper;
63  import org.junit.AfterClass;
64  import org.junit.BeforeClass;
65  import org.junit.Test;
66  import org.junit.experimental.categories.Category;
67  import org.xml.sax.InputSource;
68  import org.xml.sax.XMLReader;
69  
70  @Category(MediumTests.class)
71  public class TestTableScan {
72  
73    private static final String TABLE = "TestScanResource";
74    private static final String CFA = "a";
75    private static final String CFB = "b";
76    private static final String COLUMN_1 = CFA + ":1";
77    private static final String COLUMN_2 = CFB + ":2";
78    private static Client client;
79    private static int expectedRows1;
80    private static int expectedRows2;
81    private static Configuration conf;
82  
83    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
84    private static final HBaseRESTTestingUtility REST_TEST_UTIL =
85      new HBaseRESTTestingUtility();
86  
87    @BeforeClass
88    public static void setUpBeforeClass() throws Exception {
89      conf = TEST_UTIL.getConfiguration();
90      TEST_UTIL.startMiniCluster();
91      REST_TEST_UTIL.startServletContainer(conf);
92      client = new Client(new Cluster().add("localhost",
93        REST_TEST_UTIL.getServletPort()));
94      HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
95      if (!admin.tableExists(TABLE)) {
96      HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
97      htd.addFamily(new HColumnDescriptor(CFA));
98      htd.addFamily(new HColumnDescriptor(CFB));
99      admin.createTable(htd);
100     expectedRows1 = TestScannerResource.insertData(conf, TABLE, COLUMN_1, 1.0);
101     expectedRows2 = TestScannerResource.insertData(conf, TABLE, COLUMN_2, 0.5);
102     }
103   }
104 
105   @AfterClass
106   public static void tearDownAfterClass() throws Exception {
107     TEST_UTIL.getHBaseAdmin().disableTable(TABLE);
108     TEST_UTIL.getHBaseAdmin().deleteTable(TABLE);
109     REST_TEST_UTIL.shutdownServletContainer();
110     TEST_UTIL.shutdownMiniCluster();
111   }
112 
113   @Test
114   public void testSimpleScannerXML() throws IOException, JAXBException, XMLStreamException {
115     // Test scanning particular columns
116     StringBuilder builder = new StringBuilder();
117     builder.append("/*");
118     builder.append("?");
119     builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
120     builder.append("&");
121     builder.append(Constants.SCAN_LIMIT + "=10");
122     Response response = client.get("/" + TABLE + builder.toString(),
123       Constants.MIMETYPE_XML);
124     assertEquals(200, response.getCode());
125     assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
126     JAXBContext ctx = JAXBContext.newInstance(CellSetModel.class);
127     Unmarshaller ush = ctx.createUnmarshaller();
128     CellSetModel model = (CellSetModel) ush.unmarshal(response.getStream());
129     int count = TestScannerResource.countCellSet(model);
130     assertEquals(10, count);
131     checkRowsNotNull(model);
132 
133     //Test with no limit.
134     builder = new StringBuilder();
135     builder.append("/*");
136     builder.append("?");
137     builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
138     response = client.get("/" + TABLE + builder.toString(),
139       Constants.MIMETYPE_XML);
140     assertEquals(200, response.getCode());
141     assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type")); 
142     model = (CellSetModel) ush.unmarshal(response.getStream());
143     count = TestScannerResource.countCellSet(model);
144     assertEquals(expectedRows1, count);
145     checkRowsNotNull(model);
146 
147     //Test with start and end row.
148     builder = new StringBuilder();
149     builder.append("/*");
150     builder.append("?");
151     builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
152     builder.append("&");
153     builder.append(Constants.SCAN_START_ROW + "=aaa");
154     builder.append("&");
155     builder.append(Constants.SCAN_END_ROW + "=aay");
156     response = client.get("/" + TABLE + builder.toString(),
157       Constants.MIMETYPE_XML);
158     assertEquals(200, response.getCode());
159     model = (CellSetModel) ush.unmarshal(response.getStream());
160     count = TestScannerResource.countCellSet(model);
161     RowModel startRow = model.getRows().get(0);
162     assertEquals("aaa", Bytes.toString(startRow.getKey()));
163     RowModel endRow = model.getRows().get(model.getRows().size() - 1);
164     assertEquals("aax", Bytes.toString(endRow.getKey()));
165     assertEquals(24, count);
166     checkRowsNotNull(model);
167 
168     //Test with start row and limit.
169     builder = new StringBuilder();
170     builder.append("/*");
171     builder.append("?");
172     builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
173     builder.append("&");
174     builder.append(Constants.SCAN_START_ROW + "=aaa");
175     builder.append("&");
176     builder.append(Constants.SCAN_LIMIT + "=15");
177     response = client.get("/" + TABLE + builder.toString(),
178       Constants.MIMETYPE_XML);
179     assertEquals(200, response.getCode());
180     assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
181     model = (CellSetModel) ush.unmarshal(response.getStream());
182     startRow = model.getRows().get(0);
183     assertEquals("aaa", Bytes.toString(startRow.getKey()));
184     count = TestScannerResource.countCellSet(model);
185     assertEquals(15, count);
186     checkRowsNotNull(model);
187 
188   }
189 
190   @Test
191   public void testSimpleScannerJson() throws IOException, JAXBException {
192     // Test scanning particular columns with limit.
193     StringBuilder builder = new StringBuilder();
194     builder.append("/*");
195     builder.append("?");
196     builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
197     builder.append("&");
198     builder.append(Constants.SCAN_LIMIT + "=20");
199     Response response = client.get("/" + TABLE + builder.toString(),
200       Constants.MIMETYPE_JSON);
201     assertEquals(200, response.getCode());
202     assertEquals(Constants.MIMETYPE_JSON, response.getHeader("content-type"));
203     ObjectMapper mapper = new JacksonProvider()
204         .locateMapper(CellSetModel.class, MediaType.APPLICATION_JSON_TYPE);
205     CellSetModel model = mapper.readValue(response.getStream(), CellSetModel.class);
206     int count = TestScannerResource.countCellSet(model);
207     assertEquals(20, count);
208     checkRowsNotNull(model);
209 
210     //Test scanning with no limit.
211     builder = new StringBuilder();
212     builder.append("/*");
213     builder.append("?");
214     builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_2);
215     response = client.get("/" + TABLE + builder.toString(),
216       Constants.MIMETYPE_JSON);
217     assertEquals(200, response.getCode());
218     assertEquals(Constants.MIMETYPE_JSON, response.getHeader("content-type"));
219     model = mapper.readValue(response.getStream(), CellSetModel.class);
220     count = TestScannerResource.countCellSet(model);
221     assertEquals(expectedRows2, count);
222     checkRowsNotNull(model);
223 
224     //Test with start row and end row.
225     builder = new StringBuilder();
226     builder.append("/*");
227     builder.append("?");
228     builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
229     builder.append("&");
230     builder.append(Constants.SCAN_START_ROW + "=aaa");
231     builder.append("&");
232     builder.append(Constants.SCAN_END_ROW + "=aay");
233     response = client.get("/" + TABLE + builder.toString(),
234       Constants.MIMETYPE_JSON);
235     assertEquals(200, response.getCode());
236     model = mapper.readValue(response.getStream(), CellSetModel.class);
237     RowModel startRow = model.getRows().get(0);
238     assertEquals("aaa", Bytes.toString(startRow.getKey()));
239     RowModel endRow = model.getRows().get(model.getRows().size() - 1);
240     assertEquals("aax", Bytes.toString(endRow.getKey()));
241     count = TestScannerResource.countCellSet(model);
242     assertEquals(24, count);
243     checkRowsNotNull(model);
244   }
245 
246   /**
247    * An example to scan using listener in unmarshaller for XML.
248    * @throws Exception the exception
249    */
250   @Test
251   public void testScanUsingListenerUnmarshallerXML() throws Exception {
252     StringBuilder builder = new StringBuilder();
253     builder.append("/*");
254     builder.append("?");
255     builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
256     builder.append("&");
257     builder.append(Constants.SCAN_LIMIT + "=10");
258     Response response = client.get("/" + TABLE + builder.toString(),
259       Constants.MIMETYPE_XML);
260     assertEquals(200, response.getCode());
261     assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
262     JAXBContext context = JAXBContext.newInstance(ClientSideCellSetModel.class, RowModel.class,
263       CellModel.class);
264     Unmarshaller unmarshaller = context.createUnmarshaller();
265 
266     final ClientSideCellSetModel.Listener listener = new ClientSideCellSetModel.Listener() {
267       @Override
268       public void handleRowModel(ClientSideCellSetModel helper, RowModel row) {
269         assertTrue(row.getKey() != null);
270         assertTrue(row.getCells().size() > 0);
271       }
272     };
273 
274     // install the callback on all ClientSideCellSetModel instances
275     unmarshaller.setListener(new Unmarshaller.Listener() {
276         public void beforeUnmarshal(Object target, Object parent) {
277             if (target instanceof ClientSideCellSetModel) {
278                 ((ClientSideCellSetModel) target).setCellSetModelListener(listener);
279             }
280         }
281 
282         public void afterUnmarshal(Object target, Object parent) {
283             if (target instanceof ClientSideCellSetModel) {
284                 ((ClientSideCellSetModel) target).setCellSetModelListener(null);
285             }
286         }
287     });
288 
289     // create a new XML parser
290     SAXParserFactory factory = SAXParserFactory.newInstance();
291     factory.setNamespaceAware(true);
292     XMLReader reader = factory.newSAXParser().getXMLReader();
293     reader.setContentHandler(unmarshaller.getUnmarshallerHandler());
294     assertFalse(ClientSideCellSetModel.listenerInvoked);
295     reader.parse(new InputSource(response.getStream()));
296     assertTrue(ClientSideCellSetModel.listenerInvoked);
297 
298   }
299 
300   @Test
301   public void testStreamingJSON() throws Exception {
302     // Test scanning particular columns with limit.
303     StringBuilder builder = new StringBuilder();
304     builder.append("/*");
305     builder.append("?");
306     builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
307     builder.append("&");
308     builder.append(Constants.SCAN_LIMIT + "=20");
309     Response response = client.get("/" + TABLE + builder.toString(),
310       Constants.MIMETYPE_JSON);
311     assertEquals(200, response.getCode());
312     assertEquals(Constants.MIMETYPE_JSON, response.getHeader("content-type"));
313     ObjectMapper mapper = new JacksonProvider()
314         .locateMapper(CellSetModel.class, MediaType.APPLICATION_JSON_TYPE);
315     CellSetModel model = mapper.readValue(response.getStream(), CellSetModel.class);
316     int count = TestScannerResource.countCellSet(model);
317     assertEquals(20, count);
318     checkRowsNotNull(model);
319 
320     //Test scanning with no limit.
321     builder = new StringBuilder();
322     builder.append("/*");
323     builder.append("?");
324     builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_2);
325     response = client.get("/" + TABLE + builder.toString(),
326       Constants.MIMETYPE_JSON);
327     assertEquals(200, response.getCode());
328     assertEquals(Constants.MIMETYPE_JSON, response.getHeader("content-type"));
329     model = mapper.readValue(response.getStream(), CellSetModel.class);
330     count = TestScannerResource.countCellSet(model);
331     assertEquals(expectedRows2, count);
332     checkRowsNotNull(model);
333 
334     //Test with start row and end row.
335     builder = new StringBuilder();
336     builder.append("/*");
337     builder.append("?");
338     builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
339     builder.append("&");
340     builder.append(Constants.SCAN_START_ROW + "=aaa");
341     builder.append("&");
342     builder.append(Constants.SCAN_END_ROW + "=aay");
343     response = client.get("/" + TABLE + builder.toString(),
344       Constants.MIMETYPE_JSON);
345     assertEquals(200, response.getCode());
346 
347     count = 0;
348     JsonFactory jfactory = new JsonFactory(mapper);
349     JsonParser jParser = jfactory.createJsonParser(response.getStream());
350     boolean found = false;
351     while (jParser.nextToken() != JsonToken.END_OBJECT) {
352       if(jParser.getCurrentToken() == JsonToken.START_OBJECT && found) {
353         RowModel row = jParser.readValueAs(RowModel.class);
354         assertNotNull(row.getKey());
355         for (int i = 0; i < row.getCells().size(); i++) {
356           if (count == 0) {
357             assertEquals("aaa", Bytes.toString(row.getKey()));
358           }
359           if (count == 23) {
360             assertEquals("aax", Bytes.toString(row.getKey()));
361           }
362           count++;
363         }
364         jParser.skipChildren();
365       } else {
366         found = jParser.getCurrentToken() == JsonToken.START_ARRAY;
367       }
368     }
369     assertEquals(24, count);
370   }
371 
372   @Test
373   public void testSimpleScannerProtobuf() throws Exception {
374     StringBuilder builder = new StringBuilder();
375     builder.append("/*");
376     builder.append("?");
377     builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
378     builder.append("&");
379     builder.append(Constants.SCAN_LIMIT + "=15");
380     Response response = client.get("/" + TABLE + builder.toString(),
381       Constants.MIMETYPE_PROTOBUF);
382     assertEquals(200, response.getCode());
383     assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
384     int rowCount = readProtobufStream(response.getStream());
385     assertEquals(15, rowCount);
386 
387   //Test with start row and end row.
388     builder = new StringBuilder();
389     builder.append("/*");
390     builder.append("?");
391     builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
392     builder.append("&");
393     builder.append(Constants.SCAN_START_ROW + "=aaa");
394     builder.append("&");
395     builder.append(Constants.SCAN_END_ROW + "=aay");
396     response = client.get("/" + TABLE + builder.toString(),
397       Constants.MIMETYPE_PROTOBUF);
398     assertEquals(200, response.getCode());
399     assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
400     rowCount = readProtobufStream(response.getStream());
401     assertEquals(24, rowCount);
402   }
403 
404   private void checkRowsNotNull(CellSetModel model) {
405     for (RowModel row: model.getRows()) {
406       assertTrue(row.getKey() != null);
407       assertTrue(row.getCells().size() > 0);
408     }
409   }
410 
411   /**
412    * Read protobuf stream.
413    * @param inputStream the input stream
414    * @return The number of rows in the cell set model.
415    * @throws IOException Signals that an I/O exception has occurred.
416    */
417   public int readProtobufStream(InputStream inputStream) throws IOException{
418     DataInputStream stream = new DataInputStream(inputStream);
419     CellSetModel model = null;
420     int rowCount = 0;
421     try {
422       while (true) {
423         byte[] lengthBytes = new byte[2];
424         int readBytes = stream.read(lengthBytes);
425         if (readBytes == -1) {
426           break;
427         }
428         assertEquals(2, readBytes);
429         int length = Bytes.toShort(lengthBytes);
430         byte[] cellset = new byte[length];
431         stream.read(cellset);
432         model = new CellSetModel();
433         model.getObjectFromMessage(cellset);
434         checkRowsNotNull(model);
435         rowCount = rowCount + TestScannerResource.countCellSet(model);
436       }
437     } catch (EOFException exp) {
438       exp.printStackTrace();
439     } finally {
440       stream.close();
441     }
442     return rowCount;
443   }
444 
445   @Test
446   public void testScanningUnknownColumnJson() throws IOException, JAXBException {
447     // Test scanning particular columns with limit.
448     StringBuilder builder = new StringBuilder();
449     builder.append("/*");
450     builder.append("?");
451     builder.append(Constants.SCAN_COLUMN + "=a:test");
452     Response response = client.get("/" + TABLE  + builder.toString(),
453       Constants.MIMETYPE_JSON);
454     assertEquals(200, response.getCode());
455     assertEquals(Constants.MIMETYPE_JSON, response.getHeader("content-type"));
456     ObjectMapper mapper = new JacksonProvider().locateMapper(CellSetModel.class,
457       MediaType.APPLICATION_JSON_TYPE);
458     CellSetModel model = mapper.readValue(response.getStream(), CellSetModel.class);
459     int count = TestScannerResource.countCellSet(model);
460     assertEquals(0, count);
461   }
462 
463   /**
464    * The Class ClientSideCellSetModel which mimics cell set model, and contains listener to perform
465    * user defined operations on the row model.
466    */
467   @XmlRootElement(name = "CellSet")
468   @XmlAccessorType(XmlAccessType.FIELD)
469   public static class ClientSideCellSetModel implements Serializable {
470 
471     private static final long serialVersionUID = 1L;
472 
473     /**
474      * This list is not a real list; instead it will notify a listener whenever JAXB has
475      * unmarshalled the next row.
476      */
477     @XmlElement(name="Row")
478     private List<RowModel> row;
479 
480     static boolean listenerInvoked = false;
481 
482     /**
483      * Install a listener for row model on this object. If l is null, the listener
484      * is removed again.
485      */
486     public void setCellSetModelListener(final Listener l) {
487         row = (l == null) ? null : new ArrayList<RowModel>() {
488         private static final long serialVersionUID = 1L;
489 
490             public boolean add(RowModel o) {
491                 l.handleRowModel(ClientSideCellSetModel.this, o);
492                 listenerInvoked = true;
493                 return false;
494             }
495         };
496     }
497 
498     /**
499      * This listener is invoked every time a new row model is unmarshalled.
500      */
501     public static interface Listener {
502         void handleRowModel(ClientSideCellSetModel helper, RowModel rowModel);
503     }
504   }
505 }
506 
507 
508