View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.rest;
21  
22  import java.io.ByteArrayInputStream;
23  import java.io.IOException;
24  import java.io.StringWriter;
25  import java.util.Iterator;
26  import java.util.Random;
27  
28  import javax.xml.bind.JAXBContext;
29  import javax.xml.bind.JAXBException;
30  import javax.xml.bind.Marshaller;
31  import javax.xml.bind.Unmarshaller;
32  
33  import org.apache.commons.httpclient.Header;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.hbase.*;
36  import org.apache.hadoop.hbase.client.HBaseAdmin;
37  import org.apache.hadoop.hbase.client.HTable;
38  import org.apache.hadoop.hbase.client.Put;
39  import org.apache.hadoop.hbase.client.Durability;
40  import org.apache.hadoop.hbase.rest.client.Client;
41  import org.apache.hadoop.hbase.rest.client.Cluster;
42  import org.apache.hadoop.hbase.rest.client.Response;
43  import org.apache.hadoop.hbase.rest.model.CellModel;
44  import org.apache.hadoop.hbase.rest.model.CellSetModel;
45  import org.apache.hadoop.hbase.rest.model.RowModel;
46  import org.apache.hadoop.hbase.rest.model.ScannerModel;
47  import org.apache.hadoop.hbase.util.Bytes;
48  
49  import static org.junit.Assert.*;
50  
51  import org.junit.AfterClass;
52  import org.junit.BeforeClass;
53  import org.junit.Test;
54  import org.junit.experimental.categories.Category;
55  
56  @Category(MediumTests.class)
57  public class TestScannerResource {
58    private static final String TABLE = "TestScannerResource";
59    private static final String NONEXISTENT_TABLE = "ThisTableDoesNotExist";
60    private static final String CFA = "a";
61    private static final String CFB = "b";
62    private static final String COLUMN_1 = CFA + ":1";
63    private static final String COLUMN_2 = CFB + ":2";
64  
65    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
66    private static final HBaseRESTTestingUtility REST_TEST_UTIL = 
67      new HBaseRESTTestingUtility();
68    private static Client client;
69    private static JAXBContext context;
70    private static Marshaller marshaller;
71    private static Unmarshaller unmarshaller;
72    private static int expectedRows1;
73    private static int expectedRows2;
74    private static Configuration conf;
75  
76    static int insertData(Configuration conf, String tableName, String column, double prob)
77        throws IOException {
78      Random rng = new Random();
79      int count = 0;
80      HTable table = new HTable(conf, tableName);
81      byte[] k = new byte[3];
82      byte [][] famAndQf = KeyValue.parseColumn(Bytes.toBytes(column));
83      for (byte b1 = 'a'; b1 < 'z'; b1++) {
84        for (byte b2 = 'a'; b2 < 'z'; b2++) {
85          for (byte b3 = 'a'; b3 < 'z'; b3++) {
86            if (rng.nextDouble() < prob) {
87              k[0] = b1;
88              k[1] = b2;
89              k[2] = b3;
90              Put put = new Put(k);
91              put.setDurability(Durability.SKIP_WAL);
92              put.add(famAndQf[0], famAndQf[1], k);
93              table.put(put);
94              count++;
95            }
96          }
97        }
98      }
99      table.flushCommits();
100     table.close();
101     return count;
102   }
103 
104   static int countCellSet(CellSetModel model) {
105     int count = 0;
106     Iterator<RowModel> rows = model.getRows().iterator();
107     while (rows.hasNext()) {
108       RowModel row = rows.next();
109       Iterator<CellModel> cells = row.getCells().iterator();
110       while (cells.hasNext()) {
111         cells.next();
112         count++;
113       }
114     }
115     return count;
116   }
117 
118   private static int fullTableScan(ScannerModel model) throws IOException {
119     model.setBatch(100);
120     Response response = client.put("/" + TABLE + "/scanner",
121       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
122     assertEquals(response.getCode(), 201);
123     String scannerURI = response.getLocation();
124     assertNotNull(scannerURI);
125     int count = 0;
126     while (true) {
127       response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
128       assertTrue(response.getCode() == 200 || response.getCode() == 204);
129       if (response.getCode() == 200) {
130         assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
131         CellSetModel cellSet = new CellSetModel();
132         cellSet.getObjectFromMessage(response.getBody());
133         Iterator<RowModel> rows = cellSet.getRows().iterator();
134         while (rows.hasNext()) {
135           RowModel row = rows.next();
136           Iterator<CellModel> cells = row.getCells().iterator();
137           while (cells.hasNext()) {
138             cells.next();
139             count++;
140           }
141         }
142       } else {
143         break;
144       }
145     }
146     // delete the scanner
147     response = client.delete(scannerURI);
148     assertEquals(response.getCode(), 200);
149     return count;
150   }
151 
152   @BeforeClass
153   public static void setUpBeforeClass() throws Exception {
154     conf = TEST_UTIL.getConfiguration();
155     TEST_UTIL.startMiniCluster();
156     REST_TEST_UTIL.startServletContainer(conf);
157     client = new Client(new Cluster().add("localhost",
158       REST_TEST_UTIL.getServletPort()));
159     context = JAXBContext.newInstance(
160       CellModel.class,
161       CellSetModel.class,
162       RowModel.class,
163       ScannerModel.class);
164     marshaller = context.createMarshaller();
165     unmarshaller = context.createUnmarshaller();
166     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
167     if (admin.tableExists(TABLE)) {
168       return;
169     }
170     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
171     htd.addFamily(new HColumnDescriptor(CFA));
172     htd.addFamily(new HColumnDescriptor(CFB));
173     admin.createTable(htd);
174     expectedRows1 = insertData(TEST_UTIL.getConfiguration(), TABLE, COLUMN_1, 1.0);
175     expectedRows2 = insertData(TEST_UTIL.getConfiguration(), TABLE, COLUMN_2, 0.5);
176   }
177 
178   @AfterClass
179   public static void tearDownAfterClass() throws Exception {
180     REST_TEST_UTIL.shutdownServletContainer();
181     TEST_UTIL.shutdownMiniCluster();
182   }
183 
184   @Test
185   public void testSimpleScannerXML() throws IOException, JAXBException {
186     final int BATCH_SIZE = 5;
187     // new scanner
188     ScannerModel model = new ScannerModel();
189     model.setBatch(BATCH_SIZE);
190     model.addColumn(Bytes.toBytes(COLUMN_1));
191     StringWriter writer = new StringWriter();
192     marshaller.marshal(model, writer);
193     byte[] body = Bytes.toBytes(writer.toString());
194 
195     // test put operation is forbidden in read-only mode
196     conf.set("hbase.rest.readonly", "true");
197     Response response = client.put("/" + TABLE + "/scanner",
198       Constants.MIMETYPE_XML, body);
199     assertEquals(response.getCode(), 403);
200     String scannerURI = response.getLocation();
201     assertNull(scannerURI);
202 
203     // recall previous put operation with read-only off
204     conf.set("hbase.rest.readonly", "false");
205     response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML,
206       body);
207     assertEquals(response.getCode(), 201);
208     scannerURI = response.getLocation();
209     assertNotNull(scannerURI);
210 
211     // get a cell set
212     response = client.get(scannerURI, Constants.MIMETYPE_XML);
213     assertEquals(response.getCode(), 200);
214     assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
215     CellSetModel cellSet = (CellSetModel)
216       unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
217     // confirm batch size conformance
218     assertEquals(countCellSet(cellSet), BATCH_SIZE);
219 
220     // test delete scanner operation is forbidden in read-only mode
221     conf.set("hbase.rest.readonly", "true");
222     response = client.delete(scannerURI);
223     assertEquals(response.getCode(), 403);
224 
225     // recall previous delete scanner operation with read-only off
226     conf.set("hbase.rest.readonly", "false");
227     response = client.delete(scannerURI);
228     assertEquals(response.getCode(), 200);
229   }
230 
231   @Test
232   public void testSimpleScannerPB() throws IOException {
233     final int BATCH_SIZE = 10;
234     // new scanner
235     ScannerModel model = new ScannerModel();
236     model.setBatch(BATCH_SIZE);
237     model.addColumn(Bytes.toBytes(COLUMN_1));
238 
239     // test put operation is forbidden in read-only mode
240     conf.set("hbase.rest.readonly", "true");
241     Response response = client.put("/" + TABLE + "/scanner",
242       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
243     assertEquals(response.getCode(), 403);
244     String scannerURI = response.getLocation();
245     assertNull(scannerURI);
246 
247     // recall previous put operation with read-only off
248     conf.set("hbase.rest.readonly", "false");
249     response = client.put("/" + TABLE + "/scanner",
250       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
251     assertEquals(response.getCode(), 201);
252     scannerURI = response.getLocation();
253     assertNotNull(scannerURI);
254 
255     // get a cell set
256     response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
257     assertEquals(response.getCode(), 200);
258     assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
259     CellSetModel cellSet = new CellSetModel();
260     cellSet.getObjectFromMessage(response.getBody());
261     // confirm batch size conformance
262     assertEquals(countCellSet(cellSet), BATCH_SIZE);
263 
264     // test delete scanner operation is forbidden in read-only mode
265     conf.set("hbase.rest.readonly", "true");
266     response = client.delete(scannerURI);
267     assertEquals(response.getCode(), 403);
268 
269     // recall previous delete scanner operation with read-only off
270     conf.set("hbase.rest.readonly", "false");
271     response = client.delete(scannerURI);
272     assertEquals(response.getCode(), 200);
273   }
274 
275   @Test
276   public void testSimpleScannerBinary() throws IOException {
277     // new scanner
278     ScannerModel model = new ScannerModel();
279     model.setBatch(1);
280     model.addColumn(Bytes.toBytes(COLUMN_1));
281 
282     // test put operation is forbidden in read-only mode
283     conf.set("hbase.rest.readonly", "true");
284     Response response = client.put("/" + TABLE + "/scanner",
285       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
286     assertEquals(response.getCode(), 403);
287     String scannerURI = response.getLocation();
288     assertNull(scannerURI);
289 
290     // recall previous put operation with read-only off
291     conf.set("hbase.rest.readonly", "false");
292     response = client.put("/" + TABLE + "/scanner",
293       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
294     assertEquals(response.getCode(), 201);
295     scannerURI = response.getLocation();
296     assertNotNull(scannerURI);
297 
298     // get a cell
299     response = client.get(scannerURI, Constants.MIMETYPE_BINARY);
300     assertEquals(response.getCode(), 200);
301     assertEquals(Constants.MIMETYPE_BINARY, response.getHeader("content-type"));
302     // verify that data was returned
303     assertTrue(response.getBody().length > 0);
304     // verify that the expected X-headers are present
305     boolean foundRowHeader = false, foundColumnHeader = false,
306       foundTimestampHeader = false;
307     for (Header header: response.getHeaders()) {
308       if (header.getName().equals("X-Row")) {
309         foundRowHeader = true;
310       } else if (header.getName().equals("X-Column")) {
311         foundColumnHeader = true;
312       } else if (header.getName().equals("X-Timestamp")) {
313         foundTimestampHeader = true;
314       }
315     }
316     assertTrue(foundRowHeader);
317     assertTrue(foundColumnHeader);
318     assertTrue(foundTimestampHeader);
319 
320     // test delete scanner operation is forbidden in read-only mode
321     conf.set("hbase.rest.readonly", "true");
322     response = client.delete(scannerURI);
323     assertEquals(response.getCode(), 403);
324 
325     // recall previous delete scanner operation with read-only off
326     conf.set("hbase.rest.readonly", "false");
327     response = client.delete(scannerURI);
328     assertEquals(response.getCode(), 200);
329   }
330 
331   @Test
332   public void testFullTableScan() throws IOException {
333     ScannerModel model = new ScannerModel();
334     model.addColumn(Bytes.toBytes(COLUMN_1));
335     assertEquals(fullTableScan(model), expectedRows1);
336 
337     model = new ScannerModel();
338     model.addColumn(Bytes.toBytes(COLUMN_2));
339     assertEquals(fullTableScan(model), expectedRows2);
340   }
341 
342   @Test
343   public void testTableDoesNotExist() throws IOException, JAXBException {
344     ScannerModel model = new ScannerModel();
345     StringWriter writer = new StringWriter();
346     marshaller.marshal(model, writer);
347     byte[] body = Bytes.toBytes(writer.toString());
348     Response response = client.put("/" + NONEXISTENT_TABLE +
349       "/scanner", Constants.MIMETYPE_XML, body);
350     assertEquals(response.getCode(), 404);
351   }
352 
353 }
354