View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.rest;
21  
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertNotNull;
24  import static org.junit.Assert.assertNull;
25  import static org.junit.Assert.assertTrue;
26  
27  import java.io.ByteArrayInputStream;
28  import java.io.IOException;
29  import java.io.StringWriter;
30  import java.util.ArrayList;
31  import java.util.Iterator;
32  import java.util.List;
33  import java.util.Random;
34  
35  import javax.xml.bind.JAXBContext;
36  import javax.xml.bind.JAXBException;
37  import javax.xml.bind.Marshaller;
38  import javax.xml.bind.Unmarshaller;
39  
40  import org.apache.commons.httpclient.Header;
41  import org.apache.hadoop.conf.Configuration;
42  import org.apache.hadoop.hbase.HBaseTestingUtility;
43  import org.apache.hadoop.hbase.HColumnDescriptor;
44  import org.apache.hadoop.hbase.HTableDescriptor;
45  import org.apache.hadoop.hbase.KeyValue;
46  import org.apache.hadoop.hbase.TableName;
47  import org.apache.hadoop.hbase.client.Admin;
48  import org.apache.hadoop.hbase.client.Connection;
49  import org.apache.hadoop.hbase.client.ConnectionFactory;
50  import org.apache.hadoop.hbase.client.Durability;
51  import org.apache.hadoop.hbase.client.Put;
52  import org.apache.hadoop.hbase.client.Table;
53  import org.apache.hadoop.hbase.rest.client.Client;
54  import org.apache.hadoop.hbase.rest.client.Cluster;
55  import org.apache.hadoop.hbase.rest.client.Response;
56  import org.apache.hadoop.hbase.rest.model.CellModel;
57  import org.apache.hadoop.hbase.rest.model.CellSetModel;
58  import org.apache.hadoop.hbase.rest.model.RowModel;
59  import org.apache.hadoop.hbase.rest.model.ScannerModel;
60  import org.apache.hadoop.hbase.testclassification.MediumTests;
61  import org.apache.hadoop.hbase.util.Bytes;
62  import org.junit.AfterClass;
63  import org.junit.BeforeClass;
64  import org.junit.Test;
65  import org.junit.experimental.categories.Category;
66  
67  @Category(MediumTests.class)
68  public class TestScannerResource {
69    private static final TableName TABLE = TableName.valueOf("TestScannerResource");
70    private static final String NONEXISTENT_TABLE = "ThisTableDoesNotExist";
71    private static final String CFA = "a";
72    private static final String CFB = "b";
73    private static final String COLUMN_1 = CFA + ":1";
74    private static final String COLUMN_2 = CFB + ":2";
75  
76    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
77    private static final HBaseRESTTestingUtility REST_TEST_UTIL = 
78      new HBaseRESTTestingUtility();
79    private static Client client;
80    private static JAXBContext context;
81    private static Marshaller marshaller;
82    private static Unmarshaller unmarshaller;
83    private static int expectedRows1;
84    private static int expectedRows2;
85    private static Configuration conf;
86  
87    static int insertData(Configuration conf, TableName tableName, String column, double prob)
88        throws IOException {
89      Random rng = new Random();
90      byte[] k = new byte[3];
91      byte [][] famAndQf = KeyValue.parseColumn(Bytes.toBytes(column));
92      List<Put> puts = new ArrayList<>();
93      for (byte b1 = 'a'; b1 < 'z'; b1++) {
94        for (byte b2 = 'a'; b2 < 'z'; b2++) {
95          for (byte b3 = 'a'; b3 < 'z'; b3++) {
96            if (rng.nextDouble() < prob) {
97              k[0] = b1;
98              k[1] = b2;
99              k[2] = b3;
100             Put put = new Put(k);
101             put.setDurability(Durability.SKIP_WAL);
102             put.add(famAndQf[0], famAndQf[1], k);
103             puts.add(put);
104           }
105         }
106       }
107     }
108     try (Connection conn = ConnectionFactory.createConnection(conf);
109         Table table = conn.getTable(tableName)) {
110       table.put(puts);
111     }
112     return puts.size();
113   }
114 
115   static int countCellSet(CellSetModel model) {
116     int count = 0;
117     Iterator<RowModel> rows = model.getRows().iterator();
118     while (rows.hasNext()) {
119       RowModel row = rows.next();
120       Iterator<CellModel> cells = row.getCells().iterator();
121       while (cells.hasNext()) {
122         cells.next();
123         count++;
124       }
125     }
126     return count;
127   }
128 
129   private static int fullTableScan(ScannerModel model) throws IOException {
130     model.setBatch(100);
131     Response response = client.put("/" + TABLE + "/scanner",
132       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
133     assertEquals(response.getCode(), 201);
134     String scannerURI = response.getLocation();
135     assertNotNull(scannerURI);
136     int count = 0;
137     while (true) {
138       response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
139       assertTrue(response.getCode() == 200 || response.getCode() == 204);
140       if (response.getCode() == 200) {
141         assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
142         CellSetModel cellSet = new CellSetModel();
143         cellSet.getObjectFromMessage(response.getBody());
144         Iterator<RowModel> rows = cellSet.getRows().iterator();
145         while (rows.hasNext()) {
146           RowModel row = rows.next();
147           Iterator<CellModel> cells = row.getCells().iterator();
148           while (cells.hasNext()) {
149             cells.next();
150             count++;
151           }
152         }
153       } else {
154         break;
155       }
156     }
157     // delete the scanner
158     response = client.delete(scannerURI);
159     assertEquals(response.getCode(), 200);
160     return count;
161   }
162 
163   @BeforeClass
164   public static void setUpBeforeClass() throws Exception {
165     conf = TEST_UTIL.getConfiguration();
166     TEST_UTIL.startMiniCluster();
167     REST_TEST_UTIL.startServletContainer(conf);
168     client = new Client(new Cluster().add("localhost",
169       REST_TEST_UTIL.getServletPort()));
170     context = JAXBContext.newInstance(
171       CellModel.class,
172       CellSetModel.class,
173       RowModel.class,
174       ScannerModel.class);
175     marshaller = context.createMarshaller();
176     unmarshaller = context.createUnmarshaller();
177     Admin admin = TEST_UTIL.getHBaseAdmin();
178     if (admin.tableExists(TABLE)) {
179       return;
180     }
181     HTableDescriptor htd = new HTableDescriptor(TABLE);
182     htd.addFamily(new HColumnDescriptor(CFA));
183     htd.addFamily(new HColumnDescriptor(CFB));
184     admin.createTable(htd);
185     expectedRows1 = insertData(TEST_UTIL.getConfiguration(), TABLE, COLUMN_1, 1.0);
186     expectedRows2 = insertData(TEST_UTIL.getConfiguration(), TABLE, COLUMN_2, 0.5);
187   }
188 
189   @AfterClass
190   public static void tearDownAfterClass() throws Exception {
191     REST_TEST_UTIL.shutdownServletContainer();
192     TEST_UTIL.shutdownMiniCluster();
193   }
194 
195   @Test
196   public void testSimpleScannerXML() throws IOException, JAXBException {
197     final int BATCH_SIZE = 5;
198     // new scanner
199     ScannerModel model = new ScannerModel();
200     model.setBatch(BATCH_SIZE);
201     model.addColumn(Bytes.toBytes(COLUMN_1));
202     StringWriter writer = new StringWriter();
203     marshaller.marshal(model, writer);
204     byte[] body = Bytes.toBytes(writer.toString());
205 
206     // test put operation is forbidden in read-only mode
207     conf.set("hbase.rest.readonly", "true");
208     Response response = client.put("/" + TABLE + "/scanner",
209       Constants.MIMETYPE_XML, body);
210     assertEquals(response.getCode(), 403);
211     String scannerURI = response.getLocation();
212     assertNull(scannerURI);
213 
214     // recall previous put operation with read-only off
215     conf.set("hbase.rest.readonly", "false");
216     response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML,
217       body);
218     assertEquals(response.getCode(), 201);
219     scannerURI = response.getLocation();
220     assertNotNull(scannerURI);
221 
222     // get a cell set
223     response = client.get(scannerURI, Constants.MIMETYPE_XML);
224     assertEquals(response.getCode(), 200);
225     assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
226     CellSetModel cellSet = (CellSetModel)
227       unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
228     // confirm batch size conformance
229     assertEquals(countCellSet(cellSet), BATCH_SIZE);
230 
231     // test delete scanner operation is forbidden in read-only mode
232     conf.set("hbase.rest.readonly", "true");
233     response = client.delete(scannerURI);
234     assertEquals(response.getCode(), 403);
235 
236     // recall previous delete scanner operation with read-only off
237     conf.set("hbase.rest.readonly", "false");
238     response = client.delete(scannerURI);
239     assertEquals(response.getCode(), 200);
240   }
241 
242   @Test
243   public void testSimpleScannerPB() throws IOException {
244     final int BATCH_SIZE = 10;
245     // new scanner
246     ScannerModel model = new ScannerModel();
247     model.setBatch(BATCH_SIZE);
248     model.addColumn(Bytes.toBytes(COLUMN_1));
249 
250     // test put operation is forbidden in read-only mode
251     conf.set("hbase.rest.readonly", "true");
252     Response response = client.put("/" + TABLE + "/scanner",
253       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
254     assertEquals(response.getCode(), 403);
255     String scannerURI = response.getLocation();
256     assertNull(scannerURI);
257 
258     // recall previous put operation with read-only off
259     conf.set("hbase.rest.readonly", "false");
260     response = client.put("/" + TABLE + "/scanner",
261       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
262     assertEquals(response.getCode(), 201);
263     scannerURI = response.getLocation();
264     assertNotNull(scannerURI);
265 
266     // get a cell set
267     response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
268     assertEquals(response.getCode(), 200);
269     assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
270     CellSetModel cellSet = new CellSetModel();
271     cellSet.getObjectFromMessage(response.getBody());
272     // confirm batch size conformance
273     assertEquals(countCellSet(cellSet), BATCH_SIZE);
274 
275     // test delete scanner operation is forbidden in read-only mode
276     conf.set("hbase.rest.readonly", "true");
277     response = client.delete(scannerURI);
278     assertEquals(response.getCode(), 403);
279 
280     // recall previous delete scanner operation with read-only off
281     conf.set("hbase.rest.readonly", "false");
282     response = client.delete(scannerURI);
283     assertEquals(response.getCode(), 200);
284   }
285 
286   @Test
287   public void testSimpleScannerBinary() throws IOException {
288     // new scanner
289     ScannerModel model = new ScannerModel();
290     model.setBatch(1);
291     model.addColumn(Bytes.toBytes(COLUMN_1));
292 
293     // test put operation is forbidden in read-only mode
294     conf.set("hbase.rest.readonly", "true");
295     Response response = client.put("/" + TABLE + "/scanner",
296       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
297     assertEquals(response.getCode(), 403);
298     String scannerURI = response.getLocation();
299     assertNull(scannerURI);
300 
301     // recall previous put operation with read-only off
302     conf.set("hbase.rest.readonly", "false");
303     response = client.put("/" + TABLE + "/scanner",
304       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
305     assertEquals(response.getCode(), 201);
306     scannerURI = response.getLocation();
307     assertNotNull(scannerURI);
308 
309     // get a cell
310     response = client.get(scannerURI, Constants.MIMETYPE_BINARY);
311     assertEquals(response.getCode(), 200);
312     assertEquals(Constants.MIMETYPE_BINARY, response.getHeader("content-type"));
313     // verify that data was returned
314     assertTrue(response.getBody().length > 0);
315     // verify that the expected X-headers are present
316     boolean foundRowHeader = false, foundColumnHeader = false,
317       foundTimestampHeader = false;
318     for (Header header: response.getHeaders()) {
319       if (header.getName().equals("X-Row")) {
320         foundRowHeader = true;
321       } else if (header.getName().equals("X-Column")) {
322         foundColumnHeader = true;
323       } else if (header.getName().equals("X-Timestamp")) {
324         foundTimestampHeader = true;
325       }
326     }
327     assertTrue(foundRowHeader);
328     assertTrue(foundColumnHeader);
329     assertTrue(foundTimestampHeader);
330 
331     // test delete scanner operation is forbidden in read-only mode
332     conf.set("hbase.rest.readonly", "true");
333     response = client.delete(scannerURI);
334     assertEquals(response.getCode(), 403);
335 
336     // recall previous delete scanner operation with read-only off
337     conf.set("hbase.rest.readonly", "false");
338     response = client.delete(scannerURI);
339     assertEquals(response.getCode(), 200);
340   }
341 
342   @Test
343   public void testFullTableScan() throws IOException {
344     ScannerModel model = new ScannerModel();
345     model.addColumn(Bytes.toBytes(COLUMN_1));
346     assertEquals(fullTableScan(model), expectedRows1);
347 
348     model = new ScannerModel();
349     model.addColumn(Bytes.toBytes(COLUMN_2));
350     assertEquals(fullTableScan(model), expectedRows2);
351   }
352 
353   @Test
354   public void testTableDoesNotExist() throws IOException, JAXBException {
355     ScannerModel model = new ScannerModel();
356     StringWriter writer = new StringWriter();
357     marshaller.marshal(model, writer);
358     byte[] body = Bytes.toBytes(writer.toString());
359     Response response = client.put("/" + NONEXISTENT_TABLE +
360       "/scanner", Constants.MIMETYPE_XML, body);
361     assertEquals(response.getCode(), 404);
362   }
363 
364 }
365