1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.rest;
21
22 import java.io.ByteArrayInputStream;
23 import java.io.IOException;
24 import java.io.StringWriter;
25 import java.util.Iterator;
26 import java.util.Random;
27
28 import javax.xml.bind.JAXBContext;
29 import javax.xml.bind.JAXBException;
30 import javax.xml.bind.Marshaller;
31 import javax.xml.bind.Unmarshaller;
32
33 import org.apache.commons.httpclient.Header;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.hbase.*;
36 import org.apache.hadoop.hbase.client.HBaseAdmin;
37 import org.apache.hadoop.hbase.client.HTable;
38 import org.apache.hadoop.hbase.client.Put;
39 import org.apache.hadoop.hbase.client.Durability;
40 import org.apache.hadoop.hbase.rest.client.Client;
41 import org.apache.hadoop.hbase.rest.client.Cluster;
42 import org.apache.hadoop.hbase.rest.client.Response;
43 import org.apache.hadoop.hbase.rest.model.CellModel;
44 import org.apache.hadoop.hbase.rest.model.CellSetModel;
45 import org.apache.hadoop.hbase.rest.model.RowModel;
46 import org.apache.hadoop.hbase.rest.model.ScannerModel;
47 import org.apache.hadoop.hbase.util.Bytes;
48
49 import static org.junit.Assert.*;
50
51 import org.junit.AfterClass;
52 import org.junit.BeforeClass;
53 import org.junit.Test;
54 import org.junit.experimental.categories.Category;
55
56 @Category(MediumTests.class)
57 public class TestScannerResource {
58 private static final String TABLE = "TestScannerResource";
59 private static final String NONEXISTENT_TABLE = "ThisTableDoesNotExist";
60 private static final String CFA = "a";
61 private static final String CFB = "b";
62 private static final String COLUMN_1 = CFA + ":1";
63 private static final String COLUMN_2 = CFB + ":2";
64
65 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
66 private static final HBaseRESTTestingUtility REST_TEST_UTIL =
67 new HBaseRESTTestingUtility();
68 private static Client client;
69 private static JAXBContext context;
70 private static Marshaller marshaller;
71 private static Unmarshaller unmarshaller;
72 private static int expectedRows1;
73 private static int expectedRows2;
74 private static Configuration conf;
75
76 static int insertData(Configuration conf, String tableName, String column, double prob)
77 throws IOException {
78 Random rng = new Random();
79 int count = 0;
80 HTable table = new HTable(conf, tableName);
81 byte[] k = new byte[3];
82 byte [][] famAndQf = KeyValue.parseColumn(Bytes.toBytes(column));
83 for (byte b1 = 'a'; b1 < 'z'; b1++) {
84 for (byte b2 = 'a'; b2 < 'z'; b2++) {
85 for (byte b3 = 'a'; b3 < 'z'; b3++) {
86 if (rng.nextDouble() < prob) {
87 k[0] = b1;
88 k[1] = b2;
89 k[2] = b3;
90 Put put = new Put(k);
91 put.setDurability(Durability.SKIP_WAL);
92 put.add(famAndQf[0], famAndQf[1], k);
93 table.put(put);
94 count++;
95 }
96 }
97 }
98 }
99 table.flushCommits();
100 table.close();
101 return count;
102 }
103
104 static int countCellSet(CellSetModel model) {
105 int count = 0;
106 Iterator<RowModel> rows = model.getRows().iterator();
107 while (rows.hasNext()) {
108 RowModel row = rows.next();
109 Iterator<CellModel> cells = row.getCells().iterator();
110 while (cells.hasNext()) {
111 cells.next();
112 count++;
113 }
114 }
115 return count;
116 }
117
118 private static int fullTableScan(ScannerModel model) throws IOException {
119 model.setBatch(100);
120 Response response = client.put("/" + TABLE + "/scanner",
121 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
122 assertEquals(response.getCode(), 201);
123 String scannerURI = response.getLocation();
124 assertNotNull(scannerURI);
125 int count = 0;
126 while (true) {
127 response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
128 assertTrue(response.getCode() == 200 || response.getCode() == 204);
129 if (response.getCode() == 200) {
130 assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
131 CellSetModel cellSet = new CellSetModel();
132 cellSet.getObjectFromMessage(response.getBody());
133 Iterator<RowModel> rows = cellSet.getRows().iterator();
134 while (rows.hasNext()) {
135 RowModel row = rows.next();
136 Iterator<CellModel> cells = row.getCells().iterator();
137 while (cells.hasNext()) {
138 cells.next();
139 count++;
140 }
141 }
142 } else {
143 break;
144 }
145 }
146
147 response = client.delete(scannerURI);
148 assertEquals(response.getCode(), 200);
149 return count;
150 }
151
152 @BeforeClass
153 public static void setUpBeforeClass() throws Exception {
154 conf = TEST_UTIL.getConfiguration();
155 TEST_UTIL.startMiniCluster();
156 REST_TEST_UTIL.startServletContainer(conf);
157 client = new Client(new Cluster().add("localhost",
158 REST_TEST_UTIL.getServletPort()));
159 context = JAXBContext.newInstance(
160 CellModel.class,
161 CellSetModel.class,
162 RowModel.class,
163 ScannerModel.class);
164 marshaller = context.createMarshaller();
165 unmarshaller = context.createUnmarshaller();
166 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
167 if (admin.tableExists(TABLE)) {
168 return;
169 }
170 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
171 htd.addFamily(new HColumnDescriptor(CFA));
172 htd.addFamily(new HColumnDescriptor(CFB));
173 admin.createTable(htd);
174 expectedRows1 = insertData(TEST_UTIL.getConfiguration(), TABLE, COLUMN_1, 1.0);
175 expectedRows2 = insertData(TEST_UTIL.getConfiguration(), TABLE, COLUMN_2, 0.5);
176 }
177
178 @AfterClass
179 public static void tearDownAfterClass() throws Exception {
180 REST_TEST_UTIL.shutdownServletContainer();
181 TEST_UTIL.shutdownMiniCluster();
182 }
183
184 @Test
185 public void testSimpleScannerXML() throws IOException, JAXBException {
186 final int BATCH_SIZE = 5;
187
188 ScannerModel model = new ScannerModel();
189 model.setBatch(BATCH_SIZE);
190 model.addColumn(Bytes.toBytes(COLUMN_1));
191 StringWriter writer = new StringWriter();
192 marshaller.marshal(model, writer);
193 byte[] body = Bytes.toBytes(writer.toString());
194
195
196 conf.set("hbase.rest.readonly", "true");
197 Response response = client.put("/" + TABLE + "/scanner",
198 Constants.MIMETYPE_XML, body);
199 assertEquals(response.getCode(), 403);
200 String scannerURI = response.getLocation();
201 assertNull(scannerURI);
202
203
204 conf.set("hbase.rest.readonly", "false");
205 response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML,
206 body);
207 assertEquals(response.getCode(), 201);
208 scannerURI = response.getLocation();
209 assertNotNull(scannerURI);
210
211
212 response = client.get(scannerURI, Constants.MIMETYPE_XML);
213 assertEquals(response.getCode(), 200);
214 assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
215 CellSetModel cellSet = (CellSetModel)
216 unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
217
218 assertEquals(countCellSet(cellSet), BATCH_SIZE);
219
220
221 conf.set("hbase.rest.readonly", "true");
222 response = client.delete(scannerURI);
223 assertEquals(response.getCode(), 403);
224
225
226 conf.set("hbase.rest.readonly", "false");
227 response = client.delete(scannerURI);
228 assertEquals(response.getCode(), 200);
229 }
230
231 @Test
232 public void testSimpleScannerPB() throws IOException {
233 final int BATCH_SIZE = 10;
234
235 ScannerModel model = new ScannerModel();
236 model.setBatch(BATCH_SIZE);
237 model.addColumn(Bytes.toBytes(COLUMN_1));
238
239
240 conf.set("hbase.rest.readonly", "true");
241 Response response = client.put("/" + TABLE + "/scanner",
242 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
243 assertEquals(response.getCode(), 403);
244 String scannerURI = response.getLocation();
245 assertNull(scannerURI);
246
247
248 conf.set("hbase.rest.readonly", "false");
249 response = client.put("/" + TABLE + "/scanner",
250 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
251 assertEquals(response.getCode(), 201);
252 scannerURI = response.getLocation();
253 assertNotNull(scannerURI);
254
255
256 response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
257 assertEquals(response.getCode(), 200);
258 assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
259 CellSetModel cellSet = new CellSetModel();
260 cellSet.getObjectFromMessage(response.getBody());
261
262 assertEquals(countCellSet(cellSet), BATCH_SIZE);
263
264
265 conf.set("hbase.rest.readonly", "true");
266 response = client.delete(scannerURI);
267 assertEquals(response.getCode(), 403);
268
269
270 conf.set("hbase.rest.readonly", "false");
271 response = client.delete(scannerURI);
272 assertEquals(response.getCode(), 200);
273 }
274
275 @Test
276 public void testSimpleScannerBinary() throws IOException {
277
278 ScannerModel model = new ScannerModel();
279 model.setBatch(1);
280 model.addColumn(Bytes.toBytes(COLUMN_1));
281
282
283 conf.set("hbase.rest.readonly", "true");
284 Response response = client.put("/" + TABLE + "/scanner",
285 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
286 assertEquals(response.getCode(), 403);
287 String scannerURI = response.getLocation();
288 assertNull(scannerURI);
289
290
291 conf.set("hbase.rest.readonly", "false");
292 response = client.put("/" + TABLE + "/scanner",
293 Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
294 assertEquals(response.getCode(), 201);
295 scannerURI = response.getLocation();
296 assertNotNull(scannerURI);
297
298
299 response = client.get(scannerURI, Constants.MIMETYPE_BINARY);
300 assertEquals(response.getCode(), 200);
301 assertEquals(Constants.MIMETYPE_BINARY, response.getHeader("content-type"));
302
303 assertTrue(response.getBody().length > 0);
304
305 boolean foundRowHeader = false, foundColumnHeader = false,
306 foundTimestampHeader = false;
307 for (Header header: response.getHeaders()) {
308 if (header.getName().equals("X-Row")) {
309 foundRowHeader = true;
310 } else if (header.getName().equals("X-Column")) {
311 foundColumnHeader = true;
312 } else if (header.getName().equals("X-Timestamp")) {
313 foundTimestampHeader = true;
314 }
315 }
316 assertTrue(foundRowHeader);
317 assertTrue(foundColumnHeader);
318 assertTrue(foundTimestampHeader);
319
320
321 conf.set("hbase.rest.readonly", "true");
322 response = client.delete(scannerURI);
323 assertEquals(response.getCode(), 403);
324
325
326 conf.set("hbase.rest.readonly", "false");
327 response = client.delete(scannerURI);
328 assertEquals(response.getCode(), 200);
329 }
330
331 @Test
332 public void testFullTableScan() throws IOException {
333 ScannerModel model = new ScannerModel();
334 model.addColumn(Bytes.toBytes(COLUMN_1));
335 assertEquals(fullTableScan(model), expectedRows1);
336
337 model = new ScannerModel();
338 model.addColumn(Bytes.toBytes(COLUMN_2));
339 assertEquals(fullTableScan(model), expectedRows2);
340 }
341
342 @Test
343 public void testTableDoesNotExist() throws IOException, JAXBException {
344 ScannerModel model = new ScannerModel();
345 StringWriter writer = new StringWriter();
346 marshaller.marshal(model, writer);
347 byte[] body = Bytes.toBytes(writer.toString());
348 Response response = client.put("/" + NONEXISTENT_TABLE +
349 "/scanner", Constants.MIMETYPE_XML, body);
350 assertEquals(response.getCode(), 404);
351 }
352
353 }
354