1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.thrift2;
19
20 import static java.nio.ByteBuffer.wrap;
21 import static org.junit.Assert.assertArrayEquals;
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertNull;
24 import static org.junit.Assert.fail;
25
26 import java.io.IOException;
27 import java.nio.ByteBuffer;
28 import java.security.PrivilegedExceptionAction;
29 import java.util.ArrayList;
30 import java.util.Collections;
31 import java.util.Comparator;
32 import java.util.List;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.hbase.HBaseTestingUtility;
38 import org.apache.hadoop.hbase.HColumnDescriptor;
39 import org.apache.hadoop.hbase.HTableDescriptor;
40 import org.apache.hadoop.hbase.testclassification.MediumTests;
41 import org.apache.hadoop.hbase.TableName;
42 import org.apache.hadoop.hbase.client.Admin;
43 import org.apache.hadoop.hbase.client.Connection;
44 import org.apache.hadoop.hbase.client.ConnectionFactory;
45 import org.apache.hadoop.hbase.client.HBaseAdmin;
46 import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse;
47 import org.apache.hadoop.hbase.security.User;
48 import org.apache.hadoop.hbase.security.UserProvider;
49 import org.apache.hadoop.hbase.security.visibility.ScanLabelGenerator;
50 import org.apache.hadoop.hbase.security.visibility.SimpleScanLabelGenerator;
51 import org.apache.hadoop.hbase.security.visibility.VisibilityClient;
52 import org.apache.hadoop.hbase.security.visibility.VisibilityConstants;
53 import org.apache.hadoop.hbase.security.visibility.VisibilityController;
54 import org.apache.hadoop.hbase.security.visibility.VisibilityUtils;
55 import org.apache.hadoop.hbase.thrift2.generated.TAppend;
56 import org.apache.hadoop.hbase.thrift2.generated.TAuthorization;
57 import org.apache.hadoop.hbase.thrift2.generated.TCellVisibility;
58 import org.apache.hadoop.hbase.thrift2.generated.TColumn;
59 import org.apache.hadoop.hbase.thrift2.generated.TColumnIncrement;
60 import org.apache.hadoop.hbase.thrift2.generated.TColumnValue;
61 import org.apache.hadoop.hbase.thrift2.generated.TGet;
62 import org.apache.hadoop.hbase.thrift2.generated.TIllegalArgument;
63 import org.apache.hadoop.hbase.thrift2.generated.TIncrement;
64 import org.apache.hadoop.hbase.thrift2.generated.TPut;
65 import org.apache.hadoop.hbase.thrift2.generated.TResult;
66 import org.apache.hadoop.hbase.thrift2.generated.TScan;
67 import org.apache.hadoop.hbase.util.Bytes;
68 import org.junit.AfterClass;
69 import org.junit.Assert;
70 import org.junit.Before;
71 import org.junit.BeforeClass;
72 import org.junit.Test;
73 import org.junit.experimental.categories.Category;
74
75 @Category(MediumTests.class)
76 public class TestThriftHBaseServiceHandlerWithLabels {
77
78 private static final Log LOG = LogFactory
79 .getLog(TestThriftHBaseServiceHandlerWithLabels.class);
80 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
81
82
83 private static byte[] tableAname = Bytes.toBytes("tableA");
84 private static byte[] familyAname = Bytes.toBytes("familyA");
85 private static byte[] familyBname = Bytes.toBytes("familyB");
86 private static byte[] qualifierAname = Bytes.toBytes("qualifierA");
87 private static byte[] qualifierBname = Bytes.toBytes("qualifierB");
88 private static byte[] valueAname = Bytes.toBytes("valueA");
89 private static byte[] valueBname = Bytes.toBytes("valueB");
90 private static HColumnDescriptor[] families = new HColumnDescriptor[] {
91 new HColumnDescriptor(familyAname).setMaxVersions(3),
92 new HColumnDescriptor(familyBname).setMaxVersions(2) };
93
94 private final static String TOPSECRET = "topsecret";
95 private final static String PUBLIC = "public";
96 private final static String PRIVATE = "private";
97 private final static String CONFIDENTIAL = "confidential";
98 private final static String SECRET = "secret";
99 private static User SUPERUSER;
100
101 private static Configuration conf;
102
103 public void assertTColumnValuesEqual(List<TColumnValue> columnValuesA,
104 List<TColumnValue> columnValuesB) {
105 assertEquals(columnValuesA.size(), columnValuesB.size());
106 Comparator<TColumnValue> comparator = new Comparator<TColumnValue>() {
107 @Override
108 public int compare(TColumnValue o1, TColumnValue o2) {
109 return Bytes.compareTo(Bytes.add(o1.getFamily(), o1.getQualifier()),
110 Bytes.add(o2.getFamily(), o2.getQualifier()));
111 }
112 };
113 Collections.sort(columnValuesA, comparator);
114 Collections.sort(columnValuesB, comparator);
115
116 for (int i = 0; i < columnValuesA.size(); i++) {
117 TColumnValue a = columnValuesA.get(i);
118 TColumnValue b = columnValuesB.get(i);
119 assertArrayEquals(a.getFamily(), b.getFamily());
120 assertArrayEquals(a.getQualifier(), b.getQualifier());
121 assertArrayEquals(a.getValue(), b.getValue());
122 }
123 }
124
125 @BeforeClass
126 public static void beforeClass() throws Exception {
127 SUPERUSER = User.createUserForTesting(conf, "admin",
128 new String[] { "supergroup" });
129 conf = UTIL.getConfiguration();
130 conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS,
131 SimpleScanLabelGenerator.class, ScanLabelGenerator.class);
132 conf.set("hbase.superuser", SUPERUSER.getShortName());
133 conf.set("hbase.coprocessor.master.classes",
134 VisibilityController.class.getName());
135 conf.set("hbase.coprocessor.region.classes",
136 VisibilityController.class.getName());
137 conf.setInt("hfile.format.version", 3);
138 UTIL.startMiniCluster(1);
139
140 UTIL.waitTableEnabled(VisibilityConstants.LABELS_TABLE_NAME.getName(), 50000);
141 createLabels();
142 Admin admin = new HBaseAdmin(UTIL.getConfiguration());
143 HTableDescriptor tableDescriptor = new HTableDescriptor(
144 TableName.valueOf(tableAname));
145 for (HColumnDescriptor family : families) {
146 tableDescriptor.addFamily(family);
147 }
148 admin.createTable(tableDescriptor);
149 admin.close();
150 setAuths();
151 }
152
153 private static void createLabels() throws IOException, InterruptedException {
154 PrivilegedExceptionAction<VisibilityLabelsResponse> action = new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
155 public VisibilityLabelsResponse run() throws Exception {
156 String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, PUBLIC, TOPSECRET };
157 try (Connection conn = ConnectionFactory.createConnection(conf)) {
158 VisibilityClient.addLabels(conn, labels);
159 } catch (Throwable t) {
160 throw new IOException(t);
161 }
162 return null;
163 }
164 };
165 SUPERUSER.runAs(action);
166 }
167
168 private static void setAuths() throws IOException {
169 String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, PUBLIC, TOPSECRET };
170 try {
171 VisibilityClient.setAuths(UTIL.getConnection(), labels, User.getCurrent().getShortName());
172 } catch (Throwable t) {
173 throw new IOException(t);
174 }
175 }
176
177 @AfterClass
178 public static void afterClass() throws Exception {
179 UTIL.shutdownMiniCluster();
180 }
181
182 @Before
183 public void setup() throws Exception {
184
185 }
186
187 private ThriftHBaseServiceHandler createHandler() throws IOException {
188 return new ThriftHBaseServiceHandler(conf, UserProvider.instantiate(conf));
189 }
190
191 @Test
192 public void testScanWithVisibilityLabels() throws Exception {
193 ThriftHBaseServiceHandler handler = createHandler();
194 ByteBuffer table = wrap(tableAname);
195
196
197 TColumnValue columnValue = new TColumnValue(wrap(familyAname),
198 wrap(qualifierAname), wrap(valueAname));
199 List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
200 columnValues.add(columnValue);
201 for (int i = 0; i < 10; i++) {
202 TPut put = new TPut(wrap(("testScan" + i).getBytes()), columnValues);
203 if (i == 5) {
204 put.setCellVisibility(new TCellVisibility().setExpression(PUBLIC));
205 } else {
206 put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET
207 + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
208 }
209 handler.put(table, put);
210 }
211
212
213 TScan scan = new TScan();
214 List<TColumn> columns = new ArrayList<TColumn>();
215 TColumn column = new TColumn();
216 column.setFamily(familyAname);
217 column.setQualifier(qualifierAname);
218 columns.add(column);
219 scan.setColumns(columns);
220 scan.setStartRow("testScan".getBytes());
221 scan.setStopRow("testScan\uffff".getBytes());
222
223 TAuthorization tauth = new TAuthorization();
224 List<String> labels = new ArrayList<String>();
225 labels.add(SECRET);
226 labels.add(PRIVATE);
227 tauth.setLabels(labels);
228 scan.setAuthorizations(tauth);
229
230 int scanId = handler.openScanner(table, scan);
231 List<TResult> results = handler.getScannerRows(scanId, 10);
232 assertEquals(9, results.size());
233 Assert.assertFalse(Bytes.equals(results.get(5).getRow(),
234 ("testScan" + 5).getBytes()));
235 for (int i = 0; i < 9; i++) {
236 if (i < 5) {
237 assertArrayEquals(("testScan" + i).getBytes(), results.get(i).getRow());
238 } else if (i == 5) {
239 continue;
240 } else {
241 assertArrayEquals(("testScan" + (i + 1)).getBytes(), results.get(i)
242 .getRow());
243 }
244 }
245
246
247 results = handler.getScannerRows(scanId, 9);
248 assertEquals(0, results.size());
249
250
251 handler.closeScanner(scanId);
252 try {
253 handler.getScannerRows(scanId, 9);
254 fail("Scanner id should be invalid");
255 } catch (TIllegalArgument e) {
256 }
257 }
258
259 @Test
260 public void testGetScannerResultsWithAuthorizations() throws Exception {
261 ThriftHBaseServiceHandler handler = createHandler();
262 ByteBuffer table = wrap(tableAname);
263
264
265 TColumnValue columnValue = new TColumnValue(wrap(familyAname),
266 wrap(qualifierAname), wrap(valueAname));
267 List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
268 columnValues.add(columnValue);
269 for (int i = 0; i < 20; i++) {
270 TPut put = new TPut(
271 wrap(("testGetScannerResults" + pad(i, (byte) 2)).getBytes()),
272 columnValues);
273 if (i == 3) {
274 put.setCellVisibility(new TCellVisibility().setExpression(PUBLIC));
275 } else {
276 put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET
277 + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
278 }
279 handler.put(table, put);
280 }
281
282
283 TScan scan = new TScan();
284 List<TColumn> columns = new ArrayList<TColumn>();
285 TColumn column = new TColumn();
286 column.setFamily(familyAname);
287 column.setQualifier(qualifierAname);
288 columns.add(column);
289 scan.setColumns(columns);
290 scan.setStartRow("testGetScannerResults".getBytes());
291
292
293 scan.setStopRow("testGetScannerResults05".getBytes());
294 TAuthorization tauth = new TAuthorization();
295 List<String> labels = new ArrayList<String>();
296 labels.add(SECRET);
297 labels.add(PRIVATE);
298 tauth.setLabels(labels);
299 scan.setAuthorizations(tauth);
300 List<TResult> results = handler.getScannerResults(table, scan, 5);
301 assertEquals(4, results.size());
302 for (int i = 0; i < 4; i++) {
303 if (i < 3) {
304 assertArrayEquals(
305 ("testGetScannerResults" + pad(i, (byte) 2)).getBytes(),
306 results.get(i).getRow());
307 } else if (i == 3) {
308 continue;
309 } else {
310 assertArrayEquals(
311 ("testGetScannerResults" + pad(i + 1, (byte) 2)).getBytes(), results
312 .get(i).getRow());
313 }
314 }
315 }
316
317 @Test
318 public void testGetsWithLabels() throws Exception {
319 ThriftHBaseServiceHandler handler = createHandler();
320 byte[] rowName = "testPutGet".getBytes();
321 ByteBuffer table = wrap(tableAname);
322
323 List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
324 columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
325 wrap(valueAname)));
326 columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname),
327 wrap(valueBname)));
328 TPut put = new TPut(wrap(rowName), columnValues);
329
330 put.setColumnValues(columnValues);
331 put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET + "|"
332 + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
333 handler.put(table, put);
334 TGet get = new TGet(wrap(rowName));
335 TAuthorization tauth = new TAuthorization();
336 List<String> labels = new ArrayList<String>();
337 labels.add(SECRET);
338 labels.add(PRIVATE);
339 tauth.setLabels(labels);
340 get.setAuthorizations(tauth);
341 TResult result = handler.get(table, get);
342 assertArrayEquals(rowName, result.getRow());
343 List<TColumnValue> returnedColumnValues = result.getColumnValues();
344 assertTColumnValuesEqual(columnValues, returnedColumnValues);
345 }
346
347 @Test
348 public void testIncrementWithTags() throws Exception {
349 ThriftHBaseServiceHandler handler = createHandler();
350 byte[] rowName = "testIncrementWithTags".getBytes();
351 ByteBuffer table = wrap(tableAname);
352
353 List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
354 columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
355 wrap(Bytes.toBytes(1L))));
356 TPut put = new TPut(wrap(rowName), columnValues);
357 put.setColumnValues(columnValues);
358 put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
359 handler.put(table, put);
360
361 List<TColumnIncrement> incrementColumns = new ArrayList<TColumnIncrement>();
362 incrementColumns.add(new TColumnIncrement(wrap(familyAname),
363 wrap(qualifierAname)));
364 TIncrement increment = new TIncrement(wrap(rowName), incrementColumns);
365 increment.setCellVisibility(new TCellVisibility().setExpression(SECRET));
366 handler.increment(table, increment);
367
368 TGet get = new TGet(wrap(rowName));
369 TAuthorization tauth = new TAuthorization();
370 List<String> labels = new ArrayList<String>();
371 labels.add(SECRET);
372 tauth.setLabels(labels);
373 get.setAuthorizations(tauth);
374 TResult result = handler.get(table, get);
375
376 assertArrayEquals(rowName, result.getRow());
377 assertEquals(1, result.getColumnValuesSize());
378 TColumnValue columnValue = result.getColumnValues().get(0);
379 assertArrayEquals(Bytes.toBytes(2L), columnValue.getValue());
380 }
381
382 @Test
383 public void testIncrementWithTagsWithNotMatchLabels() throws Exception {
384 ThriftHBaseServiceHandler handler = createHandler();
385 byte[] rowName = "testIncrementWithTagsWithNotMatchLabels".getBytes();
386 ByteBuffer table = wrap(tableAname);
387
388 List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
389 columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
390 wrap(Bytes.toBytes(1L))));
391 TPut put = new TPut(wrap(rowName), columnValues);
392 put.setColumnValues(columnValues);
393 put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
394 handler.put(table, put);
395
396 List<TColumnIncrement> incrementColumns = new ArrayList<TColumnIncrement>();
397 incrementColumns.add(new TColumnIncrement(wrap(familyAname),
398 wrap(qualifierAname)));
399 TIncrement increment = new TIncrement(wrap(rowName), incrementColumns);
400 increment.setCellVisibility(new TCellVisibility().setExpression(SECRET));
401 handler.increment(table, increment);
402
403 TGet get = new TGet(wrap(rowName));
404 TAuthorization tauth = new TAuthorization();
405 List<String> labels = new ArrayList<String>();
406 labels.add(PUBLIC);
407 tauth.setLabels(labels);
408 get.setAuthorizations(tauth);
409 TResult result = handler.get(table, get);
410 assertNull(result.getRow());
411 }
412
413 @Test
414 public void testAppend() throws Exception {
415 ThriftHBaseServiceHandler handler = createHandler();
416 byte[] rowName = "testAppend".getBytes();
417 ByteBuffer table = wrap(tableAname);
418 byte[] v1 = Bytes.toBytes(1L);
419 byte[] v2 = Bytes.toBytes(5L);
420 List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
421 columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
422 wrap(Bytes.toBytes(1L))));
423 TPut put = new TPut(wrap(rowName), columnValues);
424 put.setColumnValues(columnValues);
425 put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
426 handler.put(table, put);
427
428 List<TColumnValue> appendColumns = new ArrayList<TColumnValue>();
429 appendColumns.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
430 wrap(v2)));
431 TAppend append = new TAppend(wrap(rowName), appendColumns);
432 append.setCellVisibility(new TCellVisibility().setExpression(SECRET));
433 handler.append(table, append);
434
435 TGet get = new TGet(wrap(rowName));
436 TAuthorization tauth = new TAuthorization();
437 List<String> labels = new ArrayList<String>();
438 labels.add(SECRET);
439 tauth.setLabels(labels);
440 get.setAuthorizations(tauth);
441 TResult result = handler.get(table, get);
442
443 assertArrayEquals(rowName, result.getRow());
444 assertEquals(1, result.getColumnValuesSize());
445 TColumnValue columnValue = result.getColumnValues().get(0);
446 assertArrayEquals(Bytes.add(v1, v2), columnValue.getValue());
447 }
448
449
450
451
452
453
454
455
456
457
458 private String pad(int n, byte pad) {
459 String res = Integer.toString(n);
460 while (res.length() < pad)
461 res = "0" + res;
462 return res;
463 }
464 }