1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.client;
21
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertNull;
24 import static org.junit.Assert.assertTrue;
25 import static org.junit.Assert.fail;
26
27 import java.util.ArrayList;
28 import java.util.Arrays;
29 import java.util.List;
30 import java.util.Random;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.hbase.HBaseTestingUtility;
35 import org.apache.hadoop.hbase.HColumnDescriptor;
36 import org.apache.hadoop.hbase.HRegionLocation;
37 import org.apache.hadoop.hbase.HTableDescriptor;
38 import org.apache.hadoop.hbase.TableName;
39 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
40 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
41 import org.apache.hadoop.hbase.testclassification.LargeTests;
42 import org.apache.hadoop.hbase.util.Bytes;
43 import org.apache.hadoop.hbase.util.Pair;
44 import org.junit.After;
45 import org.junit.AfterClass;
46 import org.junit.Before;
47 import org.junit.BeforeClass;
48 import org.junit.Test;
49 import org.junit.experimental.categories.Category;
50
51 @Category(LargeTests.class)
52 public class TestFromClientSide3 {
53 private static final Log LOG = LogFactory.getLog(TestFromClientSide3.class);
54 private final static HBaseTestingUtility TEST_UTIL
55 = new HBaseTestingUtility();
56 private static byte[] FAMILY = Bytes.toBytes("testFamily");
57 private static Random random = new Random();
58 private static int SLAVES = 3;
59 private static byte [] ROW = Bytes.toBytes("testRow");
60 private static final byte[] ANOTHERROW = Bytes.toBytes("anotherrow");
61 private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
62 private static byte [] VALUE = Bytes.toBytes("testValue");
63 private final static byte[] COL_QUAL = Bytes.toBytes("f1");
64 private final static byte[] VAL_BYTES = Bytes.toBytes("v1");
65 private final static byte[] ROW_BYTES = Bytes.toBytes("r1");
66
67
68
69
70 @BeforeClass
71 public static void setUpBeforeClass() throws Exception {
72 TEST_UTIL.getConfiguration().setBoolean(
73 "hbase.online.schema.update.enable", true);
74 TEST_UTIL.startMiniCluster(SLAVES);
75 }
76
77
78
79
80 @AfterClass
81 public static void tearDownAfterClass() throws Exception {
82 TEST_UTIL.shutdownMiniCluster();
83 }
84
85
86
87
88 @Before
89 public void setUp() throws Exception {
90
91 }
92
93
94
95
96 @After
97 public void tearDown() throws Exception {
98 for (HTableDescriptor htd: TEST_UTIL.getHBaseAdmin().listTables()) {
99 LOG.info("Tear down, remove table=" + htd.getTableName());
100 TEST_UTIL.deleteTable(htd.getTableName());
101 }
102 }
103
104 private void randomCFPuts(Table table, byte[] row, byte[] family, int nPuts)
105 throws Exception {
106 Put put = new Put(row);
107 for (int i = 0; i < nPuts; i++) {
108 byte[] qualifier = Bytes.toBytes(random.nextInt());
109 byte[] value = Bytes.toBytes(random.nextInt());
110 put.add(family, qualifier, value);
111 }
112 table.put(put);
113 }
114
115 private void performMultiplePutAndFlush(HBaseAdmin admin, HTable table,
116 byte[] row, byte[] family, int nFlushes, int nPuts)
117 throws Exception {
118
119
120 HRegionLocation loc = table.getRegionLocation(row, true);
121 AdminProtos.AdminService.BlockingInterface server =
122 admin.getConnection().getAdmin(loc.getServerName());
123 byte[] regName = loc.getRegionInfo().getRegionName();
124
125 for (int i = 0; i < nFlushes; i++) {
126 randomCFPuts(table, row, family, nPuts);
127 List<String> sf = ProtobufUtil.getStoreFiles(server, regName, FAMILY);
128 int sfCount = sf.size();
129
130
131 admin.flush(table.getTableName());
132
133
134 while (ProtobufUtil.getStoreFiles(
135 server, regName, FAMILY).size() == sfCount) {
136 Thread.sleep(40);
137 }
138 }
139 }
140
141
142 @Test(timeout = 60000)
143 public void testAdvancedConfigOverride() throws Exception {
144
145
146
147
148
149
150
151
152
153 TEST_UTIL.getConfiguration().setInt("hbase.hstore.compaction.min", 3);
154
155 String tableName = "testAdvancedConfigOverride";
156 TableName TABLE = TableName.valueOf(tableName);
157 HTable hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
158 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
159 ClusterConnection connection = (ClusterConnection)TEST_UTIL.getConnection();
160
161
162 byte[] row = Bytes.toBytes(random.nextInt());
163 performMultiplePutAndFlush(admin, hTable, row, FAMILY, 3, 100);
164
165
166 HRegionLocation loc = hTable.getRegionLocation(row, true);
167 byte[] regionName = loc.getRegionInfo().getRegionName();
168 AdminProtos.AdminService.BlockingInterface server =
169 connection.getAdmin(loc.getServerName());
170 assertTrue(ProtobufUtil.getStoreFiles(
171 server, regionName, FAMILY).size() > 1);
172
173
174 admin.compact(TABLE.getName());
175
176
177 for (int i = 0; i < 10 * 1000 / 40; ++i) {
178
179 loc = hTable.getRegionLocation(row, true);
180 if (!loc.getRegionInfo().isOffline()) {
181 regionName = loc.getRegionInfo().getRegionName();
182 server = connection.getAdmin(loc.getServerName());
183 if (ProtobufUtil.getStoreFiles(
184 server, regionName, FAMILY).size() <= 1) {
185 break;
186 }
187 }
188 Thread.sleep(40);
189 }
190
191 assertTrue(ProtobufUtil.getStoreFiles(
192 server, regionName, FAMILY).size() <= 1);
193
194
195 LOG.info("hbase.hstore.compaction.min should now be 5");
196 HTableDescriptor htd = new HTableDescriptor(hTable.getTableDescriptor());
197 htd.setValue("hbase.hstore.compaction.min", String.valueOf(5));
198 admin.modifyTable(TABLE, htd);
199 Pair<Integer, Integer> st;
200 while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
201 LOG.debug(st.getFirst() + " regions left to update");
202 Thread.sleep(40);
203 }
204 LOG.info("alter status finished");
205
206
207 performMultiplePutAndFlush(admin, hTable, row, FAMILY, 3, 10);
208
209
210 admin.compact(TABLE.getName());
211
212
213 Thread.sleep(10 * 1000);
214 loc = hTable.getRegionLocation(row, true);
215 regionName = loc.getRegionInfo().getRegionName();
216 server = connection.getAdmin(loc.getServerName());
217 int sfCount = ProtobufUtil.getStoreFiles(
218 server, regionName, FAMILY).size();
219 assertTrue(sfCount > 1);
220
221
222 LOG.info("hbase.hstore.compaction.min should now be 2");
223 HColumnDescriptor hcd = new HColumnDescriptor(htd.getFamily(FAMILY));
224 hcd.setValue("hbase.hstore.compaction.min", String.valueOf(2));
225 htd.modifyFamily(hcd);
226 admin.modifyTable(TABLE, htd);
227 while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
228 LOG.debug(st.getFirst() + " regions left to update");
229 Thread.sleep(40);
230 }
231 LOG.info("alter status finished");
232
233
234 admin.compact(TABLE.getName());
235
236
237 for (int i = 0; i < 10 * 1000 / 40; ++i) {
238 loc = hTable.getRegionLocation(row, true);
239 regionName = loc.getRegionInfo().getRegionName();
240 try {
241 server = connection.getAdmin(loc.getServerName());
242 if (ProtobufUtil.getStoreFiles(
243 server, regionName, FAMILY).size() < sfCount) {
244 break;
245 }
246 } catch (Exception e) {
247 LOG.debug("Waiting for region to come online: " + regionName);
248 }
249 Thread.sleep(40);
250 }
251
252 assertTrue(ProtobufUtil.getStoreFiles(
253 server, regionName, FAMILY).size() < sfCount);
254
255
256 LOG.info("Removing CF config value");
257 LOG.info("hbase.hstore.compaction.min should now be 5");
258 hcd = new HColumnDescriptor(htd.getFamily(FAMILY));
259 hcd.setValue("hbase.hstore.compaction.min", null);
260 htd.modifyFamily(hcd);
261 admin.modifyTable(TABLE, htd);
262 while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
263 LOG.debug(st.getFirst() + " regions left to update");
264 Thread.sleep(40);
265 }
266 LOG.info("alter status finished");
267 assertNull(hTable.getTableDescriptor().getFamily(FAMILY).getValue(
268 "hbase.hstore.compaction.min"));
269 }
270
271 @Test
272 public void testHTableBatchWithEmptyPut() throws Exception {
273 Table table = TEST_UTIL.createTable(
274 Bytes.toBytes("testHTableBatchWithEmptyPut"), new byte[][] { FAMILY });
275 try {
276 List actions = (List) new ArrayList();
277 Object[] results = new Object[2];
278
279 Put put1 = new Put(ROW);
280 actions.add(put1);
281
282 Put put2 = new Put(ANOTHERROW);
283 put2.add(FAMILY, QUALIFIER, VALUE);
284 actions.add(put2);
285
286 table.batch(actions, results);
287 fail("Empty Put should have failed the batch call");
288 } catch (IllegalArgumentException iae) {
289
290 } finally {
291 table.close();
292 }
293 }
294
295 @Test
296 public void testHTableExistsMethodSingleRegionSingleGet() throws Exception {
297
298
299
300 Table table = TEST_UTIL.createTable(
301 Bytes.toBytes("testHTableExistsMethodSingleRegionSingleGet"), new byte[][] { FAMILY });
302
303 Put put = new Put(ROW);
304 put.add(FAMILY, QUALIFIER, VALUE);
305
306 Get get = new Get(ROW);
307
308 boolean exist = table.exists(get);
309 assertEquals(exist, false);
310
311 table.put(put);
312
313 exist = table.exists(get);
314 assertEquals(exist, true);
315 }
316
317 public void testHTableExistsMethodSingleRegionMultipleGets() throws Exception {
318
319 HTable table = TEST_UTIL.createTable(
320 Bytes.toBytes("testHTableExistsMethodSingleRegionMultipleGets"), new byte[][] { FAMILY });
321
322 Put put = new Put(ROW);
323 put.add(FAMILY, QUALIFIER, VALUE);
324 table.put(put);
325
326 List<Get> gets = new ArrayList<Get>();
327 gets.add(new Get(ROW));
328 gets.add(null);
329 gets.add(new Get(ANOTHERROW));
330
331 Boolean[] results = table.exists(gets);
332 assertEquals(results[0], true);
333 assertEquals(results[1], false);
334 assertEquals(results[2], false);
335 }
336
337 @Test
338 public void testHTableExistsBeforeGet() throws Exception {
339 Table table = TEST_UTIL.createTable(
340 Bytes.toBytes("testHTableExistsBeforeGet"), new byte[][] { FAMILY });
341 try {
342 Put put = new Put(ROW);
343 put.add(FAMILY, QUALIFIER, VALUE);
344 table.put(put);
345
346 Get get = new Get(ROW);
347
348 boolean exist = table.exists(get);
349 assertEquals(true, exist);
350
351 Result result = table.get(get);
352 assertEquals(false, result.isEmpty());
353 assertTrue(Bytes.equals(VALUE, result.getValue(FAMILY, QUALIFIER)));
354 } finally {
355 table.close();
356 }
357 }
358
359 @Test
360 public void testHTableExistsAllBeforeGet() throws Exception {
361 final byte[] ROW2 = Bytes.add(ROW, Bytes.toBytes("2"));
362 Table table = TEST_UTIL.createTable(
363 Bytes.toBytes("testHTableExistsAllBeforeGet"), new byte[][] { FAMILY });
364 try {
365 Put put = new Put(ROW);
366 put.add(FAMILY, QUALIFIER, VALUE);
367 table.put(put);
368 put = new Put(ROW2);
369 put.add(FAMILY, QUALIFIER, VALUE);
370 table.put(put);
371
372 Get get = new Get(ROW);
373 Get get2 = new Get(ROW2);
374 ArrayList<Get> getList = new ArrayList(2);
375 getList.add(get);
376 getList.add(get2);
377
378 boolean[] exists = table.existsAll(getList);
379 assertEquals(true, exists[0]);
380 assertEquals(true, exists[1]);
381
382 Result[] result = table.get(getList);
383 assertEquals(false, result[0].isEmpty());
384 assertTrue(Bytes.equals(VALUE, result[0].getValue(FAMILY, QUALIFIER)));
385 assertEquals(false, result[1].isEmpty());
386 assertTrue(Bytes.equals(VALUE, result[1].getValue(FAMILY, QUALIFIER)));
387 } finally {
388 table.close();
389 }
390 }
391
392 @Test
393 public void testHTableExistsMethodMultipleRegionsSingleGet() throws Exception {
394
395 Table table = TEST_UTIL.createTable(
396 TableName.valueOf("testHTableExistsMethodMultipleRegionsSingleGet"), new byte[][] { FAMILY },
397 1, new byte[] { 0x00 }, new byte[] { (byte) 0xff }, 255);
398 Put put = new Put(ROW);
399 put.add(FAMILY, QUALIFIER, VALUE);
400
401 Get get = new Get(ROW);
402
403 boolean exist = table.exists(get);
404 assertEquals(exist, false);
405
406 table.put(put);
407
408 exist = table.exists(get);
409 assertEquals(exist, true);
410 }
411
412 @Test
413 public void testHTableExistsMethodMultipleRegionsMultipleGets() throws Exception {
414 HTable table = TEST_UTIL.createTable(
415 TableName.valueOf("testHTableExistsMethodMultipleRegionsMultipleGets"),
416 new byte[][] { FAMILY }, 1, new byte[] { 0x00 }, new byte[] { (byte) 0xff }, 255);
417 Put put = new Put(ROW);
418 put.add(FAMILY, QUALIFIER, VALUE);
419 table.put (put);
420
421 List<Get> gets = new ArrayList<Get>();
422 gets.add(new Get(ANOTHERROW));
423 gets.add(new Get(Bytes.add(ROW, new byte[] { 0x00 })));
424 gets.add(new Get(ROW));
425 gets.add(new Get(Bytes.add(ANOTHERROW, new byte[] { 0x00 })));
426
427 LOG.info("Calling exists");
428 Boolean[] results = table.exists(gets);
429 assertEquals(results[0], false);
430 assertEquals(results[1], false);
431 assertEquals(results[2], true);
432 assertEquals(results[3], false);
433
434
435 put = new Put(new byte[] { 0x00 });
436 put.add(FAMILY, QUALIFIER, VALUE);
437 table.put(put);
438
439 gets = new ArrayList<Get>();
440 gets.add(new Get(new byte[] { 0x00 }));
441 gets.add(new Get(new byte[] { 0x00, 0x00 }));
442 results = table.exists(gets);
443 assertEquals(results[0], true);
444 assertEquals(results[1], false);
445
446
447 put = new Put(new byte[] { (byte) 0xff, (byte) 0xff });
448 put.add(FAMILY, QUALIFIER, VALUE);
449 table.put(put);
450
451 gets = new ArrayList<Get>();
452 gets.add(new Get(new byte[] { (byte) 0xff }));
453 gets.add(new Get(new byte[] { (byte) 0xff, (byte) 0xff }));
454 gets.add(new Get(new byte[] { (byte) 0xff, (byte) 0xff, (byte) 0xff }));
455 results = table.exists(gets);
456 assertEquals(results[0], false);
457 assertEquals(results[1], true);
458 assertEquals(results[2], false);
459 }
460
461 @Test
462 public void testGetEmptyRow() throws Exception {
463
464 Admin admin = TEST_UTIL.getHBaseAdmin();
465 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(Bytes.toBytes("test")));
466 desc.addFamily(new HColumnDescriptor(FAMILY));
467 admin.createTable(desc);
468 Table table = new HTable(TEST_UTIL.getConfiguration(), desc.getTableName());
469
470 Put put = new Put(ROW_BYTES);
471 put.add(FAMILY, COL_QUAL, VAL_BYTES);
472 table.put(put);
473
474
475 Result res = null;
476 try {
477 res = table.get(new Get(new byte[0]));
478 fail();
479 } catch (IllegalArgumentException e) {
480
481 }
482 assertTrue(res == null);
483 res = table.get(new Get(Bytes.toBytes("r1-not-exist")));
484 assertTrue(res.isEmpty() == true);
485 res = table.get(new Get(ROW_BYTES));
486 assertTrue(Arrays.equals(res.getValue(FAMILY, COL_QUAL), VAL_BYTES));
487 table.close();
488 }
489 }