1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.client;
20
21 import static org.junit.Assert.assertEquals;
22
23 import java.util.ArrayList;
24 import java.util.LinkedList;
25 import java.util.List;
26 import java.util.Map.Entry;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.hbase.HBaseTestingUtility;
32 import org.apache.hadoop.hbase.HColumnDescriptor;
33 import org.apache.hadoop.hbase.HTableDescriptor;
34 import org.apache.hadoop.hbase.TableName;
35 import org.apache.hadoop.hbase.testclassification.LargeTests;
36 import org.apache.hadoop.hbase.util.Bytes;
37 import org.junit.AfterClass;
38 import org.junit.BeforeClass;
39 import org.junit.Test;
40 import org.junit.experimental.categories.Category;
41
42 import com.google.common.collect.Maps;
43
44 @Category(LargeTests.class)
45 public class TestSizeFailures {
46 private static final Log LOG = LogFactory.getLog(TestSizeFailures.class);
47 protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
48 private static byte [] FAMILY = Bytes.toBytes("testFamily");
49 protected static int SLAVES = 1;
50 private static TableName TABLENAME;
51 private static final int NUM_ROWS = 1000 * 1000, NUM_COLS = 10;
52
53 @BeforeClass
54 public static void setUpBeforeClass() throws Exception {
55
56
57
58
59
60 Configuration conf = TEST_UTIL.getConfiguration();
61 conf.setBoolean("hbase.table.sanity.checks", true);
62 TEST_UTIL.startMiniCluster(SLAVES);
63
64
65 TABLENAME = TableName.valueOf("testSizeFailures");
66 List<byte[]> qualifiers = new ArrayList<>();
67 for (int i = 1; i <= 10; i++) {
68 qualifiers.add(Bytes.toBytes(Integer.toString(i)));
69 }
70
71 HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
72 HTableDescriptor desc = new HTableDescriptor(TABLENAME);
73 desc.addFamily(hcd);
74 byte[][] splits = new byte[9][2];
75 for (int i = 1; i < 10; i++) {
76 int split = 48 + i;
77 splits[i - 1][0] = (byte) (split >>> 8);
78 splits[i - 1][0] = (byte) (split);
79 }
80 TEST_UTIL.getHBaseAdmin().createTable(desc, splits);
81 Connection conn = TEST_UTIL.getConnection();
82
83 try (Table table = conn.getTable(TABLENAME)) {
84 List<Put> puts = new LinkedList<>();
85 for (int i = 0; i < NUM_ROWS; i++) {
86 Put p = new Put(Bytes.toBytes(Integer.toString(i)));
87 for (int j = 0; j < NUM_COLS; j++) {
88 byte[] value = new byte[50];
89 Bytes.random(value);
90 p.addColumn(FAMILY, Bytes.toBytes(Integer.toString(j)), value);
91 }
92 puts.add(p);
93
94 if (puts.size() == 1000) {
95 table.batch(puts, new Object[1000]);
96 puts.clear();
97 }
98 }
99
100 if (puts.size() > 0) {
101 table.batch(puts, new Object[puts.size()]);
102 }
103 }
104 }
105
106 @AfterClass
107 public static void tearDownAfterClass() throws Exception {
108 TEST_UTIL.shutdownMiniCluster();
109 }
110
111
112
113
114 @Test
115 public void testScannerSeesAllRecords() throws Exception {
116 Connection conn = TEST_UTIL.getConnection();
117 try (Table table = conn.getTable(TABLENAME)) {
118 Scan s = new Scan();
119 s.addFamily(FAMILY);
120 s.setMaxResultSize(-1);
121 s.setBatch(-1);
122 s.setCaching(500);
123 Entry<Long,Long> entry = sumTable(table.getScanner(s));
124 long rowsObserved = entry.getKey();
125 long entriesObserved = entry.getValue();
126
127
128 assertEquals(NUM_ROWS, rowsObserved);
129 assertEquals(NUM_ROWS * NUM_COLS, entriesObserved);
130 }
131 }
132
133
134
135
136 @Test
137 public void testSmallScannerSeesAllRecords() throws Exception {
138 Connection conn = TEST_UTIL.getConnection();
139 try (Table table = conn.getTable(TABLENAME)) {
140 Scan s = new Scan();
141 s.setSmall(true);
142 s.addFamily(FAMILY);
143 s.setMaxResultSize(-1);
144 s.setBatch(-1);
145 s.setCaching(500);
146 Entry<Long,Long> entry = sumTable(table.getScanner(s));
147 long rowsObserved = entry.getKey();
148 long entriesObserved = entry.getValue();
149
150
151 assertEquals(NUM_ROWS, rowsObserved);
152 assertEquals(NUM_ROWS * NUM_COLS, entriesObserved);
153 }
154 }
155
156
157
158
159
160
161
162
163 private Entry<Long,Long> sumTable(ResultScanner scanner) {
164 long rowsObserved = 0l;
165 long entriesObserved = 0l;
166
167
168 for (Result result : scanner) {
169 rowsObserved++;
170 while (result.advance()) {
171 entriesObserved++;
172 }
173 }
174 return Maps.immutableEntry(rowsObserved,entriesObserved);
175 }
176 }