1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertTrue;
24
25 import java.io.IOException;
26 import java.util.ArrayList;
27 import java.util.Collection;
28 import java.util.HashMap;
29 import java.util.HashSet;
30 import java.util.List;
31 import java.util.Set;
32
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.hbase.Cell;
36 import org.apache.hadoop.hbase.HBaseTestingUtility;
37 import org.apache.hadoop.hbase.HColumnDescriptor;
38 import org.apache.hadoop.hbase.HRegionInfo;
39 import org.apache.hadoop.hbase.HTableDescriptor;
40 import org.apache.hadoop.hbase.KeyValue;
41 import org.apache.hadoop.hbase.KeyValueTestUtil;
42 import org.apache.hadoop.hbase.TableName;
43 import org.apache.hadoop.hbase.client.Durability;
44 import org.apache.hadoop.hbase.client.Put;
45 import org.apache.hadoop.hbase.client.Scan;
46 import org.apache.hadoop.hbase.testclassification.SmallTests;
47 import org.apache.hadoop.hbase.util.Bytes;
48 import org.junit.Rule;
49 import org.junit.Test;
50 import org.junit.experimental.categories.Category;
51 import org.junit.rules.TestName;
52
53 @Category(SmallTests.class)
54 public class TestColumnSeeking {
55 @Rule public TestName name = new TestName();
56
57 private final static HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
58
59 private static final Log LOG = LogFactory.getLog(TestColumnSeeking.class);
60
61 @SuppressWarnings("unchecked")
62 @Test
63 public void testDuplicateVersions() throws IOException {
64 String family = "Family";
65 byte[] familyBytes = Bytes.toBytes("Family");
66 TableName table = TableName.valueOf(name.getMethodName());
67
68 HColumnDescriptor hcd =
69 new HColumnDescriptor(familyBytes).setMaxVersions(1000);
70 hcd.setMaxVersions(3);
71 HTableDescriptor htd = new HTableDescriptor(table);
72 htd.addFamily(hcd);
73 HRegionInfo info = new HRegionInfo(table, null, null, false);
74
75 HRegion region = TEST_UTIL.createLocalHRegion(info, htd);
76 try {
77 List<String> rows = generateRandomWords(10, "row");
78 List<String> allColumns = generateRandomWords(10, "column");
79 List<String> values = generateRandomWords(100, "value");
80
81 long maxTimestamp = 2;
82 double selectPercent = 0.5;
83 int numberOfTests = 5;
84 double flushPercentage = 0.2;
85 double minorPercentage = 0.2;
86 double majorPercentage = 0.2;
87 double putPercentage = 0.2;
88
89 HashMap<String, KeyValue> allKVMap = new HashMap<String, KeyValue>();
90
91 HashMap<String, KeyValue>[] kvMaps = new HashMap[numberOfTests];
92 ArrayList<String>[] columnLists = new ArrayList[numberOfTests];
93
94 for (int i = 0; i < numberOfTests; i++) {
95 kvMaps[i] = new HashMap<String, KeyValue>();
96 columnLists[i] = new ArrayList<String>();
97 for (String column : allColumns) {
98 if (Math.random() < selectPercent) {
99 columnLists[i].add(column);
100 }
101 }
102 }
103
104 for (String value : values) {
105 for (String row : rows) {
106 Put p = new Put(Bytes.toBytes(row));
107 p.setDurability(Durability.SKIP_WAL);
108 for (String column : allColumns) {
109 for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
110 KeyValue kv =
111 KeyValueTestUtil.create(row, family, column, timestamp, value);
112 if (Math.random() < putPercentage) {
113 p.add(kv);
114 allKVMap.put(kv.getKeyString(), kv);
115 for (int i = 0; i < numberOfTests; i++) {
116 if (columnLists[i].contains(column)) {
117 kvMaps[i].put(kv.getKeyString(), kv);
118 }
119 }
120 }
121 }
122 }
123 region.put(p);
124 if (Math.random() < flushPercentage) {
125 LOG.info("Flushing... ");
126 region.flush(true);
127 }
128
129 if (Math.random() < minorPercentage) {
130 LOG.info("Minor compacting... ");
131 region.compact(false);
132 }
133
134 if (Math.random() < majorPercentage) {
135 LOG.info("Major compacting... ");
136 region.compact(true);
137 }
138 }
139 }
140
141 for (int i = 0; i < numberOfTests + 1; i++) {
142 Collection<KeyValue> kvSet;
143 Scan scan = new Scan();
144 scan.setMaxVersions();
145 if (i < numberOfTests) {
146 if (columnLists[i].size() == 0) continue;
147 kvSet = kvMaps[i].values();
148 for (String column : columnLists[i]) {
149 scan.addColumn(familyBytes, Bytes.toBytes(column));
150 }
151 LOG.info("ExplicitColumns scanner");
152 LOG.info("Columns: " + columnLists[i].size() + " Keys: "
153 + kvSet.size());
154 } else {
155 kvSet = allKVMap.values();
156 LOG.info("Wildcard scanner");
157 LOG.info("Columns: " + allColumns.size() + " Keys: " + kvSet.size());
158
159 }
160 InternalScanner scanner = region.getScanner(scan);
161 List<Cell> results = new ArrayList<Cell>();
162 while (scanner.next(results))
163 ;
164 assertEquals(kvSet.size(), results.size());
165 assertTrue(KeyValueTestUtil.containsIgnoreMvccVersion(results, kvSet));
166 }
167 } finally {
168 HRegion.closeHRegion(region);
169 }
170
171 HRegion.closeHRegion(region);
172 }
173
174 @SuppressWarnings("unchecked")
175 @Test
176 public void testReseeking() throws IOException {
177 String family = "Family";
178 byte[] familyBytes = Bytes.toBytes("Family");
179 TableName table = TableName.valueOf(name.getMethodName());
180
181 HTableDescriptor htd = new HTableDescriptor(table);
182 HColumnDescriptor hcd = new HColumnDescriptor(family);
183 hcd.setMaxVersions(3);
184 htd.addFamily(hcd);
185
186 HRegionInfo info = new HRegionInfo(table, null, null, false);
187 HRegion region = TEST_UTIL.createLocalHRegion(info, htd);
188
189 List<String> rows = generateRandomWords(10, "row");
190 List<String> allColumns = generateRandomWords(100, "column");
191
192 long maxTimestamp = 2;
193 double selectPercent = 0.5;
194 int numberOfTests = 5;
195 double flushPercentage = 0.2;
196 double minorPercentage = 0.2;
197 double majorPercentage = 0.2;
198 double putPercentage = 0.2;
199
200 HashMap<String, KeyValue> allKVMap = new HashMap<String, KeyValue>();
201
202 HashMap<String, KeyValue>[] kvMaps = new HashMap[numberOfTests];
203 ArrayList<String>[] columnLists = new ArrayList[numberOfTests];
204 String valueString = "Value";
205
206 for (int i = 0; i < numberOfTests; i++) {
207 kvMaps[i] = new HashMap<String, KeyValue>();
208 columnLists[i] = new ArrayList<String>();
209 for (String column : allColumns) {
210 if (Math.random() < selectPercent) {
211 columnLists[i].add(column);
212 }
213 }
214 }
215
216 for (String row : rows) {
217 Put p = new Put(Bytes.toBytes(row));
218 p.setDurability(Durability.SKIP_WAL);
219 for (String column : allColumns) {
220 for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
221 KeyValue kv =
222 KeyValueTestUtil.create(row, family, column, timestamp,
223 valueString);
224 if (Math.random() < putPercentage) {
225 p.add(kv);
226 allKVMap.put(kv.getKeyString(), kv);
227 for (int i = 0; i < numberOfTests; i++) {
228 if (columnLists[i].contains(column)) {
229 kvMaps[i].put(kv.getKeyString(), kv);
230 }
231 }
232 }
233
234 }
235 }
236 region.put(p);
237 if (Math.random() < flushPercentage) {
238 LOG.info("Flushing... ");
239 region.flush(true);
240 }
241
242 if (Math.random() < minorPercentage) {
243 LOG.info("Minor compacting... ");
244 region.compact(false);
245 }
246
247 if (Math.random() < majorPercentage) {
248 LOG.info("Major compacting... ");
249 region.compact(true);
250 }
251 }
252
253 for (int i = 0; i < numberOfTests + 1; i++) {
254 Collection<KeyValue> kvSet;
255 Scan scan = new Scan();
256 scan.setMaxVersions();
257 if (i < numberOfTests) {
258 if (columnLists[i].size() == 0) continue;
259 kvSet = kvMaps[i].values();
260 for (String column : columnLists[i]) {
261 scan.addColumn(familyBytes, Bytes.toBytes(column));
262 }
263 LOG.info("ExplicitColumns scanner");
264 LOG.info("Columns: " + columnLists[i].size() + " Keys: "
265 + kvSet.size());
266 } else {
267 kvSet = allKVMap.values();
268 LOG.info("Wildcard scanner");
269 LOG.info("Columns: " + allColumns.size() + " Keys: " + kvSet.size());
270
271 }
272 InternalScanner scanner = region.getScanner(scan);
273 List<Cell> results = new ArrayList<Cell>();
274 while (scanner.next(results))
275 ;
276 assertEquals(kvSet.size(), results.size());
277 assertTrue(KeyValueTestUtil.containsIgnoreMvccVersion(results, kvSet));
278 }
279
280 HRegion.closeHRegion(region);
281 }
282
283 List<String> generateRandomWords(int numberOfWords, String suffix) {
284 Set<String> wordSet = new HashSet<String>();
285 for (int i = 0; i < numberOfWords; i++) {
286 int lengthOfWords = (int) (Math.random() * 5) + 1;
287 char[] wordChar = new char[lengthOfWords];
288 for (int j = 0; j < wordChar.length; j++) {
289 wordChar[j] = (char) (Math.random() * 26 + 97);
290 }
291 String word;
292 if (suffix == null) {
293 word = new String(wordChar);
294 } else {
295 word = new String(wordChar) + suffix;
296 }
297 wordSet.add(word);
298 }
299 List<String> wordList = new ArrayList<String>(wordSet);
300 return wordList;
301 }
302
303 }
304