1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import static org.junit.Assert.*;
23
24 import java.io.IOException;
25 import java.util.ArrayList;
26 import java.util.Collection;
27 import java.util.HashMap;
28 import java.util.HashSet;
29 import java.util.List;
30 import java.util.Set;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.hbase.*;
35 import org.apache.hadoop.hbase.client.Put;
36 import org.apache.hadoop.hbase.client.Scan;
37 import org.apache.hadoop.hbase.client.Durability;
38 import org.apache.hadoop.hbase.util.Bytes;
39 import org.junit.Rule;
40 import org.junit.Test;
41 import org.junit.experimental.categories.Category;
42 import org.junit.rules.TestName;
43
44 @Category(SmallTests.class)
45 public class TestColumnSeeking {
46 @Rule public TestName name = new TestName();
47
48 private final static HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
49
50 static final Log LOG = LogFactory.getLog(TestColumnSeeking.class);
51
52 @SuppressWarnings("unchecked")
53 @Test
54 public void testDuplicateVersions() throws IOException {
55 String family = "Family";
56 byte[] familyBytes = Bytes.toBytes("Family");
57 TableName table = TableName.valueOf(name.getMethodName());
58
59 HColumnDescriptor hcd =
60 new HColumnDescriptor(familyBytes).setMaxVersions(1000);
61 hcd.setMaxVersions(3);
62 HTableDescriptor htd = new HTableDescriptor(table);
63 htd.addFamily(hcd);
64 HRegionInfo info = new HRegionInfo(table, null, null, false);
65
66 HRegion region = TEST_UTIL.createLocalHRegion(info, htd);
67 try {
68 List<String> rows = generateRandomWords(10, "row");
69 List<String> allColumns = generateRandomWords(10, "column");
70 List<String> values = generateRandomWords(100, "value");
71
72 long maxTimestamp = 2;
73 double selectPercent = 0.5;
74 int numberOfTests = 5;
75 double flushPercentage = 0.2;
76 double minorPercentage = 0.2;
77 double majorPercentage = 0.2;
78 double putPercentage = 0.2;
79
80 HashMap<String, KeyValue> allKVMap = new HashMap<String, KeyValue>();
81
82 HashMap<String, KeyValue>[] kvMaps = new HashMap[numberOfTests];
83 ArrayList<String>[] columnLists = new ArrayList[numberOfTests];
84
85 for (int i = 0; i < numberOfTests; i++) {
86 kvMaps[i] = new HashMap<String, KeyValue>();
87 columnLists[i] = new ArrayList<String>();
88 for (String column : allColumns) {
89 if (Math.random() < selectPercent) {
90 columnLists[i].add(column);
91 }
92 }
93 }
94
95 for (String value : values) {
96 for (String row : rows) {
97 Put p = new Put(Bytes.toBytes(row));
98 p.setDurability(Durability.SKIP_WAL);
99 for (String column : allColumns) {
100 for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
101 KeyValue kv =
102 KeyValueTestUtil.create(row, family, column, timestamp, value);
103 if (Math.random() < putPercentage) {
104 p.add(kv);
105 allKVMap.put(kv.getKeyString(), kv);
106 for (int i = 0; i < numberOfTests; i++) {
107 if (columnLists[i].contains(column)) {
108 kvMaps[i].put(kv.getKeyString(), kv);
109 }
110 }
111 }
112 }
113 }
114 region.put(p);
115 if (Math.random() < flushPercentage) {
116 LOG.info("Flushing... ");
117 region.flushcache();
118 }
119
120 if (Math.random() < minorPercentage) {
121 LOG.info("Minor compacting... ");
122 region.compactStores(false);
123 }
124
125 if (Math.random() < majorPercentage) {
126 LOG.info("Major compacting... ");
127 region.compactStores(true);
128 }
129 }
130 }
131
132 for (int i = 0; i < numberOfTests + 1; i++) {
133 Collection<KeyValue> kvSet;
134 Scan scan = new Scan();
135 scan.setMaxVersions();
136 if (i < numberOfTests) {
137 if (columnLists[i].size() == 0) continue;
138 kvSet = kvMaps[i].values();
139 for (String column : columnLists[i]) {
140 scan.addColumn(familyBytes, Bytes.toBytes(column));
141 }
142 LOG.info("ExplicitColumns scanner");
143 LOG.info("Columns: " + columnLists[i].size() + " Keys: "
144 + kvSet.size());
145 } else {
146 kvSet = allKVMap.values();
147 LOG.info("Wildcard scanner");
148 LOG.info("Columns: " + allColumns.size() + " Keys: " + kvSet.size());
149
150 }
151 InternalScanner scanner = region.getScanner(scan);
152 List<Cell> results = new ArrayList<Cell>();
153 while (scanner.next(results))
154 ;
155 assertEquals(kvSet.size(), results.size());
156 assertTrue(KeyValueTestUtil.containsIgnoreMvccVersion(results, kvSet));
157 }
158 } finally {
159 HRegion.closeHRegion(region);
160 }
161
162 HRegion.closeHRegion(region);
163 }
164
165 @SuppressWarnings("unchecked")
166 @Test
167 public void testReseeking() throws IOException {
168 String family = "Family";
169 byte[] familyBytes = Bytes.toBytes("Family");
170 TableName table = TableName.valueOf(name.getMethodName());
171
172 HTableDescriptor htd = new HTableDescriptor(table);
173 HColumnDescriptor hcd = new HColumnDescriptor(family);
174 hcd.setMaxVersions(3);
175 htd.addFamily(hcd);
176
177 HRegionInfo info = new HRegionInfo(table, null, null, false);
178 HRegion region = TEST_UTIL.createLocalHRegion(info, htd);
179
180 List<String> rows = generateRandomWords(10, "row");
181 List<String> allColumns = generateRandomWords(100, "column");
182
183 long maxTimestamp = 2;
184 double selectPercent = 0.5;
185 int numberOfTests = 5;
186 double flushPercentage = 0.2;
187 double minorPercentage = 0.2;
188 double majorPercentage = 0.2;
189 double putPercentage = 0.2;
190
191 HashMap<String, KeyValue> allKVMap = new HashMap<String, KeyValue>();
192
193 HashMap<String, KeyValue>[] kvMaps = new HashMap[numberOfTests];
194 ArrayList<String>[] columnLists = new ArrayList[numberOfTests];
195 String valueString = "Value";
196
197 for (int i = 0; i < numberOfTests; i++) {
198 kvMaps[i] = new HashMap<String, KeyValue>();
199 columnLists[i] = new ArrayList<String>();
200 for (String column : allColumns) {
201 if (Math.random() < selectPercent) {
202 columnLists[i].add(column);
203 }
204 }
205 }
206
207 for (String row : rows) {
208 Put p = new Put(Bytes.toBytes(row));
209 p.setDurability(Durability.SKIP_WAL);
210 for (String column : allColumns) {
211 for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
212 KeyValue kv =
213 KeyValueTestUtil.create(row, family, column, timestamp,
214 valueString);
215 if (Math.random() < putPercentage) {
216 p.add(kv);
217 allKVMap.put(kv.getKeyString(), kv);
218 for (int i = 0; i < numberOfTests; i++) {
219 if (columnLists[i].contains(column)) {
220 kvMaps[i].put(kv.getKeyString(), kv);
221 }
222 }
223 }
224
225 }
226 }
227 region.put(p);
228 if (Math.random() < flushPercentage) {
229 LOG.info("Flushing... ");
230 region.flushcache();
231 }
232
233 if (Math.random() < minorPercentage) {
234 LOG.info("Minor compacting... ");
235 region.compactStores(false);
236 }
237
238 if (Math.random() < majorPercentage) {
239 LOG.info("Major compacting... ");
240 region.compactStores(true);
241 }
242 }
243
244 for (int i = 0; i < numberOfTests + 1; i++) {
245 Collection<KeyValue> kvSet;
246 Scan scan = new Scan();
247 scan.setMaxVersions();
248 if (i < numberOfTests) {
249 if (columnLists[i].size() == 0) continue;
250 kvSet = kvMaps[i].values();
251 for (String column : columnLists[i]) {
252 scan.addColumn(familyBytes, Bytes.toBytes(column));
253 }
254 LOG.info("ExplicitColumns scanner");
255 LOG.info("Columns: " + columnLists[i].size() + " Keys: "
256 + kvSet.size());
257 } else {
258 kvSet = allKVMap.values();
259 LOG.info("Wildcard scanner");
260 LOG.info("Columns: " + allColumns.size() + " Keys: " + kvSet.size());
261
262 }
263 InternalScanner scanner = region.getScanner(scan);
264 List<Cell> results = new ArrayList<Cell>();
265 while (scanner.next(results))
266 ;
267 assertEquals(kvSet.size(), results.size());
268 assertTrue(KeyValueTestUtil.containsIgnoreMvccVersion(results, kvSet));
269 }
270
271 HRegion.closeHRegion(region);
272 }
273
274 List<String> generateRandomWords(int numberOfWords, String suffix) {
275 Set<String> wordSet = new HashSet<String>();
276 for (int i = 0; i < numberOfWords; i++) {
277 int lengthOfWords = (int) (Math.random() * 5) + 1;
278 char[] wordChar = new char[lengthOfWords];
279 for (int j = 0; j < wordChar.length; j++) {
280 wordChar[j] = (char) (Math.random() * 26 + 97);
281 }
282 String word;
283 if (suffix == null) {
284 word = new String(wordChar);
285 } else {
286 word = new String(wordChar) + suffix;
287 }
288 wordSet.add(word);
289 }
290 List<String> wordList = new ArrayList<String>(wordSet);
291 return wordList;
292 }
293
294 }
295