1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.filter;
21
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.Arrays;
25 import java.util.List;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.hbase.*;
30 import org.apache.hadoop.hbase.client.Put;
31 import org.apache.hadoop.hbase.client.Scan;
32 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
33 import org.apache.hadoop.hbase.filter.Filter.ReturnCode;
34 import org.apache.hadoop.hbase.regionserver.HRegion;
35 import org.apache.hadoop.hbase.regionserver.InternalScanner;
36 import org.apache.hadoop.hbase.util.Bytes;
37
38 import org.junit.After;
39 import org.junit.Before;
40 import org.junit.Test;
41 import static org.junit.Assert.assertEquals;
42 import static org.junit.Assert.assertNotNull;
43 import static org.junit.Assert.assertTrue;
44
45 import org.junit.experimental.categories.Category;
46
47 @Category(SmallTests.class)
48 public class TestDependentColumnFilter {
49 private final Log LOG = LogFactory.getLog(this.getClass());
50 private static final byte[][] ROWS = {
51 Bytes.toBytes("test1"),Bytes.toBytes("test2")
52 };
53 private static final byte[][] FAMILIES = {
54 Bytes.toBytes("familyOne"),Bytes.toBytes("familyTwo")
55 };
56 private static final long STAMP_BASE = System.currentTimeMillis();
57 private static final long[] STAMPS = {
58 STAMP_BASE-100, STAMP_BASE-200, STAMP_BASE-300
59 };
60 private static final byte[] QUALIFIER = Bytes.toBytes("qualifier");
61 private static final byte[][] BAD_VALS = {
62 Bytes.toBytes("bad1"), Bytes.toBytes("bad2"), Bytes.toBytes("bad3")
63 };
64 private static final byte[] MATCH_VAL = Bytes.toBytes("match");
65 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
66
67 List<KeyValue> testVals;
68 private HRegion region;
69
70 @Before
71 public void setUp() throws Exception {
72 testVals = makeTestVals();
73
74 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(this.getClass().getSimpleName()));
75 HColumnDescriptor hcd0 = new HColumnDescriptor(FAMILIES[0]);
76 hcd0.setMaxVersions(3);
77 htd.addFamily(hcd0);
78 HColumnDescriptor hcd1 = new HColumnDescriptor(FAMILIES[1]);
79 hcd1.setMaxVersions(3);
80 htd.addFamily(hcd1);
81 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
82 this.region = HRegion.createHRegion(info, TEST_UTIL.getDataTestDir(),
83 TEST_UTIL.getConfiguration(), htd);
84 addData();
85 }
86
87 @After
88 public void tearDown() throws Exception {
89 HRegion.closeHRegion(this.region);
90 }
91
92 private void addData() throws IOException {
93 Put put = new Put(ROWS[0]);
94
95 put.add(FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0]);
96 put.add(FAMILIES[0], QUALIFIER, STAMPS[1], BAD_VALS[1]);
97 put.add(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL);
98
99
100
101
102 put.add(FAMILIES[1], QUALIFIER, STAMPS[0], BAD_VALS[0]);
103 put.add(FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]);
104
105 this.region.put(put);
106
107 put = new Put(ROWS[1]);
108 put.add(FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0]);
109
110 put.add(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL);
111
112 put.add(FAMILIES[1], QUALIFIER, STAMPS[0], MATCH_VAL);
113
114 put.add(FAMILIES[1], QUALIFIER, STAMPS[1], BAD_VALS[2]);
115
116 this.region.put(put);
117 }
118
119 private List<KeyValue> makeTestVals() {
120 List<KeyValue> testVals = new ArrayList<KeyValue>();
121 testVals.add(new KeyValue(ROWS[0], FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0]));
122 testVals.add(new KeyValue(ROWS[0], FAMILIES[0], QUALIFIER, STAMPS[1], BAD_VALS[1]));
123 testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[1], BAD_VALS[2]));
124 testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[0], MATCH_VAL));
125 testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]));
126
127 return testVals;
128 }
129
130
131
132
133
134
135
136
137
138
139 private void verifyScan(Scan s, long expectedRows, long expectedCells)
140 throws IOException {
141 InternalScanner scanner = this.region.getScanner(s);
142 List<Cell> results = new ArrayList<Cell>();
143 int i = 0;
144 int cells = 0;
145 for (boolean done = true; done; i++) {
146 done = scanner.next(results);
147 Arrays.sort(results.toArray(new KeyValue[results.size()]),
148 KeyValue.COMPARATOR);
149 LOG.info("counter=" + i + ", " + results);
150 if (results.isEmpty()) break;
151 cells += results.size();
152 assertTrue("Scanned too many rows! Only expected " + expectedRows +
153 " total but already scanned " + (i+1), expectedRows > i);
154 assertTrue("Expected " + expectedCells + " cells total but " +
155 "already scanned " + cells, expectedCells >= cells);
156 results.clear();
157 }
158 assertEquals("Expected " + expectedRows + " rows but scanned " + i +
159 " rows", expectedRows, i);
160 assertEquals("Expected " + expectedCells + " cells but scanned " + cells +
161 " cells", expectedCells, cells);
162 }
163
164
165
166
167 @Test
168 public void testScans() throws Exception {
169 Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER);
170
171 Scan scan = new Scan();
172 scan.setFilter(filter);
173 scan.setMaxVersions(Integer.MAX_VALUE);
174
175 verifyScan(scan, 2, 8);
176
177
178 filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true);
179 scan = new Scan();
180 scan.setFilter(filter);
181 scan.setMaxVersions(Integer.MAX_VALUE);
182
183 verifyScan(scan, 2, 3);
184
185
186 filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, false,
187 CompareOp.EQUAL, new BinaryComparator(MATCH_VAL));
188 scan = new Scan();
189 scan.setFilter(filter);
190 scan.setMaxVersions(Integer.MAX_VALUE);
191
192
193
194
195
196
197
198
199
200 verifyScan(scan, 2, 3);
201
202
203 filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true,
204 CompareOp.EQUAL, new BinaryComparator(MATCH_VAL));
205 scan = new Scan();
206 scan.setFilter(filter);
207 scan.setMaxVersions(Integer.MAX_VALUE);
208
209
210
211
212
213
214 verifyScan(scan, 1, 1);
215
216 }
217
218
219
220
221
222
223 @Test
224 public void testFilterDropping() throws Exception {
225 Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER);
226 List<Cell> accepted = new ArrayList<Cell>();
227 for(Cell val : testVals) {
228 if(filter.filterKeyValue(val) == ReturnCode.INCLUDE) {
229 accepted.add(val);
230 }
231 }
232 assertEquals("check all values accepted from filterKeyValue", 5, accepted.size());
233
234 filter.filterRowCells(accepted);
235 assertEquals("check filterRow(List<KeyValue>) dropped cell without corresponding column entry", 4, accepted.size());
236
237
238 filter = new DependentColumnFilter(FAMILIES[1], QUALIFIER, true);
239 accepted.clear();
240 for(KeyValue val : testVals) {
241 if(filter.filterKeyValue(val) == ReturnCode.INCLUDE) {
242 accepted.add(val);
243 }
244 }
245 assertEquals("check the filtering column cells got dropped", 2, accepted.size());
246
247 filter.filterRowCells(accepted);
248 assertEquals("check cell retention", 2, accepted.size());
249 }
250
251
252
253
254 @Test
255 public void testToStringWithNullComparator() {
256
257 Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER);
258 assertNotNull(filter.toString());
259 assertTrue("check string contains 'null' as compatator is null",
260 filter.toString().contains("null"));
261
262
263 filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true, CompareOp.EQUAL, null);
264 assertNotNull(filter.toString());
265 assertTrue("check string contains 'null' as compatator is null",
266 filter.toString().contains("null"));
267 }
268
269 @Test
270 public void testToStringWithNonNullComparator() {
271 Filter filter =
272 new DependentColumnFilter(FAMILIES[0], QUALIFIER, true, CompareOp.EQUAL,
273 new BinaryComparator(MATCH_VAL));
274 assertNotNull(filter.toString());
275 assertTrue("check string contains comparator value", filter.toString().contains("match"));
276 }
277
278 }
279