View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.filter;
21  
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.Arrays;
25  import java.util.List;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.hbase.*;
30  import org.apache.hadoop.hbase.client.Put;
31  import org.apache.hadoop.hbase.client.Scan;
32  import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
33  import org.apache.hadoop.hbase.filter.Filter.ReturnCode;
34  import org.apache.hadoop.hbase.regionserver.HRegion;
35  import org.apache.hadoop.hbase.regionserver.InternalScanner;
36  import org.apache.hadoop.hbase.util.Bytes;
37  
38  import org.junit.After;
39  import org.junit.Before;
40  import org.junit.Test;
41  import static org.junit.Assert.assertEquals;
42  import static org.junit.Assert.assertNotNull;
43  import static org.junit.Assert.assertTrue;
44  
45  import org.junit.experimental.categories.Category;
46  
47  @Category(SmallTests.class)
48  public class TestDependentColumnFilter {
49    private final Log LOG = LogFactory.getLog(this.getClass());
50    private static final byte[][] ROWS = {
51  	  Bytes.toBytes("test1"),Bytes.toBytes("test2")
52    };
53    private static final byte[][] FAMILIES = {
54  	  Bytes.toBytes("familyOne"),Bytes.toBytes("familyTwo")
55    };
56    private static final long STAMP_BASE = System.currentTimeMillis();
57    private static final long[] STAMPS = {
58  	  STAMP_BASE-100, STAMP_BASE-200, STAMP_BASE-300
59    };
60    private static final byte[] QUALIFIER = Bytes.toBytes("qualifier");
61    private static final byte[][] BAD_VALS = {
62  	  Bytes.toBytes("bad1"), Bytes.toBytes("bad2"), Bytes.toBytes("bad3")
63    };
64    private static final byte[] MATCH_VAL = Bytes.toBytes("match");
65    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
66  
67    List<KeyValue> testVals;
68    private HRegion region;
69  
70    @Before
71    public void setUp() throws Exception {
72      testVals = makeTestVals();
73  
74      HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(this.getClass().getSimpleName()));
75      HColumnDescriptor hcd0 = new HColumnDescriptor(FAMILIES[0]);
76      hcd0.setMaxVersions(3);
77      htd.addFamily(hcd0);
78      HColumnDescriptor hcd1 = new HColumnDescriptor(FAMILIES[1]);
79      hcd1.setMaxVersions(3);
80      htd.addFamily(hcd1);
81      HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
82      this.region = HRegion.createHRegion(info, TEST_UTIL.getDataTestDir(),
83        TEST_UTIL.getConfiguration(), htd);
84      addData();
85    }
86  
87    @After
88    public void tearDown() throws Exception {
89      HRegion.closeHRegion(this.region);
90    }
91  
92    private void addData() throws IOException {
93      Put put = new Put(ROWS[0]);
94      // add in an entry for each stamp, with 2 as a "good" value
95      put.add(FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0]);
96      put.add(FAMILIES[0], QUALIFIER, STAMPS[1], BAD_VALS[1]);
97      put.add(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL);
98      // add in entries for stamps 0 and 2.
99      // without a value check both will be "accepted"
100     // with one 2 will be accepted(since the corresponding ts entry
101     // has a matching value
102     put.add(FAMILIES[1], QUALIFIER, STAMPS[0], BAD_VALS[0]);
103     put.add(FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]);
104 
105     this.region.put(put);
106 
107     put = new Put(ROWS[1]);
108     put.add(FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0]);
109     // there is no corresponding timestamp for this so it should never pass
110     put.add(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL);
111     // if we reverse the qualifiers this one should pass
112     put.add(FAMILIES[1], QUALIFIER, STAMPS[0], MATCH_VAL);
113     // should pass
114     put.add(FAMILIES[1], QUALIFIER, STAMPS[1], BAD_VALS[2]);
115 
116     this.region.put(put);
117   }
118 
119   private List<KeyValue> makeTestVals() {
120 	List<KeyValue> testVals = new ArrayList<KeyValue>();
121 	testVals.add(new KeyValue(ROWS[0], FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0]));
122 	testVals.add(new KeyValue(ROWS[0], FAMILIES[0], QUALIFIER, STAMPS[1], BAD_VALS[1]));
123 	testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[1], BAD_VALS[2]));
124 	testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[0], MATCH_VAL));
125 	testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]));
126 
127 	return testVals;
128   }
129 
130   /**
131    * This shouldn't be confused with TestFilter#verifyScan
132    * as expectedKeys is not the per row total, but the scan total
133    *
134    * @param s
135    * @param expectedRows
136    * @param expectedCells
137    * @throws IOException
138    */
139   private void verifyScan(Scan s, long expectedRows, long expectedCells)
140   throws IOException {
141     InternalScanner scanner = this.region.getScanner(s);
142     List<Cell> results = new ArrayList<Cell>();
143     int i = 0;
144     int cells = 0;
145     for (boolean done = true; done; i++) {
146       done = scanner.next(results);
147       Arrays.sort(results.toArray(new KeyValue[results.size()]),
148           KeyValue.COMPARATOR);
149       LOG.info("counter=" + i + ", " + results);
150       if (results.isEmpty()) break;
151       cells += results.size();
152       assertTrue("Scanned too many rows! Only expected " + expectedRows +
153           " total but already scanned " + (i+1), expectedRows > i);
154       assertTrue("Expected " + expectedCells + " cells total but " +
155           "already scanned " + cells, expectedCells >= cells);
156       results.clear();
157     }
158     assertEquals("Expected " + expectedRows + " rows but scanned " + i +
159         " rows", expectedRows, i);
160     assertEquals("Expected " + expectedCells + " cells but scanned " + cells +
161             " cells", expectedCells, cells);
162   }
163 
164   /**
165    * Test scans using a DependentColumnFilter
166    */
167   @Test
168   public void testScans() throws Exception {
169     Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER);
170 
171     Scan scan = new Scan();
172     scan.setFilter(filter);
173     scan.setMaxVersions(Integer.MAX_VALUE);
174 
175     verifyScan(scan, 2, 8);
176 
177     // drop the filtering cells
178     filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true);
179     scan = new Scan();
180     scan.setFilter(filter);
181     scan.setMaxVersions(Integer.MAX_VALUE);
182 
183     verifyScan(scan, 2, 3);
184 
185     // include a comparator operation
186     filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, false,
187         CompareOp.EQUAL, new BinaryComparator(MATCH_VAL));
188     scan = new Scan();
189     scan.setFilter(filter);
190     scan.setMaxVersions(Integer.MAX_VALUE);
191 
192     /*
193      * expecting to get the following 3 cells
194      * row 0
195      *   put.add(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL);
196      *   put.add(FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]);
197      * row 1
198      *   put.add(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL);
199      */
200     verifyScan(scan, 2, 3);
201 
202     // include a comparator operation and drop comparator
203     filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true,
204         CompareOp.EQUAL, new BinaryComparator(MATCH_VAL));
205     scan = new Scan();
206     scan.setFilter(filter);
207     scan.setMaxVersions(Integer.MAX_VALUE);
208 
209     /*
210      * expecting to get the following 1 cell
211      * row 0
212      *   put.add(FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]);
213      */
214     verifyScan(scan, 1, 1);
215 
216   }
217 
218   /**
219    * Test that the filter correctly drops rows without a corresponding timestamp
220    *
221    * @throws Exception
222    */
223   @Test
224   public void testFilterDropping() throws Exception {
225     Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER);
226     List<Cell> accepted = new ArrayList<Cell>();
227     for(Cell val : testVals) {
228       if(filter.filterKeyValue(val) == ReturnCode.INCLUDE) {
229         accepted.add(val);
230       }
231     }
232     assertEquals("check all values accepted from filterKeyValue", 5, accepted.size());
233 
234     filter.filterRowCells(accepted);
235     assertEquals("check filterRow(List<KeyValue>) dropped cell without corresponding column entry", 4, accepted.size());
236 
237     // start do it again with dependent column dropping on
238     filter = new DependentColumnFilter(FAMILIES[1], QUALIFIER, true);
239     accepted.clear();
240     for(KeyValue val : testVals) {
241         if(filter.filterKeyValue(val) == ReturnCode.INCLUDE) {
242           accepted.add(val);
243         }
244       }
245       assertEquals("check the filtering column cells got dropped", 2, accepted.size());
246 
247       filter.filterRowCells(accepted);
248       assertEquals("check cell retention", 2, accepted.size());
249   }
250 
251   /**
252    * Test for HBASE-8794. Avoid NullPointerException in DependentColumnFilter.toString().
253    */
254   @Test
255   public void testToStringWithNullComparator() {
256     // Test constructor that implicitly sets a null comparator
257     Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER);
258     assertNotNull(filter.toString());
259     assertTrue("check string contains 'null' as compatator is null",
260       filter.toString().contains("null"));
261 
262     // Test constructor with explicit null comparator
263     filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true, CompareOp.EQUAL, null);
264     assertNotNull(filter.toString());
265     assertTrue("check string contains 'null' as compatator is null",
266       filter.toString().contains("null"));
267   }
268 
269   @Test
270   public void testToStringWithNonNullComparator() {
271     Filter filter =
272         new DependentColumnFilter(FAMILIES[0], QUALIFIER, true, CompareOp.EQUAL,
273             new BinaryComparator(MATCH_VAL));
274     assertNotNull(filter.toString());
275     assertTrue("check string contains comparator value", filter.toString().contains("match"));
276   }
277 
278 }
279