View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver;
21  
22  import static org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode.INCLUDE;
23  import static org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode.SKIP;
24  
25  import java.io.IOException;
26  import java.util.ArrayList;
27  import java.util.List;
28  import java.util.NavigableSet;
29  
30  import org.apache.hadoop.hbase.HBaseTestCase;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.KeyValue;
33  import org.apache.hadoop.hbase.KeyValue.KVComparator;
34  import org.apache.hadoop.hbase.KeyValue.Type;
35  import org.apache.hadoop.hbase.SmallTests;
36  import org.apache.hadoop.hbase.client.Get;
37  import org.apache.hadoop.hbase.client.Scan;
38  import org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode;
39  import org.apache.hadoop.hbase.util.Bytes;
40  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
41  import org.junit.experimental.categories.Category;
42  
43  @Category(SmallTests.class)
44  public class TestQueryMatcher extends HBaseTestCase {
45    private static final boolean PRINT = false;
46  
47    private byte[] row1;
48    private byte[] row2;
49    private byte[] row3;
50    private byte[] fam1;
51    private byte[] fam2;
52    private byte[] col1;
53    private byte[] col2;
54    private byte[] col3;
55    private byte[] col4;
56    private byte[] col5;
57  
58    private byte[] data;
59  
60    private Get get;
61  
62    long ttl = Long.MAX_VALUE;
63    KVComparator rowComparator;
64    private Scan scan;
65  
66    public void setUp() throws Exception {
67      super.setUp();
68      row1 = Bytes.toBytes("row1");
69      row2 = Bytes.toBytes("row2");
70      row3 = Bytes.toBytes("row3");
71      fam1 = Bytes.toBytes("fam1");
72      fam2 = Bytes.toBytes("fam2");
73      col1 = Bytes.toBytes("col1");
74      col2 = Bytes.toBytes("col2");
75      col3 = Bytes.toBytes("col3");
76      col4 = Bytes.toBytes("col4");
77      col5 = Bytes.toBytes("col5");
78  
79      data = Bytes.toBytes("data");
80  
81      //Create Get
82      get = new Get(row1);
83      get.addFamily(fam1);
84      get.addColumn(fam2, col2);
85      get.addColumn(fam2, col4);
86      get.addColumn(fam2, col5);
87      this.scan = new Scan(get);
88  
89      rowComparator = KeyValue.COMPARATOR;
90  
91    }
92  
93      private void _testMatch_ExplicitColumns(Scan scan, List<MatchCode> expected) throws IOException {
94      // 2,4,5    
95      ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2,
96          0, 1, ttl, false, 0, rowComparator), get.getFamilyMap().get(fam2),
97          EnvironmentEdgeManager.currentTimeMillis() - ttl);
98  
99      List<KeyValue> memstore = new ArrayList<KeyValue>();
100     memstore.add(new KeyValue(row1, fam2, col1, 1, data));
101     memstore.add(new KeyValue(row1, fam2, col2, 1, data));
102     memstore.add(new KeyValue(row1, fam2, col3, 1, data));
103     memstore.add(new KeyValue(row1, fam2, col4, 1, data));
104     memstore.add(new KeyValue(row1, fam2, col5, 1, data));
105 
106     memstore.add(new KeyValue(row2, fam1, col1, data));
107 
108     List<ScanQueryMatcher.MatchCode> actual = new ArrayList<ScanQueryMatcher.MatchCode>();
109     KeyValue k = memstore.get(0);
110     qm.setRow(k.getBuffer(), k.getRowOffset(), k.getRowLength());
111 
112     for (KeyValue kv : memstore){
113       actual.add(qm.match(kv));
114     }
115 
116     assertEquals(expected.size(), actual.size());
117     for(int i=0; i< expected.size(); i++){
118       assertEquals(expected.get(i), actual.get(i));
119       if(PRINT){
120         System.out.println("expected "+expected.get(i)+
121             ", actual " +actual.get(i));
122       }
123     }
124   }
125 
126   public void testMatch_ExplicitColumns()
127   throws IOException {
128     //Moving up from the Tracker by using Gets and List<KeyValue> instead
129     //of just byte []
130 
131     //Expected result
132     List<MatchCode> expected = new ArrayList<ScanQueryMatcher.MatchCode>();
133     expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);
134     expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
135     expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);
136     expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
137     expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW);
138     expected.add(ScanQueryMatcher.MatchCode.DONE);
139 
140     _testMatch_ExplicitColumns(scan, expected);
141   }
142 
143   public void testMatch_ExplicitColumnsWithLookAhead()
144   throws IOException {
145     //Moving up from the Tracker by using Gets and List<KeyValue> instead
146     //of just byte []
147 
148     //Expected result
149     List<MatchCode> expected = new ArrayList<ScanQueryMatcher.MatchCode>();
150     expected.add(ScanQueryMatcher.MatchCode.SKIP);
151     expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
152     expected.add(ScanQueryMatcher.MatchCode.SKIP);
153     expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
154     expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW);
155     expected.add(ScanQueryMatcher.MatchCode.DONE);
156 
157     Scan s = new Scan(scan);
158     s.setAttribute(Scan.HINT_LOOKAHEAD, Bytes.toBytes(2));
159     _testMatch_ExplicitColumns(s, expected);
160   }
161 
162 
163   public void testMatch_Wildcard()
164   throws IOException {
165     //Moving up from the Tracker by using Gets and List<KeyValue> instead
166     //of just byte []
167 
168     //Expected result
169     List<MatchCode> expected = new ArrayList<ScanQueryMatcher.MatchCode>();
170     expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
171     expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
172     expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
173     expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
174     expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
175     expected.add(ScanQueryMatcher.MatchCode.DONE);
176 
177     ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2,
178         0, 1, ttl, false, 0, rowComparator), null,
179         EnvironmentEdgeManager.currentTimeMillis() - ttl);
180 
181     List<KeyValue> memstore = new ArrayList<KeyValue>();
182     memstore.add(new KeyValue(row1, fam2, col1, 1, data));
183     memstore.add(new KeyValue(row1, fam2, col2, 1, data));
184     memstore.add(new KeyValue(row1, fam2, col3, 1, data));
185     memstore.add(new KeyValue(row1, fam2, col4, 1, data));
186     memstore.add(new KeyValue(row1, fam2, col5, 1, data));
187     memstore.add(new KeyValue(row2, fam1, col1, 1, data));
188 
189     List<ScanQueryMatcher.MatchCode> actual = new ArrayList<ScanQueryMatcher.MatchCode>();
190 
191     KeyValue k = memstore.get(0);
192     qm.setRow(k.getBuffer(), k.getRowOffset(), k.getRowLength());
193 
194     for(KeyValue kv : memstore) {
195       actual.add(qm.match(kv));
196     }
197 
198     assertEquals(expected.size(), actual.size());
199     for(int i=0; i< expected.size(); i++){
200       assertEquals(expected.get(i), actual.get(i));
201       if(PRINT){
202         System.out.println("expected "+expected.get(i)+
203             ", actual " +actual.get(i));
204       }
205     }
206   }
207 
208 
209   /**
210    * Verify that {@link ScanQueryMatcher} only skips expired KeyValue
211    * instances and does not exit early from the row (skipping
212    * later non-expired KeyValues).  This version mimics a Get with
213    * explicitly specified column qualifiers.
214    *
215    * @throws IOException
216    */
217   public void testMatch_ExpiredExplicit()
218   throws IOException {
219 
220     long testTTL = 1000;
221     MatchCode [] expected = new MatchCode[] {
222         ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
223         ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL,
224         ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
225         ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL,
226         ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW,
227         ScanQueryMatcher.MatchCode.DONE
228     };
229 
230     long now = EnvironmentEdgeManager.currentTimeMillis();
231     ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2,
232         0, 1, testTTL, false, 0, rowComparator), get.getFamilyMap().get(fam2),
233         now - testTTL);
234 
235     KeyValue [] kvs = new KeyValue[] {
236         new KeyValue(row1, fam2, col1, now-100, data),
237         new KeyValue(row1, fam2, col2, now-50, data),
238         new KeyValue(row1, fam2, col3, now-5000, data),
239         new KeyValue(row1, fam2, col4, now-500, data),
240         new KeyValue(row1, fam2, col5, now-10000, data),
241         new KeyValue(row2, fam1, col1, now-10, data)
242     };
243 
244     KeyValue k = kvs[0];
245     qm.setRow(k.getBuffer(), k.getRowOffset(), k.getRowLength());
246 
247     List<MatchCode> actual = new ArrayList<MatchCode>(kvs.length);
248     for (KeyValue kv : kvs) {
249       actual.add( qm.match(kv) );
250     }
251 
252     assertEquals(expected.length, actual.size());
253     for (int i=0; i<expected.length; i++) {
254       if(PRINT){
255         System.out.println("expected "+expected[i]+
256             ", actual " +actual.get(i));
257       }
258       assertEquals(expected[i], actual.get(i));
259     }
260   }
261 
262 
263   /**
264    * Verify that {@link ScanQueryMatcher} only skips expired KeyValue
265    * instances and does not exit early from the row (skipping
266    * later non-expired KeyValues).  This version mimics a Get with
267    * wildcard-inferred column qualifiers.
268    *
269    * @throws IOException
270    */
271   public void testMatch_ExpiredWildcard()
272   throws IOException {
273 
274     long testTTL = 1000;
275     MatchCode [] expected = new MatchCode[] {
276         ScanQueryMatcher.MatchCode.INCLUDE,
277         ScanQueryMatcher.MatchCode.INCLUDE,
278         ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
279         ScanQueryMatcher.MatchCode.INCLUDE,
280         ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
281         ScanQueryMatcher.MatchCode.DONE
282     };
283 
284     long now = EnvironmentEdgeManager.currentTimeMillis();
285     ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2,
286         0, 1, testTTL, false, 0, rowComparator), null,
287         now - testTTL);
288 
289     KeyValue [] kvs = new KeyValue[] {
290         new KeyValue(row1, fam2, col1, now-100, data),
291         new KeyValue(row1, fam2, col2, now-50, data),
292         new KeyValue(row1, fam2, col3, now-5000, data),
293         new KeyValue(row1, fam2, col4, now-500, data),
294         new KeyValue(row1, fam2, col5, now-10000, data),
295         new KeyValue(row2, fam1, col1, now-10, data)
296     };
297     KeyValue k = kvs[0];
298     qm.setRow(k.getBuffer(), k.getRowOffset(), k.getRowLength());
299 
300     List<ScanQueryMatcher.MatchCode> actual =
301         new ArrayList<ScanQueryMatcher.MatchCode>(kvs.length);
302     for (KeyValue kv : kvs) {
303       actual.add( qm.match(kv) );
304     }
305 
306     assertEquals(expected.length, actual.size());
307     for (int i=0; i<expected.length; i++) {
308       if(PRINT){
309         System.out.println("expected "+expected[i]+
310             ", actual " +actual.get(i));
311       }
312       assertEquals(expected[i], actual.get(i));
313     }
314   }
315 
316   public void testMatch_PartialRangeDropDeletes() throws Exception {
317     // Some ranges.
318     testDropDeletes(
319         row2, row3, new byte[][] { row1, row2, row2, row3 }, INCLUDE, SKIP, SKIP, INCLUDE);
320     testDropDeletes(row2, row3, new byte[][] { row1, row1, row2 }, INCLUDE, INCLUDE, SKIP);
321     testDropDeletes(row2, row3, new byte[][] { row2, row3, row3 }, SKIP, INCLUDE, INCLUDE);
322     testDropDeletes(row1, row3, new byte[][] { row1, row2, row3 }, SKIP, SKIP, INCLUDE);
323     // Open ranges.
324     testDropDeletes(HConstants.EMPTY_START_ROW, row3,
325         new byte[][] { row1, row2, row3 }, SKIP, SKIP, INCLUDE);
326     testDropDeletes(row2, HConstants.EMPTY_END_ROW,
327         new byte[][] { row1, row2, row3 }, INCLUDE, SKIP, SKIP);
328     testDropDeletes(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
329         new byte[][] { row1, row2, row3, row3 }, SKIP, SKIP, SKIP, SKIP);
330 
331     // No KVs in range.
332     testDropDeletes(row2, row3, new byte[][] { row1, row1, row3 }, INCLUDE, INCLUDE, INCLUDE);
333     testDropDeletes(row2, row3, new byte[][] { row3, row3 }, INCLUDE, INCLUDE);
334     testDropDeletes(row2, row3, new byte[][] { row1, row1 }, INCLUDE, INCLUDE);
335   }
336 
337   private void testDropDeletes(
338       byte[] from, byte[] to, byte[][] rows, MatchCode... expected) throws IOException {
339     long now = EnvironmentEdgeManager.currentTimeMillis();
340     // Set time to purge deletes to negative value to avoid it ever happening.
341     ScanInfo scanInfo = new ScanInfo(fam2, 0, 1, ttl, false, -1L, rowComparator);
342     NavigableSet<byte[]> cols = get.getFamilyMap().get(fam2);
343 
344     ScanQueryMatcher qm = new ScanQueryMatcher(scan, scanInfo, cols, Long.MAX_VALUE,
345         HConstants.OLDEST_TIMESTAMP, HConstants.OLDEST_TIMESTAMP, from, to, null);
346     List<ScanQueryMatcher.MatchCode> actual =
347         new ArrayList<ScanQueryMatcher.MatchCode>(rows.length);
348     byte[] prevRow = null;
349     for (byte[] row : rows) {
350       if (prevRow == null || !Bytes.equals(prevRow, row)) {
351         qm.setRow(row, 0, (short)row.length);
352         prevRow = row;
353       }
354       actual.add(qm.match(new KeyValue(row, fam2, null, now, Type.Delete)));
355     }
356 
357     assertEquals(expected.length, actual.size());
358     for (int i = 0; i < expected.length; i++) {
359       if (PRINT) System.out.println("expected " + expected[i] + ", actual " + actual.get(i));
360       assertEquals(expected[i], actual.get(i));
361     }
362   }
363 }
364