View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS;
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertTrue;
24  
25  import java.util.ArrayList;
26  import java.util.List;
27  
28  import org.apache.hadoop.hbase.Cell;
29  import org.apache.hadoop.hbase.CellUtil;
30  import org.apache.hadoop.hbase.HBaseTestingUtility;
31  import org.apache.hadoop.hbase.HTableDescriptor;
32  import org.apache.hadoop.hbase.KeepDeletedCells;
33  import org.apache.hadoop.hbase.testclassification.SmallTests;
34  import org.apache.hadoop.hbase.client.Delete;
35  import org.apache.hadoop.hbase.client.Get;
36  import org.apache.hadoop.hbase.client.Put;
37  import org.apache.hadoop.hbase.client.Result;
38  import org.apache.hadoop.hbase.filter.TimestampsFilter;
39  import org.apache.hadoop.hbase.util.Bytes;
40  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
41  import org.junit.Rule;
42  import org.junit.Test;
43  import org.junit.experimental.categories.Category;
44  import org.junit.rules.TestName;
45  
46  /**
47   * Test Minimum Versions feature (HBASE-4071).
48   */
49  @Category(SmallTests.class)
50  public class TestMinVersions {
51    HBaseTestingUtility hbu = HBaseTestingUtility.createLocalHTU();
52    private final byte[] T0 = Bytes.toBytes("0");
53    private final byte[] T1 = Bytes.toBytes("1");
54    private final byte[] T2 = Bytes.toBytes("2");
55    private final byte[] T3 = Bytes.toBytes("3");
56    private final byte[] T4 = Bytes.toBytes("4");
57    private final byte[] T5 = Bytes.toBytes("5");
58  
59    private final byte[] c0 = COLUMNS[0];
60  
61    @Rule public TestName name = new TestName();
62  
63    /**
64     * Verify behavior of getClosestBefore(...)
65     */
66    @Test
67    public void testGetClosestBefore() throws Exception {
68      HTableDescriptor htd =
69          hbu.createTableDescriptor(name.getMethodName(), 1, 1000, 1, KeepDeletedCells.FALSE);
70      HRegion region = hbu.createLocalHRegion(htd, null, null);
71      try {
72  
73        // 2s in the past
74        long ts = EnvironmentEdgeManager.currentTime() - 2000;
75  
76        Put p = new Put(T1, ts);
77        p.add(c0, c0, T1);
78        region.put(p);
79  
80        p = new Put(T1, ts+1);
81        p.add(c0, c0, T4);
82        region.put(p);
83  
84        p = new Put(T3, ts);
85        p.add(c0, c0, T3);
86        region.put(p);
87  
88        // now make sure that getClosestBefore(...) get can
89        // rows that would be expired without minVersion.
90        // also make sure it gets the latest version
91        Result r = region.getClosestRowBefore(T1, c0);
92        checkResult(r, c0, T4);
93  
94        r = region.getClosestRowBefore(T2, c0);
95        checkResult(r, c0, T4);
96  
97        // now flush/compact
98        region.flush(true);
99        region.compact(true);
100 
101       r = region.getClosestRowBefore(T1, c0);
102       checkResult(r, c0, T4);
103 
104       r = region.getClosestRowBefore(T2, c0);
105       checkResult(r, c0, T4);
106     } finally {
107       HRegion.closeHRegion(region);
108     }
109   }
110 
111   /**
112    * Test mixed memstore and storefile scanning
113    * with minimum versions.
114    */
115   @Test
116   public void testStoreMemStore() throws Exception {
117     // keep 3 versions minimum
118     HTableDescriptor htd =
119         hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, KeepDeletedCells.FALSE);
120     HRegion region = hbu.createLocalHRegion(htd, null, null);
121     // 2s in the past
122     long ts = EnvironmentEdgeManager.currentTime() - 2000;
123 
124     try {
125       Put p = new Put(T1, ts-1);
126       p.add(c0, c0, T2);
127       region.put(p);
128 
129       p = new Put(T1, ts-3);
130       p.add(c0, c0, T0);
131       region.put(p);
132 
133       // now flush/compact
134       region.flush(true);
135       region.compact(true);
136 
137       p = new Put(T1, ts);
138       p.add(c0, c0, T3);
139       region.put(p);
140 
141       p = new Put(T1, ts-2);
142       p.add(c0, c0, T1);
143       region.put(p);
144 
145       p = new Put(T1, ts-3);
146       p.add(c0, c0, T0);
147       region.put(p);
148 
149       // newest version in the memstore
150       // the 2nd oldest in the store file
151       // and the 3rd, 4th oldest also in the memstore
152 
153       Get g = new Get(T1);
154       g.setMaxVersions();
155       Result r = region.get(g); // this'll use ScanWildcardColumnTracker
156       checkResult(r, c0, T3,T2,T1);
157 
158       g = new Get(T1);
159       g.setMaxVersions();
160       g.addColumn(c0, c0);
161       r = region.get(g);  // this'll use ExplicitColumnTracker
162       checkResult(r, c0, T3,T2,T1);
163     } finally {
164       HRegion.closeHRegion(region);
165     }
166   }
167 
168   /**
169    * Make sure the Deletes behave as expected with minimum versions
170    */
171   @Test
172   public void testDelete() throws Exception {
173     HTableDescriptor htd =
174         hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, KeepDeletedCells.FALSE);
175     HRegion region = hbu.createLocalHRegion(htd, null, null);
176 
177     // 2s in the past
178     long ts = EnvironmentEdgeManager.currentTime() - 2000;
179 
180     try {
181       Put p = new Put(T1, ts-2);
182       p.add(c0, c0, T1);
183       region.put(p);
184 
185       p = new Put(T1, ts-1);
186       p.add(c0, c0, T2);
187       region.put(p);
188 
189       p = new Put(T1, ts);
190       p.add(c0, c0, T3);
191       region.put(p);
192 
193       Delete d = new Delete(T1, ts-1);
194       region.delete(d);
195 
196       Get g = new Get(T1);
197       g.setMaxVersions();
198       Result r = region.get(g);  // this'll use ScanWildcardColumnTracker
199       checkResult(r, c0, T3);
200 
201       g = new Get(T1);
202       g.setMaxVersions();
203       g.addColumn(c0, c0);
204       r = region.get(g);  // this'll use ExplicitColumnTracker
205       checkResult(r, c0, T3);
206 
207       // now flush/compact
208       region.flush(true);
209       region.compact(true);
210 
211       // try again
212       g = new Get(T1);
213       g.setMaxVersions();
214       r = region.get(g);  // this'll use ScanWildcardColumnTracker
215       checkResult(r, c0, T3);
216 
217       g = new Get(T1);
218       g.setMaxVersions();
219       g.addColumn(c0, c0);
220       r = region.get(g);  // this'll use ExplicitColumnTracker
221       checkResult(r, c0, T3);
222     } finally {
223       HRegion.closeHRegion(region);
224     }
225   }
226 
227   /**
228    * Make sure the memstor behaves correctly with minimum versions
229    */
230   @Test
231   public void testMemStore() throws Exception {
232     HTableDescriptor htd =
233         hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE);
234     HRegion region = hbu.createLocalHRegion(htd, null, null);
235 
236     // 2s in the past
237     long ts = EnvironmentEdgeManager.currentTime() - 2000;
238 
239     try {
240       // 2nd version
241       Put p = new Put(T1, ts-2);
242       p.add(c0, c0, T2);
243       region.put(p);
244 
245       // 3rd version
246       p = new Put(T1, ts-1);
247       p.add(c0, c0, T3);
248       region.put(p);
249 
250       // 4th version
251       p = new Put(T1, ts);
252       p.add(c0, c0, T4);
253       region.put(p);
254 
255       // now flush/compact
256       region.flush(true);
257       region.compact(true);
258 
259       // now put the first version (backdated)
260       p = new Put(T1, ts-3);
261       p.add(c0, c0, T1);
262       region.put(p);
263 
264       // now the latest change is in the memstore,
265       // but it is not the latest version
266 
267       Result r = region.get(new Get(T1));
268       checkResult(r, c0, T4);
269 
270       Get g = new Get(T1);
271       g.setMaxVersions();
272       r = region.get(g); // this'll use ScanWildcardColumnTracker
273       checkResult(r, c0, T4,T3);
274 
275       g = new Get(T1);
276       g.setMaxVersions();
277       g.addColumn(c0, c0);
278       r = region.get(g);  // this'll use ExplicitColumnTracker
279       checkResult(r, c0, T4,T3);
280 
281       p = new Put(T1, ts+1);
282       p.add(c0, c0, T5);
283       region.put(p);
284 
285       // now the latest version is in the memstore
286 
287       g = new Get(T1);
288       g.setMaxVersions();
289       r = region.get(g);  // this'll use ScanWildcardColumnTracker
290       checkResult(r, c0, T5,T4);
291 
292       g = new Get(T1);
293       g.setMaxVersions();
294       g.addColumn(c0, c0);
295       r = region.get(g);  // this'll use ExplicitColumnTracker
296       checkResult(r, c0, T5,T4);
297     } finally {
298       HRegion.closeHRegion(region);
299     }
300   }
301 
302   /**
303    * Verify basic minimum versions functionality
304    */
305   @Test
306   public void testBaseCase() throws Exception {
307     // 1 version minimum, 1000 versions maximum, ttl = 1s
308     HTableDescriptor htd =
309         hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE);
310     HRegion region = hbu.createLocalHRegion(htd, null, null);
311     try {
312 
313       // 2s in the past
314       long ts = EnvironmentEdgeManager.currentTime() - 2000;
315 
316        // 1st version
317       Put p = new Put(T1, ts-3);
318       p.add(c0, c0, T1);
319       region.put(p);
320 
321       // 2nd version
322       p = new Put(T1, ts-2);
323       p.add(c0, c0, T2);
324       region.put(p);
325 
326       // 3rd version
327       p = new Put(T1, ts-1);
328       p.add(c0, c0, T3);
329       region.put(p);
330 
331       // 4th version
332       p = new Put(T1, ts);
333       p.add(c0, c0, T4);
334       region.put(p);
335 
336       Result r = region.get(new Get(T1));
337       checkResult(r, c0, T4);
338 
339       Get g = new Get(T1);
340       g.setTimeRange(0L, ts+1);
341       r = region.get(g);
342       checkResult(r, c0, T4);
343 
344   // oldest version still exists
345       g.setTimeRange(0L, ts-2);
346       r = region.get(g);
347       checkResult(r, c0, T1);
348 
349       // gets see only available versions
350       // even before compactions
351       g = new Get(T1);
352       g.setMaxVersions();
353       r = region.get(g); // this'll use ScanWildcardColumnTracker
354       checkResult(r, c0, T4,T3);
355 
356       g = new Get(T1);
357       g.setMaxVersions();
358       g.addColumn(c0, c0);
359       r = region.get(g);  // this'll use ExplicitColumnTracker
360       checkResult(r, c0, T4,T3);
361 
362       // now flush
363       region.flush(true);
364 
365       // with HBASE-4241 a flush will eliminate the expired rows
366       g = new Get(T1);
367       g.setTimeRange(0L, ts-2);
368       r = region.get(g);
369       assertTrue(r.isEmpty());
370 
371       // major compaction
372       region.compact(true);
373 
374       // after compaction the 4th version is still available
375       g = new Get(T1);
376       g.setTimeRange(0L, ts+1);
377       r = region.get(g);
378       checkResult(r, c0, T4);
379 
380       // so is the 3rd
381       g.setTimeRange(0L, ts);
382       r = region.get(g);
383       checkResult(r, c0, T3);
384 
385       // but the 2nd and earlier versions are gone
386       g.setTimeRange(0L, ts-1);
387       r = region.get(g);
388       assertTrue(r.isEmpty());
389     } finally {
390       HRegion.closeHRegion(region);
391     }
392   }
393 
394   /**
395    * Verify that basic filters still behave correctly with
396    * minimum versions enabled.
397    */
398   @Test
399   public void testFilters() throws Exception {
400     HTableDescriptor htd =
401         hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE);
402     HRegion region = hbu.createLocalHRegion(htd, null, null);
403     final byte [] c1 = COLUMNS[1];
404 
405     // 2s in the past
406     long ts = EnvironmentEdgeManager.currentTime() - 2000;
407     try {
408 
409       Put p = new Put(T1, ts-3);
410       p.add(c0, c0, T0);
411       p.add(c1, c1, T0);
412       region.put(p);
413 
414       p = new Put(T1, ts-2);
415       p.add(c0, c0, T1);
416       p.add(c1, c1, T1);
417       region.put(p);
418 
419       p = new Put(T1, ts-1);
420       p.add(c0, c0, T2);
421       p.add(c1, c1, T2);
422       region.put(p);
423 
424       p = new Put(T1, ts);
425       p.add(c0, c0, T3);
426       p.add(c1, c1, T3);
427       region.put(p);
428 
429       List<Long> tss = new ArrayList<Long>();
430       tss.add(ts-1);
431       tss.add(ts-2);
432 
433       Get g = new Get(T1);
434       g.addColumn(c1,c1);
435       g.setFilter(new TimestampsFilter(tss));
436       g.setMaxVersions();
437       Result r = region.get(g);
438       checkResult(r, c1, T2,T1);
439 
440       g = new Get(T1);
441       g.addColumn(c0,c0);
442       g.setFilter(new TimestampsFilter(tss));
443       g.setMaxVersions();
444       r = region.get(g);
445       checkResult(r, c0, T2,T1);
446 
447       // now flush/compact
448       region.flush(true);
449       region.compact(true);
450 
451       g = new Get(T1);
452       g.addColumn(c1,c1);
453       g.setFilter(new TimestampsFilter(tss));
454       g.setMaxVersions();
455       r = region.get(g);
456       checkResult(r, c1, T2);
457 
458       g = new Get(T1);
459       g.addColumn(c0,c0);
460       g.setFilter(new TimestampsFilter(tss));
461       g.setMaxVersions();
462       r = region.get(g);
463       checkResult(r, c0, T2);
464     } finally {
465       HRegion.closeHRegion(region);
466     }
467   }
468 
469   private void checkResult(Result r, byte[] col, byte[] ... vals) {
470     assertEquals(r.size(), vals.length);
471     List<Cell> kvs = r.getColumnCells(col, col);
472     assertEquals(kvs.size(), vals.length);
473     for (int i=0;i<vals.length;i++) {
474       assertTrue(CellUtil.matchingValue(kvs.get(i), vals[i]));
475     }
476   }
477 
478 }
479