View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS;
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertTrue;
24  
25  import java.util.ArrayList;
26  import java.util.List;
27  
28  import org.apache.hadoop.hbase.Cell;
29  import org.apache.hadoop.hbase.CellUtil;
30  import org.apache.hadoop.hbase.HBaseTestingUtility;
31  import org.apache.hadoop.hbase.HTableDescriptor;
32  import org.apache.hadoop.hbase.SmallTests;
33  import org.apache.hadoop.hbase.client.Delete;
34  import org.apache.hadoop.hbase.client.Get;
35  import org.apache.hadoop.hbase.client.Put;
36  import org.apache.hadoop.hbase.client.Result;
37  import org.apache.hadoop.hbase.filter.TimestampsFilter;
38  import org.apache.hadoop.hbase.util.Bytes;
39  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
40  import org.junit.Rule;
41  import org.junit.Test;
42  import org.junit.experimental.categories.Category;
43  import org.junit.rules.TestName;
44  
45  /**
46   * Test Minimum Versions feature (HBASE-4071).
47   */
48  @Category(SmallTests.class)
49  public class TestMinVersions {
50    HBaseTestingUtility hbu = HBaseTestingUtility.createLocalHTU();
51    private final byte[] T0 = Bytes.toBytes("0");
52    private final byte[] T1 = Bytes.toBytes("1");
53    private final byte[] T2 = Bytes.toBytes("2");
54    private final byte[] T3 = Bytes.toBytes("3");
55    private final byte[] T4 = Bytes.toBytes("4");
56    private final byte[] T5 = Bytes.toBytes("5");
57  
58    private final byte[] c0 = COLUMNS[0];
59  
60    @Rule public TestName name = new TestName();
61  
62    /**
63     * Verify behavior of getClosestBefore(...)
64     */
65    @Test
66    public void testGetClosestBefore() throws Exception {
67      HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 1, 1000, 1, false);
68      HRegion region = hbu.createLocalHRegion(htd, null, null);
69      try {
70  
71        // 2s in the past
72        long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000;
73  
74        Put p = new Put(T1, ts);
75        p.add(c0, c0, T1);
76        region.put(p);
77  
78        p = new Put(T1, ts+1);
79        p.add(c0, c0, T4);
80        region.put(p);
81  
82        p = new Put(T3, ts);
83        p.add(c0, c0, T3);
84        region.put(p);
85  
86        // now make sure that getClosestBefore(...) get can
87        // rows that would be expired without minVersion.
88        // also make sure it gets the latest version
89        Result r = region.getClosestRowBefore(T1, c0);
90        checkResult(r, c0, T4);
91  
92        r = region.getClosestRowBefore(T2, c0);
93        checkResult(r, c0, T4);
94  
95        // now flush/compact
96        region.flushcache();
97        region.compactStores(true);
98  
99        r = region.getClosestRowBefore(T1, c0);
100       checkResult(r, c0, T4);
101 
102       r = region.getClosestRowBefore(T2, c0);
103       checkResult(r, c0, T4);
104     } finally {
105       HRegion.closeHRegion(region);
106     }
107   }
108 
109   /**
110    * Test mixed memstore and storefile scanning
111    * with minimum versions.
112    */
113   @Test
114   public void testStoreMemStore() throws Exception {
115     // keep 3 versions minimum
116     HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, false);
117     HRegion region = hbu.createLocalHRegion(htd, null, null);
118     // 2s in the past
119     long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000;
120 
121     try {
122       Put p = new Put(T1, ts-1);
123       p.add(c0, c0, T2);
124       region.put(p);
125 
126       p = new Put(T1, ts-3);
127       p.add(c0, c0, T0);
128       region.put(p);
129 
130       // now flush/compact
131       region.flushcache();
132       region.compactStores(true);
133 
134       p = new Put(T1, ts);
135       p.add(c0, c0, T3);
136       region.put(p);
137 
138       p = new Put(T1, ts-2);
139       p.add(c0, c0, T1);
140       region.put(p);
141 
142       p = new Put(T1, ts-3);
143       p.add(c0, c0, T0);
144       region.put(p);
145 
146       // newest version in the memstore
147       // the 2nd oldest in the store file
148       // and the 3rd, 4th oldest also in the memstore
149 
150       Get g = new Get(T1);
151       g.setMaxVersions();
152       Result r = region.get(g); // this'll use ScanWildcardColumnTracker
153       checkResult(r, c0, T3,T2,T1);
154 
155       g = new Get(T1);
156       g.setMaxVersions();
157       g.addColumn(c0, c0);
158       r = region.get(g);  // this'll use ExplicitColumnTracker
159       checkResult(r, c0, T3,T2,T1);
160     } finally {
161       HRegion.closeHRegion(region);
162     }
163   }
164 
165   /**
166    * Make sure the Deletes behave as expected with minimum versions
167    */
168   @Test
169   public void testDelete() throws Exception {
170     HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, false);
171     HRegion region = hbu.createLocalHRegion(htd, null, null);
172 
173     // 2s in the past
174     long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000;
175 
176     try {
177       Put p = new Put(T1, ts-2);
178       p.add(c0, c0, T1);
179       region.put(p);
180 
181       p = new Put(T1, ts-1);
182       p.add(c0, c0, T2);
183       region.put(p);
184 
185       p = new Put(T1, ts);
186       p.add(c0, c0, T3);
187       region.put(p);
188 
189       Delete d = new Delete(T1, ts-1);
190       region.delete(d);
191 
192       Get g = new Get(T1);
193       g.setMaxVersions();
194       Result r = region.get(g);  // this'll use ScanWildcardColumnTracker
195       checkResult(r, c0, T3);
196 
197       g = new Get(T1);
198       g.setMaxVersions();
199       g.addColumn(c0, c0);
200       r = region.get(g);  // this'll use ExplicitColumnTracker
201       checkResult(r, c0, T3);
202 
203       // now flush/compact
204       region.flushcache();
205       region.compactStores(true);
206 
207       // try again
208       g = new Get(T1);
209       g.setMaxVersions();
210       r = region.get(g);  // this'll use ScanWildcardColumnTracker
211       checkResult(r, c0, T3);
212 
213       g = new Get(T1);
214       g.setMaxVersions();
215       g.addColumn(c0, c0);
216       r = region.get(g);  // this'll use ExplicitColumnTracker
217       checkResult(r, c0, T3);
218     } finally {
219       HRegion.closeHRegion(region);
220     }
221   }
222 
223   /**
224    * Make sure the memstor behaves correctly with minimum versions
225    */
226   @Test
227   public void testMemStore() throws Exception {
228     HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, false);
229     HRegion region = hbu.createLocalHRegion(htd, null, null);
230 
231     // 2s in the past
232     long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000;
233 
234     try {
235       // 2nd version
236       Put p = new Put(T1, ts-2);
237       p.add(c0, c0, T2);
238       region.put(p);
239 
240       // 3rd version
241       p = new Put(T1, ts-1);
242       p.add(c0, c0, T3);
243       region.put(p);
244 
245       // 4th version
246       p = new Put(T1, ts);
247       p.add(c0, c0, T4);
248       region.put(p);
249 
250       // now flush/compact
251       region.flushcache();
252       region.compactStores(true);
253 
254       // now put the first version (backdated)
255       p = new Put(T1, ts-3);
256       p.add(c0, c0, T1);
257       region.put(p);
258 
259       // now the latest change is in the memstore,
260       // but it is not the latest version
261 
262       Result r = region.get(new Get(T1));
263       checkResult(r, c0, T4);
264 
265       Get g = new Get(T1);
266       g.setMaxVersions();
267       r = region.get(g); // this'll use ScanWildcardColumnTracker
268       checkResult(r, c0, T4,T3);
269 
270       g = new Get(T1);
271       g.setMaxVersions();
272       g.addColumn(c0, c0);
273       r = region.get(g);  // this'll use ExplicitColumnTracker
274       checkResult(r, c0, T4,T3);
275 
276       p = new Put(T1, ts+1);
277       p.add(c0, c0, T5);
278       region.put(p);
279 
280       // now the latest version is in the memstore
281 
282       g = new Get(T1);
283       g.setMaxVersions();
284       r = region.get(g);  // this'll use ScanWildcardColumnTracker
285       checkResult(r, c0, T5,T4);
286 
287       g = new Get(T1);
288       g.setMaxVersions();
289       g.addColumn(c0, c0);
290       r = region.get(g);  // this'll use ExplicitColumnTracker
291       checkResult(r, c0, T5,T4);
292     } finally {
293       HRegion.closeHRegion(region);
294     }
295   }
296 
297   /**
298    * Verify basic minimum versions functionality
299    */
300   @Test
301   public void testBaseCase() throws Exception {
302     // 1 version minimum, 1000 versions maximum, ttl = 1s
303     HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, false);
304     HRegion region = hbu.createLocalHRegion(htd, null, null);
305     try {
306 
307       // 2s in the past
308       long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000;
309 
310        // 1st version
311       Put p = new Put(T1, ts-3);
312       p.add(c0, c0, T1);
313       region.put(p);
314 
315       // 2nd version
316       p = new Put(T1, ts-2);
317       p.add(c0, c0, T2);
318       region.put(p);
319 
320       // 3rd version
321       p = new Put(T1, ts-1);
322       p.add(c0, c0, T3);
323       region.put(p);
324 
325       // 4th version
326       p = new Put(T1, ts);
327       p.add(c0, c0, T4);
328       region.put(p);
329 
330       Result r = region.get(new Get(T1));
331       checkResult(r, c0, T4);
332 
333       Get g = new Get(T1);
334       g.setTimeRange(0L, ts+1);
335       r = region.get(g);
336       checkResult(r, c0, T4);
337 
338   // oldest version still exists
339       g.setTimeRange(0L, ts-2);
340       r = region.get(g);
341       checkResult(r, c0, T1);
342 
343       // gets see only available versions
344       // even before compactions
345       g = new Get(T1);
346       g.setMaxVersions();
347       r = region.get(g); // this'll use ScanWildcardColumnTracker
348       checkResult(r, c0, T4,T3);
349 
350       g = new Get(T1);
351       g.setMaxVersions();
352       g.addColumn(c0, c0);
353       r = region.get(g);  // this'll use ExplicitColumnTracker
354       checkResult(r, c0, T4,T3);
355 
356       // now flush
357       region.flushcache();
358 
359       // with HBASE-4241 a flush will eliminate the expired rows
360       g = new Get(T1);
361       g.setTimeRange(0L, ts-2);
362       r = region.get(g);
363       assertTrue(r.isEmpty());
364 
365       // major compaction
366       region.compactStores(true);
367 
368       // after compaction the 4th version is still available
369       g = new Get(T1);
370       g.setTimeRange(0L, ts+1);
371       r = region.get(g);
372       checkResult(r, c0, T4);
373 
374       // so is the 3rd
375       g.setTimeRange(0L, ts);
376       r = region.get(g);
377       checkResult(r, c0, T3);
378 
379       // but the 2nd and earlier versions are gone
380       g.setTimeRange(0L, ts-1);
381       r = region.get(g);
382       assertTrue(r.isEmpty());
383     } finally {
384       HRegion.closeHRegion(region);
385     }
386   }
387 
388   /**
389    * Verify that basic filters still behave correctly with
390    * minimum versions enabled.
391    */
392   @Test
393   public void testFilters() throws Exception {
394     HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, false);
395     HRegion region = hbu.createLocalHRegion(htd, null, null);
396     final byte [] c1 = COLUMNS[1];
397 
398     // 2s in the past
399     long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000;
400     try {
401 
402       Put p = new Put(T1, ts-3);
403       p.add(c0, c0, T0);
404       p.add(c1, c1, T0);
405       region.put(p);
406 
407       p = new Put(T1, ts-2);
408       p.add(c0, c0, T1);
409       p.add(c1, c1, T1);
410       region.put(p);
411 
412       p = new Put(T1, ts-1);
413       p.add(c0, c0, T2);
414       p.add(c1, c1, T2);
415       region.put(p);
416 
417       p = new Put(T1, ts);
418       p.add(c0, c0, T3);
419       p.add(c1, c1, T3);
420       region.put(p);
421 
422       List<Long> tss = new ArrayList<Long>();
423       tss.add(ts-1);
424       tss.add(ts-2);
425 
426       Get g = new Get(T1);
427       g.addColumn(c1,c1);
428       g.setFilter(new TimestampsFilter(tss));
429       g.setMaxVersions();
430       Result r = region.get(g);
431       checkResult(r, c1, T2,T1);
432 
433       g = new Get(T1);
434       g.addColumn(c0,c0);
435       g.setFilter(new TimestampsFilter(tss));
436       g.setMaxVersions();
437       r = region.get(g);
438       checkResult(r, c0, T2,T1);
439 
440       // now flush/compact
441       region.flushcache();
442       region.compactStores(true);
443 
444       g = new Get(T1);
445       g.addColumn(c1,c1);
446       g.setFilter(new TimestampsFilter(tss));
447       g.setMaxVersions();
448       r = region.get(g);
449       checkResult(r, c1, T2);
450 
451       g = new Get(T1);
452       g.addColumn(c0,c0);
453       g.setFilter(new TimestampsFilter(tss));
454       g.setMaxVersions();
455       r = region.get(g);
456       checkResult(r, c0, T2);
457     } finally {
458       HRegion.closeHRegion(region);
459     }
460   }
461 
462   private void checkResult(Result r, byte[] col, byte[] ... vals) {
463     assertEquals(r.size(), vals.length);
464     List<Cell> kvs = r.getColumnCells(col, col);
465     assertEquals(kvs.size(), vals.length);
466     for (int i=0;i<vals.length;i++) {
467       assertTrue(CellUtil.matchingValue(kvs.get(i), vals[i]));
468     }
469   }
470 
471 }
472