View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase;
20  
21  import java.io.IOException;
22  
23  import org.apache.hadoop.hbase.client.Delete;
24  import org.apache.hadoop.hbase.client.Get;
25  import org.apache.hadoop.hbase.client.Put;
26  import org.apache.hadoop.hbase.client.Result;
27  import org.apache.hadoop.hbase.client.Durability;
28  import org.apache.hadoop.hbase.util.Bytes;
29  
30  /**
31   * Tests user specifiable time stamps putting, getting and scanning.  Also
32   * tests same in presence of deletes.  Test cores are written so can be
33   * run against an HRegion and against an HTable: i.e. both local and remote.
34   */
35  public class TimestampTestBase extends HBaseTestCase {
36    private static final long T0 = 10L;
37    private static final long T1 = 100L;
38    private static final long T2 = 200L;
39  
40    public static final byte [] FAMILY_NAME = Bytes.toBytes("colfamily11");
41    private static final byte [] QUALIFIER_NAME = Bytes.toBytes("contents");
42  
43    private static final byte [] ROW = Bytes.toBytes("row");
44  
45      /*
46     * Run test that delete works according to description in <a
47     * href="https://issues.apache.org/jira/browse/HADOOP-1784">hadoop-1784</a>.
48     * @param incommon
49     * @param flusher
50     * @throws IOException
51     */
52    public static void doTestDelete(final Incommon incommon, FlushCache flusher)
53    throws IOException {
54      // Add values at various timestamps (Values are timestampes as bytes).
55      put(incommon, T0);
56      put(incommon, T1);
57      put(incommon, T2);
58      put(incommon);
59      // Verify that returned versions match passed timestamps.
60      assertVersions(incommon, new long [] {HConstants.LATEST_TIMESTAMP, T2, T1});
61  
62      // If I delete w/o specifying a timestamp, this means I'm deleting the
63      // latest.
64      delete(incommon);
65      // Verify that I get back T2 through T1 -- that the latest version has
66      // been deleted.
67      assertVersions(incommon, new long [] {T2, T1, T0});
68  
69      // Flush everything out to disk and then retry
70      flusher.flushcache();
71      assertVersions(incommon, new long [] {T2, T1, T0});
72  
73      // Now add, back a latest so I can test remove other than the latest.
74      put(incommon);
75      assertVersions(incommon, new long [] {HConstants.LATEST_TIMESTAMP, T2, T1});
76      delete(incommon, T2);
77      assertVersions(incommon, new long [] {HConstants.LATEST_TIMESTAMP, T1, T0});
78      // Flush everything out to disk and then retry
79      flusher.flushcache();
80      assertVersions(incommon, new long [] {HConstants.LATEST_TIMESTAMP, T1, T0});
81  
82      // Now try deleting all from T2 back inclusive (We first need to add T2
83      // back into the mix and to make things a little interesting, delete and
84      // then readd T1.
85      put(incommon, T2);
86      delete(incommon, T1);
87      put(incommon, T1);
88  
89      Delete delete = new Delete(ROW);
90      delete.deleteColumns(FAMILY_NAME, QUALIFIER_NAME, T2);
91      incommon.delete(delete, true);
92  
93      // Should only be current value in set.  Assert this is so
94      assertOnlyLatest(incommon, HConstants.LATEST_TIMESTAMP);
95  
96      // Flush everything out to disk and then redo above tests
97      flusher.flushcache();
98      assertOnlyLatest(incommon, HConstants.LATEST_TIMESTAMP);
99    }
100 
101   private static void assertOnlyLatest(final Incommon incommon,
102     final long currentTime)
103   throws IOException {
104     Get get = null;
105     get = new Get(ROW);
106     get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
107     get.setMaxVersions(3);
108     Result result = incommon.get(get);
109     assertEquals(1, result.size());
110     long time = Bytes.toLong(CellUtil.cloneValue(result.rawCells()[0]));
111     assertEquals(time, currentTime);
112   }
113 
114   /*
115    * Assert that returned versions match passed in timestamps and that results
116    * are returned in the right order.  Assert that values when converted to
117    * longs match the corresponding passed timestamp.
118    * @param r
119    * @param tss
120    * @throws IOException
121    */
122   public static void assertVersions(final Incommon incommon, final long [] tss)
123   throws IOException {
124     // Assert that 'latest' is what we expect.
125     Get get = null;
126     get = new Get(ROW);
127     get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
128     Result r = incommon.get(get);
129     byte [] bytes = r.getValue(FAMILY_NAME, QUALIFIER_NAME);
130     long t = Bytes.toLong(bytes);
131     assertEquals(tss[0], t);
132 
133     // Now assert that if we ask for multiple versions, that they come out in
134     // order.
135     get = new Get(ROW);
136     get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
137     get.setMaxVersions(tss.length);
138     Result result = incommon.get(get);
139     Cell [] kvs = result.rawCells();
140     assertEquals(kvs.length, tss.length);
141     for(int i=0;i<kvs.length;i++) {
142       t = Bytes.toLong(CellUtil.cloneValue(kvs[i]));
143       assertEquals(tss[i], t);
144     }
145 
146     // Determine highest stamp to set as next max stamp
147     long maxStamp = kvs[0].getTimestamp();
148 
149     // Specify a timestamp get multiple versions.
150     get = new Get(ROW);
151     get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
152     get.setTimeRange(0, maxStamp);
153     get.setMaxVersions(kvs.length - 1);
154     result = incommon.get(get);
155     kvs = result.rawCells();
156     assertEquals(kvs.length, tss.length - 1);
157     for(int i=1;i<kvs.length;i++) {
158       t = Bytes.toLong(CellUtil.cloneValue(kvs[i-1]));
159       assertEquals(tss[i], t);
160     }
161 
162     // Test scanner returns expected version
163     assertScanContentTimestamp(incommon, tss[0]);
164   }
165 
166   /*
167    * Run test scanning different timestamps.
168    * @param incommon
169    * @param flusher
170    * @throws IOException
171    */
172   public static void doTestTimestampScanning(final Incommon incommon,
173     final FlushCache flusher)
174   throws IOException {
175     // Add a couple of values for three different timestamps.
176     put(incommon, T0);
177     put(incommon, T1);
178     put(incommon, HConstants.LATEST_TIMESTAMP);
179     // Get count of latest items.
180     int count = assertScanContentTimestamp(incommon,
181       HConstants.LATEST_TIMESTAMP);
182     // Assert I get same count when I scan at each timestamp.
183     assertEquals(count, assertScanContentTimestamp(incommon, T0));
184     assertEquals(count, assertScanContentTimestamp(incommon, T1));
185     // Flush everything out to disk and then retry
186     flusher.flushcache();
187     assertEquals(count, assertScanContentTimestamp(incommon, T0));
188     assertEquals(count, assertScanContentTimestamp(incommon, T1));
189   }
190 
191   /*
192    * Assert that the scan returns only values < timestamp.
193    * @param r
194    * @param ts
195    * @return Count of items scanned.
196    * @throws IOException
197    */
198   public static int assertScanContentTimestamp(final Incommon in, final long ts)
199   throws IOException {
200     ScannerIncommon scanner =
201       in.getScanner(COLUMNS[0], null, HConstants.EMPTY_START_ROW, ts);
202     int count = 0;
203     try {
204       // TODO FIX
205 //      HStoreKey key = new HStoreKey();
206 //      TreeMap<byte [], Cell>value =
207 //        new TreeMap<byte [], Cell>(Bytes.BYTES_COMPARATOR);
208 //      while (scanner.next(key, value)) {
209 //        assertTrue(key.getTimestamp() <= ts);
210 //        // Content matches the key or HConstants.LATEST_TIMESTAMP.
211 //        // (Key does not match content if we 'put' with LATEST_TIMESTAMP).
212 //        long l = Bytes.toLong(value.get(COLUMN).getValue());
213 //        assertTrue(key.getTimestamp() == l ||
214 //          HConstants.LATEST_TIMESTAMP == l);
215 //        count++;
216 //        value.clear();
217 //      }
218     } finally {
219       scanner.close();
220     }
221     return count;
222   }
223 
224   public static void put(final Incommon loader, final long ts)
225   throws IOException {
226     put(loader, Bytes.toBytes(ts), ts);
227   }
228 
229   public static void put(final Incommon loader)
230   throws IOException {
231     long ts = HConstants.LATEST_TIMESTAMP;
232     put(loader, Bytes.toBytes(ts), ts);
233   }
234 
235   /*
236    * Put values.
237    * @param loader
238    * @param bytes
239    * @param ts
240    * @throws IOException
241    */
242   public static void put(final Incommon loader, final byte [] bytes,
243     final long ts)
244   throws IOException {
245     Put put = new Put(ROW, ts);
246     put.setDurability(Durability.SKIP_WAL);
247     put.add(FAMILY_NAME, QUALIFIER_NAME, bytes);
248     loader.put(put);
249   }
250 
251   public static void delete(final Incommon loader) throws IOException {
252     delete(loader, null);
253   }
254 
255   public static void delete(final Incommon loader, final byte [] column)
256   throws IOException {
257     delete(loader, column, HConstants.LATEST_TIMESTAMP);
258   }
259 
260   public static void delete(final Incommon loader, final long ts)
261   throws IOException {
262     delete(loader, null, ts);
263   }
264 
265   public static void delete(final Incommon loader, final byte [] column,
266       final long ts)
267   throws IOException {
268     Delete delete = ts == HConstants.LATEST_TIMESTAMP?
269       new Delete(ROW): new Delete(ROW, ts);
270     delete.deleteColumn(FAMILY_NAME, QUALIFIER_NAME, ts);
271     loader.delete(delete, true);
272   }
273 
274   public static Result get(final Incommon loader) throws IOException {
275     return loader.get(new Get(ROW));
276   }
277 }