View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.util;
20  
21  import org.apache.hadoop.conf.Configuration;
22  import org.apache.hadoop.fs.FileSystem;
23  import org.apache.hadoop.fs.Path;
24  import org.apache.hadoop.hbase.Cell;
25  import org.apache.hadoop.hbase.KeyValue;
26  import org.apache.hadoop.hbase.Tag;
27  import org.apache.hadoop.hbase.TagType;
28  import org.apache.hadoop.hbase.client.Result;
29  import org.apache.hadoop.hbase.client.ResultScanner;
30  import org.apache.hadoop.hbase.client.Scan;
31  import org.apache.hadoop.hbase.client.Table;
32  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
33  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
34  import org.apache.hadoop.hbase.io.hfile.HFile;
35  import org.apache.hadoop.hbase.io.hfile.HFileContext;
36  import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
37  import org.apache.hadoop.hbase.regionserver.StoreFile;
38  
39  import java.io.IOException;
40  import java.util.ArrayList;
41  import java.util.List;
42  
43  import static org.junit.Assert.assertArrayEquals;
44  import static org.junit.Assert.fail;
45  
46  /**
47   * Utility class for HFile-related testing.
48   */
49  public class HFileTestUtil {
50  
51    /**
52     * Create an HFile with the given number of rows between a given
53     * start key and end key @ family:qualifier.  The value will be the key value.
54     * This file will not have tags.
55     */
56    public static void createHFile(
57        Configuration configuration,
58        FileSystem fs, Path path,
59        byte[] family, byte[] qualifier,
60        byte[] startKey, byte[] endKey, int numRows) throws IOException {
61        createHFile(configuration, fs, path, DataBlockEncoding.NONE, family, qualifier,
62          startKey, endKey, numRows, false);
63    }
64  
65    /**
66     * Create an HFile with the given number of rows between a given
67     * start key and end key @ family:qualifier.  The value will be the key value.
68     * This file will use certain data block encoding algorithm.
69     */
70    public static void createHFileWithDataBlockEncoding(
71        Configuration configuration,
72        FileSystem fs, Path path, DataBlockEncoding encoding,
73        byte[] family, byte[] qualifier,
74        byte[] startKey, byte[] endKey, int numRows) throws IOException {
75        createHFile(configuration, fs, path, encoding, family, qualifier, startKey, endKey,
76          numRows, false);
77    }
78  
79    /**
80     * Create an HFile with the given number of rows between a given
81     * start key and end key @ family:qualifier.  The value will be the key value.
82     * This cells will also have a tag whose value is the key.
83     */
84    public static void createHFileWithTags(
85        Configuration configuration,
86        FileSystem fs, Path path,
87        byte[] family, byte[] qualifier,
88        byte[] startKey, byte[] endKey, int numRows) throws IOException {
89        createHFile(configuration, fs, path, DataBlockEncoding.NONE, family, qualifier,
90          startKey, endKey, numRows, true);
91    }
92  
93    /**
94     * Create an HFile with the given number of rows between a given
95     * start key and end key @ family:qualifier.
96     * If withTag is true, we add the rowKey as the tag value for
97     * tagtype ACL_TAG_TYPE
98     */
99    public static void createHFile(
100       Configuration configuration,
101       FileSystem fs, Path path, DataBlockEncoding encoding,
102       byte[] family, byte[] qualifier,
103       byte[] startKey, byte[] endKey, int numRows, boolean withTag) throws IOException {
104     HFileContext meta = new HFileContextBuilder()
105         .withIncludesTags(withTag)
106         .withDataBlockEncoding(encoding)
107         .build();
108     HFile.Writer writer = HFile.getWriterFactory(configuration, new CacheConfig(configuration))
109         .withPath(fs, path)
110         .withFileContext(meta)
111         .create();
112     long now = System.currentTimeMillis();
113     try {
114       // subtract 2 since iterateOnSplits doesn't include boundary keys
115       for (byte[] key : Bytes.iterateOnSplits(startKey, endKey, numRows - 2)) {
116         KeyValue kv = new KeyValue(key, family, qualifier, now, key);
117         if (withTag) {
118           // add a tag.  Arbitrarily chose mob tag since we have a helper already.
119           List<Tag> tags = new ArrayList<Tag>();
120           tags.add(new Tag(TagType.ACL_TAG_TYPE, key));
121           kv = new KeyValue(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(),
122               kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(),
123               kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength(),
124               kv.getTimestamp(), KeyValue.Type.Put, kv.getValueArray(), kv.getValueOffset(),
125               kv.getValueLength(), tags);
126 
127           // verify that the kv has the tag.
128           byte[] ta = kv.getTagsArray();
129           int toff = kv.getTagsOffset();
130           int tlen = kv.getTagsLength();
131           Tag t = Tag.getTag(ta, toff, tlen, TagType.ACL_TAG_TYPE);
132           if (t == null) {
133             throw new IllegalStateException("Tag didn't stick to KV " + kv.toString());
134           }
135         }
136         writer.append(kv);
137       }
138     } finally {
139       writer.appendFileInfo(StoreFile.BULKLOAD_TIME_KEY,
140           Bytes.toBytes(System.currentTimeMillis()));
141       writer.close();
142     }
143   }
144 
145   /**
146    * This verifies that each cell has a tag that is equal to its rowkey name.  For this to work
147    * the hbase instance must have HConstants.RPC_CODEC_CONF_KEY set to
148    * KeyValueCodecWithTags.class.getCanonicalName());
149    * @param table table containing tagged cells
150    * @throws IOException if problems reading table
151    */
152   public static void verifyTags(Table table) throws IOException {
153     ResultScanner s = table.getScanner(new Scan());
154     for (Result r : s) {
155       for (Cell c : r.listCells()) {
156         byte[] ta = c.getTagsArray();
157         int toff = c.getTagsOffset();
158         int tlen = c.getTagsLength();
159         Tag t = Tag.getTag(ta, toff, tlen, TagType.ACL_TAG_TYPE);
160         if (t == null) {
161           fail(c.toString() + " has null tag");
162           continue;
163         }
164         byte[] tval = t.getValue();
165         assertArrayEquals(c.toString() + " has tag" + Bytes.toString(tval),
166             r.getRow(), tval);
167       }
168     }
169   }
170 }