View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.filter;
19  
20  import static org.junit.Assert.*;
21  
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.HashMap;
25  import java.util.HashSet;
26  import java.util.List;
27  import java.util.Map;
28  import java.util.Set;
29  
30  import org.apache.hadoop.hbase.*;
31  import org.apache.hadoop.hbase.client.Put;
32  import org.apache.hadoop.hbase.client.Scan;
33  import org.apache.hadoop.hbase.client.Durability;
34  import org.apache.hadoop.hbase.regionserver.HRegion;
35  import org.apache.hadoop.hbase.regionserver.InternalScanner;
36  import org.apache.hadoop.hbase.util.Bytes;
37  import org.junit.Test;
38  import org.junit.experimental.categories.Category;
39  
40  @Category(SmallTests.class)
41  public class TestMultipleColumnPrefixFilter {
42  
43    private final static HBaseTestingUtility TEST_UTIL = new
44        HBaseTestingUtility();
45  
46    @Test
47    public void testMultipleColumnPrefixFilter() throws IOException {
48      String family = "Family";
49      HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("TestMultipleColumnPrefixFilter"));
50      HColumnDescriptor hcd = new HColumnDescriptor(family);
51      hcd.setMaxVersions(3);
52      htd.addFamily(hcd);
53      // HRegionInfo info = new HRegionInfo(htd, null, null, false);
54      HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
55      HRegion region = HRegion.createHRegion(info, TEST_UTIL.
56        getDataTestDir(), TEST_UTIL.getConfiguration(), htd);
57  
58      List<String> rows = generateRandomWords(100, "row");
59      List<String> columns = generateRandomWords(10000, "column");
60      long maxTimestamp = 2;
61  
62      List<Cell> kvList = new ArrayList<Cell>();
63  
64      Map<String, List<Cell>> prefixMap = new HashMap<String,
65          List<Cell>>();
66  
67      prefixMap.put("p", new ArrayList<Cell>());
68      prefixMap.put("q", new ArrayList<Cell>());
69      prefixMap.put("s", new ArrayList<Cell>());
70  
71      String valueString = "ValueString";
72  
73      for (String row: rows) {
74        Put p = new Put(Bytes.toBytes(row));
75        p.setDurability(Durability.SKIP_WAL);
76        for (String column: columns) {
77          for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
78            KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp,
79                valueString);
80            p.add(kv);
81            kvList.add(kv);
82            for (String s: prefixMap.keySet()) {
83              if (column.startsWith(s)) {
84                prefixMap.get(s).add(kv);
85              }
86            }
87          }
88        }
89        region.put(p);
90      }
91  
92      MultipleColumnPrefixFilter filter;
93      Scan scan = new Scan();
94      scan.setMaxVersions();
95      byte [][] filter_prefix = new byte [2][];
96      filter_prefix[0] = new byte [] {'p'};
97      filter_prefix[1] = new byte [] {'q'};
98      
99      filter = new MultipleColumnPrefixFilter(filter_prefix);
100     scan.setFilter(filter);
101     List<Cell> results = new ArrayList<Cell>();  
102     InternalScanner scanner = region.getScanner(scan);
103     while(scanner.next(results));
104     assertEquals(prefixMap.get("p").size() + prefixMap.get("q").size(), results.size());
105 
106     HRegion.closeHRegion(region);
107   }
108 
109   @Test
110   public void testMultipleColumnPrefixFilterWithManyFamilies() throws IOException {
111     String family1 = "Family1";
112     String family2 = "Family2";
113     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("TestMultipleColumnPrefixFilter"));
114     HColumnDescriptor hcd1 = new HColumnDescriptor(family1);
115     hcd1.setMaxVersions(3);
116     htd.addFamily(hcd1);
117     HColumnDescriptor hcd2 = new HColumnDescriptor(family2);
118     hcd2.setMaxVersions(3);
119     htd.addFamily(hcd2);
120     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
121     HRegion region = HRegion.createHRegion(info, TEST_UTIL.
122       getDataTestDir(), TEST_UTIL.getConfiguration(), htd);
123 
124     List<String> rows = generateRandomWords(100, "row");
125     List<String> columns = generateRandomWords(10000, "column");
126     long maxTimestamp = 3;
127 
128     List<Cell> kvList = new ArrayList<Cell>();
129 
130     Map<String, List<Cell>> prefixMap = new HashMap<String,
131         List<Cell>>();
132 
133     prefixMap.put("p", new ArrayList<Cell>());
134     prefixMap.put("q", new ArrayList<Cell>());
135     prefixMap.put("s", new ArrayList<Cell>());
136 
137     String valueString = "ValueString";
138 
139     for (String row: rows) {
140       Put p = new Put(Bytes.toBytes(row));
141       p.setDurability(Durability.SKIP_WAL);
142       for (String column: columns) {
143         for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
144           double rand = Math.random();
145           Cell kv;
146           if (rand < 0.5) 
147             kv = KeyValueTestUtil.create(row, family1, column, timestamp,
148                 valueString);
149           else 
150             kv = KeyValueTestUtil.create(row, family2, column, timestamp,
151                 valueString);
152           p.add(kv);
153           kvList.add(kv);
154           for (String s: prefixMap.keySet()) {
155             if (column.startsWith(s)) {
156               prefixMap.get(s).add(kv);
157             }
158           }
159         }
160       }
161       region.put(p);
162     }
163 
164     MultipleColumnPrefixFilter filter;
165     Scan scan = new Scan();
166     scan.setMaxVersions();
167     byte [][] filter_prefix = new byte [2][];
168     filter_prefix[0] = new byte [] {'p'};
169     filter_prefix[1] = new byte [] {'q'};
170     
171     filter = new MultipleColumnPrefixFilter(filter_prefix);
172     scan.setFilter(filter);
173     List<Cell> results = new ArrayList<Cell>();  
174     InternalScanner scanner = region.getScanner(scan);
175     while(scanner.next(results));
176     assertEquals(prefixMap.get("p").size() + prefixMap.get("q").size(), results.size());
177 
178     HRegion.closeHRegion(region);
179   }
180   
181   @Test
182   public void testMultipleColumnPrefixFilterWithColumnPrefixFilter() throws IOException {
183     String family = "Family";
184     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("TestMultipleColumnPrefixFilter"));
185     htd.addFamily(new HColumnDescriptor(family));
186     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
187     HRegion region = HRegion.createHRegion(info, TEST_UTIL.
188       getDataTestDir(), TEST_UTIL.getConfiguration(),htd);
189 
190     List<String> rows = generateRandomWords(100, "row");
191     List<String> columns = generateRandomWords(10000, "column");
192     long maxTimestamp = 2;
193 
194     String valueString = "ValueString";
195 
196     for (String row: rows) {
197       Put p = new Put(Bytes.toBytes(row));
198       p.setDurability(Durability.SKIP_WAL);
199       for (String column: columns) {
200         for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
201           KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp,
202               valueString);
203           p.add(kv);
204         }
205       }
206       region.put(p);
207     }
208 
209     MultipleColumnPrefixFilter multiplePrefixFilter;
210     Scan scan1 = new Scan();
211     scan1.setMaxVersions();
212     byte [][] filter_prefix = new byte [1][];
213     filter_prefix[0] = new byte [] {'p'};
214  
215     multiplePrefixFilter = new MultipleColumnPrefixFilter(filter_prefix);
216     scan1.setFilter(multiplePrefixFilter);
217     List<Cell> results1 = new ArrayList<Cell>();  
218     InternalScanner scanner1 = region.getScanner(scan1);
219     while(scanner1.next(results1));
220     
221     ColumnPrefixFilter singlePrefixFilter;
222     Scan scan2 = new Scan();
223     scan2.setMaxVersions();
224     singlePrefixFilter = new ColumnPrefixFilter(Bytes.toBytes("p"));
225  
226     scan2.setFilter(singlePrefixFilter);
227     List<Cell> results2 = new ArrayList<Cell>();  
228     InternalScanner scanner2 = region.getScanner(scan1);
229     while(scanner2.next(results2));
230     
231     assertEquals(results1.size(), results2.size());
232 
233     HRegion.closeHRegion(region);
234   }
235   
236   List<String> generateRandomWords(int numberOfWords, String suffix) {
237     Set<String> wordSet = new HashSet<String>();
238     for (int i = 0; i < numberOfWords; i++) {
239       int lengthOfWords = (int) (Math.random()*2) + 1;
240       char[] wordChar = new char[lengthOfWords];
241       for (int j = 0; j < wordChar.length; j++) {
242         wordChar[j] = (char) (Math.random() * 26 + 97);
243       }
244       String word;
245       if (suffix == null) {
246         word = new String(wordChar);
247       } else {
248         word = new String(wordChar) + suffix;
249       }
250       wordSet.add(word);
251     }
252     List<String> wordList = new ArrayList<String>(wordSet);
253     return wordList;
254   }
255 
256 }
257 
258