View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.Arrays;
24  import java.util.Collection;
25  import java.util.Collections;
26  import java.util.Comparator;
27  import java.util.List;
28  import java.util.NavigableSet;
29  import java.util.TreeSet;
30  
31  import org.apache.commons.logging.Log;
32  import org.apache.commons.logging.LogFactory;
33  import org.apache.hadoop.conf.Configuration;
34  import org.apache.hadoop.fs.FileSystem;
35  import org.apache.hadoop.fs.Path;
36  import org.apache.hadoop.hbase.Cell;
37  import org.apache.hadoop.hbase.CellUtil;
38  import org.apache.hadoop.hbase.HBaseTestingUtility;
39  import org.apache.hadoop.hbase.HColumnDescriptor;
40  import org.apache.hadoop.hbase.KeyValue;
41  import org.apache.hadoop.hbase.KeyValueTestUtil;
42  import org.apache.hadoop.hbase.SmallTests;
43  import org.apache.hadoop.hbase.client.Put;
44  import org.apache.hadoop.hbase.client.Scan;
45  import org.apache.hadoop.hbase.io.compress.Compression;
46  import org.apache.hadoop.hbase.io.hfile.HFilePrettyPrinter;
47  import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
48  import org.apache.hadoop.hbase.util.Bytes;
49  import org.junit.Before;
50  import org.junit.Test;
51  import org.junit.experimental.categories.Category;
52  import org.junit.runner.RunWith;
53  import org.junit.runners.Parameterized;
54  import org.junit.runners.Parameterized.Parameters;
55  
56  import static org.junit.Assert.*;
57  
58  /**
59   * Test a multi-column scanner when there is a Bloom filter false-positive.
60   * This is needed for the multi-column Bloom filter optimization.
61   */
62  @RunWith(Parameterized.class)
63  @Category(SmallTests.class)
64  public class TestScanWithBloomError {
65  
66    private static final Log LOG =
67      LogFactory.getLog(TestScanWithBloomError.class);
68  
69    private static final String TABLE_NAME = "ScanWithBloomError";
70    private static final String FAMILY = "myCF";
71    private static final byte[] FAMILY_BYTES = Bytes.toBytes(FAMILY);
72    private static final String ROW = "theRow";
73    private static final String QUALIFIER_PREFIX = "qual";
74    private static final byte[] ROW_BYTES = Bytes.toBytes(ROW);
75    private static NavigableSet<Integer> allColIds = new TreeSet<Integer>();
76    private HRegion region;
77    private BloomType bloomType;
78    private FileSystem fs;
79    private Configuration conf;
80  
81    private final static HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
82  
83    @Parameters
84    public static final Collection<Object[]> parameters() {
85      List<Object[]> configurations = new ArrayList<Object[]>();
86      for (BloomType bloomType : BloomType.values()) {
87        configurations.add(new Object[] { bloomType });
88      }
89      return configurations;
90    }
91  
92    public TestScanWithBloomError(BloomType bloomType) {
93      this.bloomType = bloomType;
94    }
95  
96    @Before
97    public void setUp() throws IOException{
98      conf = TEST_UTIL.getConfiguration();
99      fs = FileSystem.get(conf);
100   }
101 
102   @Test
103   public void testThreeStoreFiles() throws IOException {
104     region = TEST_UTIL.createTestRegion(TABLE_NAME,
105         new HColumnDescriptor(FAMILY)
106             .setCompressionType(Compression.Algorithm.GZ)
107             .setBloomFilterType(bloomType)
108             .setMaxVersions(TestMultiColumnScanner.MAX_VERSIONS));
109     createStoreFile(new int[] {1, 2, 6});
110     createStoreFile(new int[] {1, 2, 3, 7});
111     createStoreFile(new int[] {1, 9});
112     scanColSet(new int[]{1, 4, 6, 7}, new int[]{1, 6, 7});
113 
114     HRegion.closeHRegion(region);
115   }
116 
117   private void scanColSet(int[] colSet, int[] expectedResultCols)
118       throws IOException {
119     LOG.info("Scanning column set: " + Arrays.toString(colSet));
120     Scan scan = new Scan(ROW_BYTES, ROW_BYTES);
121     addColumnSetToScan(scan, colSet);
122     RegionScannerImpl scanner = (RegionScannerImpl) region.getScanner(scan);
123     KeyValueHeap storeHeap = scanner.getStoreHeapForTesting();
124     assertEquals(0, storeHeap.getHeap().size());
125     StoreScanner storeScanner =
126         (StoreScanner) storeHeap.getCurrentForTesting();
127     @SuppressWarnings({ "unchecked", "rawtypes" })
128     List<StoreFileScanner> scanners = (List<StoreFileScanner>)
129         (List) storeScanner.getAllScannersForTesting();
130 
131     // Sort scanners by their HFile's modification time.
132     Collections.sort(scanners, new Comparator<StoreFileScanner>() {
133       @Override
134       public int compare(StoreFileScanner s1, StoreFileScanner s2) {
135         Path p1 = s1.getReaderForTesting().getHFileReader().getPath();
136         Path p2 = s2.getReaderForTesting().getHFileReader().getPath();
137         long t1, t2;
138         try {
139           t1 = fs.getFileStatus(p1).getModificationTime();
140           t2 = fs.getFileStatus(p2).getModificationTime();
141         } catch (IOException ex) {
142           throw new RuntimeException(ex);
143         }
144         return t1 < t2 ? -1 : t1 == t2 ? 1 : 0;
145       }
146     });
147 
148     StoreFile.Reader lastStoreFileReader = null;
149     for (StoreFileScanner sfScanner : scanners)
150       lastStoreFileReader = sfScanner.getReaderForTesting();
151 
152     new HFilePrettyPrinter().run(new String[]{ "-m", "-p", "-f",
153         lastStoreFileReader.getHFileReader().getPath().toString()});
154 
155     // Disable Bloom filter for the last store file. The disabled Bloom filter
156     // will always return "true".
157     LOG.info("Disabling Bloom filter for: "
158         + lastStoreFileReader.getHFileReader().getName());
159     lastStoreFileReader.disableBloomFilterForTesting();
160 
161     List<Cell> allResults = new ArrayList<Cell>();
162 
163     { // Limit the scope of results.
164       List<Cell> results = new ArrayList<Cell>();
165       while (scanner.next(results) || results.size() > 0) {
166         allResults.addAll(results);
167         results.clear();
168       }
169     }
170 
171     List<Integer> actualIds = new ArrayList<Integer>();
172     for (Cell kv : allResults) {
173       String qual = Bytes.toString(CellUtil.cloneQualifier(kv));
174       assertTrue(qual.startsWith(QUALIFIER_PREFIX));
175       actualIds.add(Integer.valueOf(qual.substring(
176           QUALIFIER_PREFIX.length())));
177     }
178     List<Integer> expectedIds = new ArrayList<Integer>();
179     for (int expectedId : expectedResultCols)
180       expectedIds.add(expectedId);
181 
182     LOG.info("Column ids returned: " + actualIds + ", expected: "
183         + expectedIds);
184     assertEquals(expectedIds.toString(), actualIds.toString());
185   }
186 
187   private void addColumnSetToScan(Scan scan, int[] colIds) {
188     for (int colId : colIds) {
189       scan.addColumn(FAMILY_BYTES,
190           Bytes.toBytes(qualFromId(colId)));
191     }
192   }
193 
194   private String qualFromId(int colId) {
195     return QUALIFIER_PREFIX + colId;
196   }
197 
198   private void createStoreFile(int[] colIds)
199       throws IOException {
200     Put p = new Put(ROW_BYTES);
201     for (int colId : colIds) {
202       long ts = Long.MAX_VALUE;
203       String qual = qualFromId(colId);
204       allColIds.add(colId);
205       KeyValue kv = KeyValueTestUtil.create(ROW, FAMILY,
206           qual, ts, TestMultiColumnScanner.createValue(ROW, qual, ts));
207       p.add(kv);
208     }
209     region.put(p);
210     region.flushcache();
211   }
212 
213 
214 }
215