1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertTrue;
23
24 import java.io.IOException;
25 import java.util.ArrayList;
26 import java.util.Arrays;
27 import java.util.Collection;
28 import java.util.Collections;
29 import java.util.Comparator;
30 import java.util.List;
31 import java.util.NavigableSet;
32 import java.util.TreeSet;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FileSystem;
38 import org.apache.hadoop.fs.Path;
39 import org.apache.hadoop.hbase.Cell;
40 import org.apache.hadoop.hbase.CellUtil;
41 import org.apache.hadoop.hbase.HBaseTestingUtility;
42 import org.apache.hadoop.hbase.HColumnDescriptor;
43 import org.apache.hadoop.hbase.KeyValue;
44 import org.apache.hadoop.hbase.KeyValueTestUtil;
45 import org.apache.hadoop.hbase.client.Put;
46 import org.apache.hadoop.hbase.client.Scan;
47 import org.apache.hadoop.hbase.io.compress.Compression;
48 import org.apache.hadoop.hbase.io.hfile.HFilePrettyPrinter;
49 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
50 import org.apache.hadoop.hbase.testclassification.SmallTests;
51 import org.apache.hadoop.hbase.util.Bytes;
52 import org.junit.Before;
53 import org.junit.Test;
54 import org.junit.experimental.categories.Category;
55 import org.junit.runner.RunWith;
56 import org.junit.runners.Parameterized;
57 import org.junit.runners.Parameterized.Parameters;
58
59
60
61
62
63 @RunWith(Parameterized.class)
64 @Category(SmallTests.class)
65 public class TestScanWithBloomError {
66
67 private static final Log LOG =
68 LogFactory.getLog(TestScanWithBloomError.class);
69
70 private static final String TABLE_NAME = "ScanWithBloomError";
71 private static final String FAMILY = "myCF";
72 private static final byte[] FAMILY_BYTES = Bytes.toBytes(FAMILY);
73 private static final String ROW = "theRow";
74 private static final String QUALIFIER_PREFIX = "qual";
75 private static final byte[] ROW_BYTES = Bytes.toBytes(ROW);
76 private static NavigableSet<Integer> allColIds = new TreeSet<Integer>();
77 private HRegion region;
78 private BloomType bloomType;
79 private FileSystem fs;
80 private Configuration conf;
81
82 private final static HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
83
84 @Parameters
85 public static final Collection<Object[]> parameters() {
86 List<Object[]> configurations = new ArrayList<Object[]>();
87 for (BloomType bloomType : BloomType.values()) {
88 configurations.add(new Object[] { bloomType });
89 }
90 return configurations;
91 }
92
93 public TestScanWithBloomError(BloomType bloomType) {
94 this.bloomType = bloomType;
95 }
96
97 @Before
98 public void setUp() throws IOException{
99 conf = TEST_UTIL.getConfiguration();
100 fs = FileSystem.get(conf);
101 }
102
103 @Test
104 public void testThreeStoreFiles() throws IOException {
105 region = TEST_UTIL.createTestRegion(TABLE_NAME,
106 new HColumnDescriptor(FAMILY)
107 .setCompressionType(Compression.Algorithm.GZ)
108 .setBloomFilterType(bloomType)
109 .setMaxVersions(TestMultiColumnScanner.MAX_VERSIONS));
110 createStoreFile(new int[] {1, 2, 6});
111 createStoreFile(new int[] {1, 2, 3, 7});
112 createStoreFile(new int[] {1, 9});
113 scanColSet(new int[]{1, 4, 6, 7}, new int[]{1, 6, 7});
114
115 HRegion.closeHRegion(region);
116 }
117
118 private void scanColSet(int[] colSet, int[] expectedResultCols)
119 throws IOException {
120 LOG.info("Scanning column set: " + Arrays.toString(colSet));
121 Scan scan = new Scan(ROW_BYTES, ROW_BYTES);
122 addColumnSetToScan(scan, colSet);
123 RegionScannerImpl scanner = (RegionScannerImpl) region.getScanner(scan);
124 KeyValueHeap storeHeap = scanner.getStoreHeapForTesting();
125 assertEquals(0, storeHeap.getHeap().size());
126 StoreScanner storeScanner =
127 (StoreScanner) storeHeap.getCurrentForTesting();
128 @SuppressWarnings({ "unchecked", "rawtypes" })
129 List<StoreFileScanner> scanners = (List<StoreFileScanner>)
130 (List) storeScanner.getAllScannersForTesting();
131
132
133 Collections.sort(scanners, new Comparator<StoreFileScanner>() {
134 @Override
135 public int compare(StoreFileScanner s1, StoreFileScanner s2) {
136 Path p1 = s1.getReader().getHFileReader().getPath();
137 Path p2 = s2.getReader().getHFileReader().getPath();
138 long t1, t2;
139 try {
140 t1 = fs.getFileStatus(p1).getModificationTime();
141 t2 = fs.getFileStatus(p2).getModificationTime();
142 } catch (IOException ex) {
143 throw new RuntimeException(ex);
144 }
145 return t1 < t2 ? -1 : t1 == t2 ? 1 : 0;
146 }
147 });
148
149 StoreFile.Reader lastStoreFileReader = null;
150 for (StoreFileScanner sfScanner : scanners)
151 lastStoreFileReader = sfScanner.getReader();
152
153 new HFilePrettyPrinter(conf).run(new String[]{ "-m", "-p", "-f",
154 lastStoreFileReader.getHFileReader().getPath().toString()});
155
156
157
158 LOG.info("Disabling Bloom filter for: "
159 + lastStoreFileReader.getHFileReader().getName());
160 lastStoreFileReader.disableBloomFilterForTesting();
161
162 List<Cell> allResults = new ArrayList<Cell>();
163
164 {
165 List<Cell> results = new ArrayList<Cell>();
166 while (scanner.next(results) || results.size() > 0) {
167 allResults.addAll(results);
168 results.clear();
169 }
170 }
171
172 List<Integer> actualIds = new ArrayList<Integer>();
173 for (Cell kv : allResults) {
174 String qual = Bytes.toString(CellUtil.cloneQualifier(kv));
175 assertTrue(qual.startsWith(QUALIFIER_PREFIX));
176 actualIds.add(Integer.valueOf(qual.substring(
177 QUALIFIER_PREFIX.length())));
178 }
179 List<Integer> expectedIds = new ArrayList<Integer>();
180 for (int expectedId : expectedResultCols)
181 expectedIds.add(expectedId);
182
183 LOG.info("Column ids returned: " + actualIds + ", expected: "
184 + expectedIds);
185 assertEquals(expectedIds.toString(), actualIds.toString());
186 }
187
188 private void addColumnSetToScan(Scan scan, int[] colIds) {
189 for (int colId : colIds) {
190 scan.addColumn(FAMILY_BYTES,
191 Bytes.toBytes(qualFromId(colId)));
192 }
193 }
194
195 private String qualFromId(int colId) {
196 return QUALIFIER_PREFIX + colId;
197 }
198
199 private void createStoreFile(int[] colIds)
200 throws IOException {
201 Put p = new Put(ROW_BYTES);
202 for (int colId : colIds) {
203 long ts = Long.MAX_VALUE;
204 String qual = qualFromId(colId);
205 allColIds.add(colId);
206 KeyValue kv = KeyValueTestUtil.create(ROW, FAMILY,
207 qual, ts, TestMultiColumnScanner.createValue(ROW, qual, ts));
208 p.add(kv);
209 }
210 region.put(p);
211 region.flush(true);
212 }
213
214
215 }
216