1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.mapreduce;
20
21 import static org.junit.Assert.assertArrayEquals;
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertTrue;
24 import static org.junit.Assert.fail;
25
26 import java.io.IOException;
27 import java.util.TreeMap;
28
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.fs.FileSystem;
31 import org.apache.hadoop.fs.Path;
32 import org.apache.hadoop.hbase.HBaseTestingUtility;
33 import org.apache.hadoop.hbase.HColumnDescriptor;
34 import org.apache.hadoop.hbase.HTableDescriptor;
35 import org.apache.hadoop.hbase.LargeTests;
36 import org.apache.hadoop.hbase.TableName;
37 import org.apache.hadoop.hbase.client.HBaseAdmin;
38 import org.apache.hadoop.hbase.client.HTable;
39 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
40 import org.apache.hadoop.hbase.io.hfile.HFile;
41 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
42 import org.apache.hadoop.hbase.regionserver.BloomType;
43 import org.apache.hadoop.hbase.util.Bytes;
44 import org.apache.hadoop.hbase.util.HFileTestUtil;
45 import org.junit.AfterClass;
46 import org.junit.BeforeClass;
47 import org.junit.Test;
48 import org.junit.experimental.categories.Category;
49
50
51
52
53
54
55 @Category(LargeTests.class)
56 public class TestLoadIncrementalHFiles {
57 private static final byte[] QUALIFIER = Bytes.toBytes("myqual");
58 private static final byte[] FAMILY = Bytes.toBytes("myfam");
59 static final String EXPECTED_MSG_FOR_NON_EXISTING_FAMILY = "Unmatched family names found";
60 static final int MAX_FILES_PER_REGION_PER_FAMILY = 4;
61
62 private static final byte[][] SPLIT_KEYS = new byte[][] {
63 Bytes.toBytes("ddd"),
64 Bytes.toBytes("ppp")
65 };
66
67 static HBaseTestingUtility util = new HBaseTestingUtility();
68
69 @BeforeClass
70 public static void setUpBeforeClass() throws Exception {
71 util.getConfiguration().setInt(
72 LoadIncrementalHFiles.MAX_FILES_PER_REGION_PER_FAMILY,
73 MAX_FILES_PER_REGION_PER_FAMILY);
74 util.startMiniCluster();
75 }
76
77 @AfterClass
78 public static void tearDownAfterClass() throws Exception {
79 util.shutdownMiniCluster();
80 }
81
82
83
84
85
86 @Test
87 public void testSimpleLoad() throws Exception {
88 runTest("testSimpleLoad", BloomType.NONE,
89 new byte[][][] {
90 new byte[][]{ Bytes.toBytes("aaaa"), Bytes.toBytes("cccc") },
91 new byte[][]{ Bytes.toBytes("ddd"), Bytes.toBytes("ooo") },
92 });
93 }
94
95
96
97
98
99 @Test
100 public void testRegionCrossingLoad() throws Exception {
101 runTest("testRegionCrossingLoad", BloomType.NONE,
102 new byte[][][] {
103 new byte[][]{ Bytes.toBytes("aaaa"), Bytes.toBytes("eee") },
104 new byte[][]{ Bytes.toBytes("fff"), Bytes.toBytes("zzz") },
105 });
106 }
107
108
109
110
111 @Test
112 public void testRegionCrossingRowBloom() throws Exception {
113 runTest("testRegionCrossingLoadRowBloom", BloomType.ROW,
114 new byte[][][] {
115 new byte[][]{ Bytes.toBytes("aaaa"), Bytes.toBytes("eee") },
116 new byte[][]{ Bytes.toBytes("fff"), Bytes.toBytes("zzz") },
117 });
118 }
119
120
121
122
123 @Test
124 public void testRegionCrossingRowColBloom() throws Exception {
125 runTest("testRegionCrossingLoadRowColBloom", BloomType.ROWCOL,
126 new byte[][][] {
127 new byte[][]{ Bytes.toBytes("aaaa"), Bytes.toBytes("eee") },
128 new byte[][]{ Bytes.toBytes("fff"), Bytes.toBytes("zzz") },
129 });
130 }
131
132 private void runTest(String testName, BloomType bloomType,
133 byte[][][] hfileRanges) throws Exception {
134 Path dir = util.getDataTestDirOnTestFS(testName);
135 FileSystem fs = util.getTestFileSystem();
136 dir = dir.makeQualified(fs);
137 Path familyDir = new Path(dir, Bytes.toString(FAMILY));
138
139 int hfileIdx = 0;
140 for (byte[][] range : hfileRanges) {
141 byte[] from = range[0];
142 byte[] to = range[1];
143 HFileTestUtil.createHFile(util.getConfiguration(), fs, new Path(familyDir, "hfile_"
144 + hfileIdx++), FAMILY, QUALIFIER, from, to, 1000);
145 }
146 int expectedRows = hfileIdx * 1000;
147
148 final byte[] TABLE = Bytes.toBytes("mytable_"+testName);
149
150 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
151 HColumnDescriptor familyDesc = new HColumnDescriptor(FAMILY);
152 familyDesc.setBloomFilterType(bloomType);
153 htd.addFamily(familyDesc);
154
155 LoadIncrementalHFiles loader = new LoadIncrementalHFiles(util.getConfiguration());
156 String [] args= {dir.toString(),"mytable_"+testName};
157 loader.run(args);
158 HTable table = new HTable(util.getConfiguration(), TABLE);
159
160 assertEquals(expectedRows, util.countRows(table));
161 }
162
163
164
165
166 @Test
167 public void testNonexistentColumnFamilyLoad() throws Exception {
168 String testName = "testNonexistentColumnFamilyLoad";
169 byte[][][] hFileRanges = new byte[][][] {
170 new byte[][]{ Bytes.toBytes("aaa"), Bytes.toBytes("ccc") },
171 new byte[][]{ Bytes.toBytes("ddd"), Bytes.toBytes("ooo") },
172 };
173
174 Path dir = util.getDataTestDirOnTestFS(testName);
175 FileSystem fs = util.getTestFileSystem();
176 dir = dir.makeQualified(fs);
177 Path familyDir = new Path(dir, Bytes.toString(FAMILY));
178
179 int hFileIdx = 0;
180 for (byte[][] range : hFileRanges) {
181 byte[] from = range[0];
182 byte[] to = range[1];
183 HFileTestUtil.createHFile(util.getConfiguration(), fs, new Path(familyDir, "hfile_"
184 + hFileIdx++), FAMILY, QUALIFIER, from, to, 1000);
185 }
186
187 final byte[] TABLE = Bytes.toBytes("mytable_"+testName);
188
189 HBaseAdmin admin = new HBaseAdmin(util.getConfiguration());
190 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
191
192
193 HColumnDescriptor family =
194 new HColumnDescriptor(Bytes.toBytes(new String(FAMILY).toUpperCase()));
195 htd.addFamily(family);
196 admin.createTable(htd, SPLIT_KEYS);
197
198 HTable table = new HTable(util.getConfiguration(), TABLE);
199 util.waitTableEnabled(TABLE);
200 LoadIncrementalHFiles loader = new LoadIncrementalHFiles(util.getConfiguration());
201 try {
202 loader.doBulkLoad(dir, table);
203 assertTrue("Loading into table with non-existent family should have failed", false);
204 } catch (Exception e) {
205 assertTrue("IOException expected", e instanceof IOException);
206
207 String errMsg = e.getMessage();
208 assertTrue("Incorrect exception message, expected message: ["
209 + EXPECTED_MSG_FOR_NON_EXISTING_FAMILY + "], current message: [" + errMsg + "]",
210 errMsg.contains(EXPECTED_MSG_FOR_NON_EXISTING_FAMILY));
211 }
212 table.close();
213 admin.close();
214 }
215
216 @Test
217 public void testSplitStoreFile() throws IOException {
218 Path dir = util.getDataTestDirOnTestFS("testSplitHFile");
219 FileSystem fs = util.getTestFileSystem();
220 Path testIn = new Path(dir, "testhfile");
221 HColumnDescriptor familyDesc = new HColumnDescriptor(FAMILY);
222 HFileTestUtil.createHFile(util.getConfiguration(), fs, testIn, FAMILY, QUALIFIER,
223 Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), 1000);
224
225 Path bottomOut = new Path(dir, "bottom.out");
226 Path topOut = new Path(dir, "top.out");
227
228 LoadIncrementalHFiles.splitStoreFile(
229 util.getConfiguration(), testIn,
230 familyDesc, Bytes.toBytes("ggg"),
231 bottomOut,
232 topOut);
233
234 int rowCount = verifyHFile(bottomOut);
235 rowCount += verifyHFile(topOut);
236 assertEquals(1000, rowCount);
237 }
238
239 private int verifyHFile(Path p) throws IOException {
240 Configuration conf = util.getConfiguration();
241 HFile.Reader reader = HFile.createReader(
242 p.getFileSystem(conf), p, new CacheConfig(conf), conf);
243 reader.loadFileInfo();
244 HFileScanner scanner = reader.getScanner(false, false);
245 scanner.seekTo();
246 int count = 0;
247 do {
248 count++;
249 } while (scanner.next());
250 assertTrue(count > 0);
251 reader.close();
252 return count;
253 }
254
255 private void addStartEndKeysForTest(TreeMap<byte[], Integer> map, byte[] first, byte[] last) {
256 Integer value = map.containsKey(first)?map.get(first):0;
257 map.put(first, value+1);
258
259 value = map.containsKey(last)?map.get(last):0;
260 map.put(last, value-1);
261 }
262
263 @Test
264 public void testInferBoundaries() {
265 TreeMap<byte[], Integer> map = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
266
267
268
269
270
271
272
273
274
275
276
277 String first;
278 String last;
279
280 first = "a"; last = "e";
281 addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
282
283 first = "r"; last = "s";
284 addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
285
286 first = "o"; last = "p";
287 addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
288
289 first = "g"; last = "k";
290 addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
291
292 first = "v"; last = "x";
293 addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
294
295 first = "c"; last = "i";
296 addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
297
298 first = "m"; last = "q";
299 addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
300
301 first = "s"; last = "t";
302 addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
303
304 first = "u"; last = "w";
305 addStartEndKeysForTest(map, first.getBytes(), last.getBytes());
306
307 byte[][] keysArray = LoadIncrementalHFiles.inferBoundaries(map);
308 byte[][] compare = new byte[3][];
309 compare[0] = "m".getBytes();
310 compare[1] = "r".getBytes();
311 compare[2] = "u".getBytes();
312
313 assertEquals(keysArray.length, 3);
314
315 for (int row = 0; row<keysArray.length; row++){
316 assertArrayEquals(keysArray[row], compare[row]);
317 }
318 }
319
320 @Test
321 public void testLoadTooMayHFiles() throws Exception {
322 Path dir = util.getDataTestDirOnTestFS("testLoadTooMayHFiles");
323 FileSystem fs = util.getTestFileSystem();
324 dir = dir.makeQualified(fs);
325 Path familyDir = new Path(dir, Bytes.toString(FAMILY));
326
327 byte[] from = Bytes.toBytes("begin");
328 byte[] to = Bytes.toBytes("end");
329 for (int i = 0; i <= MAX_FILES_PER_REGION_PER_FAMILY; i++) {
330 HFileTestUtil.createHFile(util.getConfiguration(), fs, new Path(familyDir, "hfile_"
331 + i), FAMILY, QUALIFIER, from, to, 1000);
332 }
333
334 LoadIncrementalHFiles loader = new LoadIncrementalHFiles(util.getConfiguration());
335 String [] args= {dir.toString(), "mytable_testLoadTooMayHFiles"};
336 try {
337 loader.run(args);
338 fail("Bulk loading too many files should fail");
339 } catch (IOException ie) {
340 assertTrue(ie.getMessage().contains("Trying to load more than "
341 + MAX_FILES_PER_REGION_PER_FAMILY + " hfiles"));
342 }
343 }
344 }
345