1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.codec.prefixtree.column;
20
21 import java.io.ByteArrayOutputStream;
22 import java.io.IOException;
23 import java.util.Collection;
24 import java.util.List;
25
26 import org.apache.hadoop.hbase.SmallTests;
27 import org.apache.hadoop.hbase.codec.prefixtree.PrefixTreeBlockMeta;
28 import org.apache.hadoop.hbase.codec.prefixtree.decode.column.ColumnReader;
29 import org.apache.hadoop.hbase.codec.prefixtree.encode.column.ColumnSectionWriter;
30 import org.apache.hadoop.hbase.codec.prefixtree.encode.other.ColumnNodeType;
31 import org.apache.hadoop.hbase.codec.prefixtree.encode.tokenize.Tokenizer;
32 import org.apache.hadoop.hbase.codec.prefixtree.encode.tokenize.TokenizerNode;
33 import org.apache.hadoop.hbase.util.ByteRange;
34 import org.apache.hadoop.hbase.util.ByteRangeUtils;
35 import org.apache.hadoop.hbase.util.Bytes;
36 import org.apache.hadoop.hbase.util.byterange.impl.ByteRangeTreeSet;
37 import org.junit.Assert;
38 import org.junit.Test;
39 import org.junit.experimental.categories.Category;
40 import org.junit.runner.RunWith;
41 import org.junit.runners.Parameterized;
42 import org.junit.runners.Parameterized.Parameters;
43
44 import com.google.common.collect.Lists;
45
46 @Category(SmallTests.class)
47 @RunWith(Parameterized.class)
48 public class TestColumnBuilder {
49
50 @Parameters
51 public static Collection<Object[]> parameters() {
52 return new TestColumnData.InMemory().getAllAsObjectArray();
53 }
54
55
56
57 protected TestColumnData columns;
58 protected ByteRangeTreeSet columnSorter;
59 protected List<ByteRange> sortedUniqueColumns;
60 protected PrefixTreeBlockMeta blockMeta;
61 protected Tokenizer builder;
62 protected ColumnSectionWriter writer;
63 protected byte[] bytes;
64 protected byte[] buffer;
65 protected ColumnReader reader;
66
67
68
69 public TestColumnBuilder(TestColumnData columns) {
70 this.columns = columns;
71 List<ByteRange> inputs = columns.getInputs();
72 this.columnSorter = new ByteRangeTreeSet(inputs);
73 this.sortedUniqueColumns = columnSorter.compile().getSortedRanges();
74 List<byte[]> copies = ByteRangeUtils.copyToNewArrays(sortedUniqueColumns);
75 Assert.assertTrue(Bytes.isSorted(copies));
76 this.blockMeta = new PrefixTreeBlockMeta();
77 this.blockMeta.setNumMetaBytes(0);
78 this.blockMeta.setNumRowBytes(0);
79 this.builder = new Tokenizer();
80 }
81
82
83
84 @Test
85 public void testReaderRoundTrip() throws IOException {
86 for (int i = 0; i < sortedUniqueColumns.size(); ++i) {
87 ByteRange column = sortedUniqueColumns.get(i);
88 builder.addSorted(column);
89 }
90 List<byte[]> builderOutputArrays = builder.getArrays();
91 for (int i = 0; i < builderOutputArrays.size(); ++i) {
92 byte[] inputArray = sortedUniqueColumns.get(i).deepCopyToNewArray();
93 byte[] outputArray = builderOutputArrays.get(i);
94 boolean same = Bytes.equals(inputArray, outputArray);
95 Assert.assertTrue(same);
96 }
97 Assert.assertEquals(sortedUniqueColumns.size(), builderOutputArrays.size());
98
99 writer = new ColumnSectionWriter(blockMeta, builder, ColumnNodeType.QUALIFIER);
100 ByteArrayOutputStream baos = new ByteArrayOutputStream();
101 writer.compile().writeBytes(baos);
102 bytes = baos.toByteArray();
103 buffer = new byte[blockMeta.getMaxQualifierLength()];
104 reader = new ColumnReader(buffer, ColumnNodeType.QUALIFIER);
105 reader.initOnBlock(blockMeta, bytes);
106
107 List<TokenizerNode> builderNodes = Lists.newArrayList();
108 builder.appendNodes(builderNodes, true, true);
109 int i = 0;
110 for (TokenizerNode builderNode : builderNodes) {
111 if (!builderNode.hasOccurrences()) {
112 continue;
113 }
114 Assert.assertEquals(1, builderNode.getNumOccurrences());
115
116 int position = builderNode.getOutputArrayOffset();
117 byte[] output = reader.populateBuffer(position).copyBufferToNewArray();
118 boolean same = Bytes.equals(sortedUniqueColumns.get(i).deepCopyToNewArray(), output);
119 Assert.assertTrue(same);
120 ++i;
121 }
122 }
123
124 }