1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.hadoop.hbase.regionserver;
18
19 import java.io.IOException;
20 import java.util.ArrayList;
21 import java.util.List;
22 import java.util.Random;
23
24 import org.apache.hadoop.conf.Configuration;
25 import org.apache.hadoop.fs.Path;
26 import org.apache.hadoop.hbase.HBaseTestingUtility;
27 import org.apache.hadoop.hbase.HConstants;
28 import org.apache.hadoop.hbase.KeyValue;
29 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
30 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
31 import org.apache.hadoop.hbase.io.hfile.LruBlockCache;
32
33
34
35
36
37 public class EncodedSeekPerformanceTest {
38 private static final double NANOSEC_IN_SEC = 1000.0 * 1000.0 * 1000.0;
39 private static final double BYTES_IN_MEGABYTES = 1024.0 * 1024.0;
40
41 public static int DEFAULT_NUMBER_OF_SEEKS = 10000;
42
43 private final HBaseTestingUtility testingUtility = new HBaseTestingUtility();
44 private Configuration configuration = testingUtility.getConfiguration();
45 private CacheConfig cacheConf = new CacheConfig(configuration);
46 private Random randomizer;
47 private int numberOfSeeks;
48
49
50 public EncodedSeekPerformanceTest() {
51 configuration.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.5f);
52 randomizer = new Random(42l);
53 numberOfSeeks = DEFAULT_NUMBER_OF_SEEKS;
54 }
55
56 private List<KeyValue> prepareListOfTestSeeks(Path path) throws IOException {
57 List<KeyValue> allKeyValues = new ArrayList<KeyValue>();
58
59
60 StoreFile storeFile = new StoreFile(testingUtility.getTestFileSystem(),
61 path, configuration, cacheConf, BloomType.NONE);
62
63 StoreFile.Reader reader = storeFile.createReader();
64 StoreFileScanner scanner = reader.getStoreFileScanner(true, false);
65 KeyValue current;
66
67 scanner.seek(KeyValue.LOWESTKEY);
68 while (null != (current = scanner.next())) {
69 allKeyValues.add(current);
70 }
71
72 storeFile.closeReader(cacheConf.shouldEvictOnClose());
73
74
75 List<KeyValue> seeks = new ArrayList<KeyValue>();
76 for (int i = 0; i < numberOfSeeks; ++i) {
77 KeyValue keyValue = allKeyValues.get(
78 randomizer.nextInt(allKeyValues.size()));
79 seeks.add(keyValue);
80 }
81
82 clearBlockCache();
83
84 return seeks;
85 }
86
87 private void runTest(Path path, DataBlockEncoding blockEncoding,
88 List<KeyValue> seeks) throws IOException {
89
90 StoreFile storeFile = new StoreFile(testingUtility.getTestFileSystem(),
91 path, configuration, cacheConf, BloomType.NONE);
92
93 long totalSize = 0;
94
95 StoreFile.Reader reader = storeFile.createReader();
96 StoreFileScanner scanner = reader.getStoreFileScanner(true, false);
97
98 long startReadingTime = System.nanoTime();
99 KeyValue current;
100 scanner.seek(KeyValue.LOWESTKEY);
101 while (null != (current = scanner.next())) {
102 if (current.getLength() < 0) {
103 throw new IOException("Negative KV size: " + current);
104 }
105 totalSize += current.getLength();
106 }
107 long finishReadingTime = System.nanoTime();
108
109
110 long startSeeksTime = System.nanoTime();
111 for (KeyValue keyValue : seeks) {
112 scanner.seek(keyValue);
113 KeyValue toVerify = scanner.next();
114 if (!keyValue.equals(toVerify)) {
115 System.out.println(String.format("KeyValue doesn't match:\n" +
116 "Orig key: %s\n" +
117 "Ret key: %s", keyValue.getKeyString(), toVerify.getKeyString()));
118 break;
119 }
120 }
121 long finishSeeksTime = System.nanoTime();
122 if (finishSeeksTime < startSeeksTime) {
123 throw new AssertionError("Finish time " + finishSeeksTime +
124 " is earlier than start time " + startSeeksTime);
125 }
126
127
128 double readInMbPerSec = (totalSize * NANOSEC_IN_SEC) /
129 (BYTES_IN_MEGABYTES * (finishReadingTime - startReadingTime));
130 double seeksPerSec = (seeks.size() * NANOSEC_IN_SEC) /
131 (finishSeeksTime - startSeeksTime);
132
133 storeFile.closeReader(cacheConf.shouldEvictOnClose());
134 clearBlockCache();
135
136 System.out.println(blockEncoding);
137 System.out.printf(" Read speed: %8.2f (MB/s)\n", readInMbPerSec);
138 System.out.printf(" Seeks per second: %8.2f (#/s)\n", seeksPerSec);
139 System.out.printf(" Total KV size: %d\n", totalSize);
140 }
141
142
143
144
145
146
147 public void runTests(Path path, DataBlockEncoding[] encodings)
148 throws IOException {
149 List<KeyValue> seeks = prepareListOfTestSeeks(path);
150
151 for (DataBlockEncoding blockEncoding : encodings) {
152 runTest(path, blockEncoding, seeks);
153 }
154 }
155
156
157
158
159
160
161 public static void main(final String[] args) throws IOException {
162 if (args.length < 1) {
163 printUsage();
164 System.exit(-1);
165 }
166
167 Path path = new Path(args[0]);
168
169
170 EncodedSeekPerformanceTest utility = new EncodedSeekPerformanceTest();
171 utility.runTests(path, DataBlockEncoding.values());
172
173 System.exit(0);
174 }
175
176 private static void printUsage() {
177 System.out.println("Usage: one argument, name of the HFile");
178 }
179
180 private void clearBlockCache() {
181 ((LruBlockCache) cacheConf.getBlockCache()).clearCache();
182 }
183 }