1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.regionserver;
19
20 import java.io.IOException;
21 import java.util.ArrayList;
22 import java.util.Arrays;
23 import java.util.List;
24
25 import junit.framework.TestCase;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.fs.FileSystem;
31 import org.apache.hadoop.fs.Path;
32 import org.apache.hadoop.hbase.HBaseTestingUtility;
33 import org.apache.hadoop.hbase.HColumnDescriptor;
34 import org.apache.hadoop.hbase.HConstants;
35 import org.apache.hadoop.hbase.HRegionInfo;
36 import org.apache.hadoop.hbase.HTableDescriptor;
37 import org.apache.hadoop.hbase.SmallTests;
38 import org.apache.hadoop.hbase.TableName;
39 import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
40 import org.apache.hadoop.hbase.regionserver.compactions.RatioBasedCompactionPolicy;
41 import org.apache.hadoop.hbase.regionserver.wal.HLog;
42 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
43 import org.apache.hadoop.hbase.util.Bytes;
44 import org.apache.hadoop.hbase.util.FSUtils;
45 import org.junit.After;
46 import org.junit.experimental.categories.Category;
47
48 import com.google.common.collect.Lists;
49
50 @Category(SmallTests.class)
51 public class TestDefaultCompactSelection extends TestCase {
52 private final static Log LOG = LogFactory.getLog(TestDefaultCompactSelection.class);
53 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
54
55 protected Configuration conf;
56 protected HStore store;
57 private static final String DIR=
58 TEST_UTIL.getDataTestDir(TestDefaultCompactSelection.class.getSimpleName()).toString();
59 private static Path TEST_FILE;
60
61 protected static final int minFiles = 3;
62 protected static final int maxFiles = 5;
63
64 protected static final long minSize = 10;
65 protected static final long maxSize = 2100;
66
67 private HLog hlog;
68 private HRegion region;
69
70 @Override
71 public void setUp() throws Exception {
72
73 this.conf = TEST_UTIL.getConfiguration();
74 this.conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 0);
75 this.conf.setInt("hbase.hstore.compaction.min", minFiles);
76 this.conf.setInt("hbase.hstore.compaction.max", maxFiles);
77 this.conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, minSize);
78 this.conf.setLong("hbase.hstore.compaction.max.size", maxSize);
79 this.conf.setFloat("hbase.hstore.compaction.ratio", 1.0F);
80
81
82 Path basedir = new Path(DIR);
83 String logName = "logs";
84 Path logdir = new Path(DIR, logName);
85 HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes("family"));
86 FileSystem fs = FileSystem.get(conf);
87
88 fs.delete(logdir, true);
89
90 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(Bytes.toBytes("table")));
91 htd.addFamily(hcd);
92 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
93
94 hlog = HLogFactory.createHLog(fs, basedir, logName, conf);
95 region = HRegion.createHRegion(info, basedir, conf, htd);
96 HRegion.closeHRegion(region);
97 Path tableDir = FSUtils.getTableDir(basedir, htd.getTableName());
98 region = new HRegion(tableDir, hlog, fs, conf, info, htd, null);
99
100 store = new HStore(region, hcd, conf);
101
102 TEST_FILE = region.getRegionFileSystem().createTempName();
103 fs.createNewFile(TEST_FILE);
104 }
105
106 @After
107 public void tearDown() throws IOException {
108 IOException ex = null;
109 try {
110 region.close();
111 } catch (IOException e) {
112 LOG.warn("Caught Exception", e);
113 ex = e;
114 }
115 try {
116 hlog.closeAndDelete();
117 } catch (IOException e) {
118 LOG.warn("Caught Exception", e);
119 ex = e;
120 }
121 if (ex != null) {
122 throw ex;
123 }
124 }
125
126 ArrayList<Long> toArrayList(long... numbers) {
127 ArrayList<Long> result = new ArrayList<Long>();
128 for (long i : numbers) {
129 result.add(i);
130 }
131 return result;
132 }
133
134 List<StoreFile> sfCreate(long... sizes) throws IOException {
135 ArrayList<Long> ageInDisk = new ArrayList<Long>();
136 for (int i = 0; i < sizes.length; i++) {
137 ageInDisk.add(0L);
138 }
139 return sfCreate(toArrayList(sizes), ageInDisk);
140 }
141
142 List<StoreFile> sfCreate(ArrayList<Long> sizes, ArrayList<Long> ageInDisk)
143 throws IOException {
144 return sfCreate(false, sizes, ageInDisk);
145 }
146
147 List<StoreFile> sfCreate(boolean isReference, long... sizes) throws IOException {
148 ArrayList<Long> ageInDisk = new ArrayList<Long>(sizes.length);
149 for (int i = 0; i < sizes.length; i++) {
150 ageInDisk.add(0L);
151 }
152 return sfCreate(isReference, toArrayList(sizes), ageInDisk);
153 }
154
155 List<StoreFile> sfCreate(boolean isReference, ArrayList<Long> sizes, ArrayList<Long> ageInDisk)
156 throws IOException {
157 List<StoreFile> ret = Lists.newArrayList();
158 for (int i = 0; i < sizes.size(); i++) {
159 ret.add(new MockStoreFile(TEST_UTIL, TEST_FILE,
160 sizes.get(i), ageInDisk.get(i), isReference, i));
161 }
162 return ret;
163 }
164
165 long[] getSizes(List<StoreFile> sfList) {
166 long[] aNums = new long[sfList.size()];
167 for (int i = 0; i < sfList.size(); ++i) {
168 aNums[i] = sfList.get(i).getReader().length();
169 }
170 return aNums;
171 }
172
173 void compactEquals(List<StoreFile> candidates, long... expected)
174 throws IOException {
175 compactEquals(candidates, false, false, expected);
176 }
177
178 void compactEquals(List<StoreFile> candidates, boolean forcemajor, long... expected)
179 throws IOException {
180 compactEquals(candidates, forcemajor, false, expected);
181 }
182
183 void compactEquals(List<StoreFile> candidates, boolean forcemajor, boolean isOffPeak,
184 long ... expected)
185 throws IOException {
186 store.forceMajor = forcemajor;
187
188 CompactionRequest result = ((RatioBasedCompactionPolicy)store.storeEngine.getCompactionPolicy())
189 .selectCompaction(candidates, new ArrayList<StoreFile>(), false, isOffPeak, forcemajor);
190 List<StoreFile> actual = new ArrayList<StoreFile>(result.getFiles());
191 if (isOffPeak && !forcemajor) {
192 assertTrue(result.isOffPeak());
193 }
194 assertEquals(Arrays.toString(expected), Arrays.toString(getSizes(actual)));
195 store.forceMajor = false;
196 }
197
198 public void testCompactionRatio() throws IOException {
199
200
201
202
203
204 long tooBig = maxSize + 1;
205
206
207 compactEquals(sfCreate(100,50,23,12,12), 23, 12, 12);
208
209 compactEquals(sfCreate(100,50,25,12,12)
210
211 compactEquals(sfCreate(tooBig, tooBig, 700, 700, 700), 700, 700, 700);
212
213 compactEquals(sfCreate(tooBig, tooBig, 700,700)
214
215 compactEquals(sfCreate(7,1,1), 7,1,1);
216
217
218
219 compactEquals(sfCreate(7, 6, 5, 4, 3, 2, 1), 5, 4, 3, 2, 1);
220
221 compactEquals(sfCreate(50, 10, 10 ,10, 10), 10, 10, 10, 10);
222
223 compactEquals(sfCreate(10, 10, 10, 10, 50), 10, 10, 10, 10);
224
225 compactEquals(sfCreate(251, 253, 251, maxSize -1), 251, 253, 251);
226
227 compactEquals(sfCreate(maxSize -1,maxSize -1,maxSize -1)
228
229
230 this.conf.setLong("hbase.hstore.compaction.min.size", 1);
231 store.storeEngine.getCompactionPolicy().setConf(conf);
232 compactEquals(sfCreate(512,256,128,64,32,16,8,4,2,1), 4,2,1);
233 this.conf.setLong("hbase.hstore.compaction.min.size", minSize);
234 store.storeEngine.getCompactionPolicy().setConf(conf);
235
236
237
238 compactEquals(sfCreate(50,25,12,12), true, 50, 25, 12, 12);
239
240 compactEquals(sfCreate(12,12), true, 12, 12);
241
242 compactEquals(sfCreate(tooBig, 12,12), true, tooBig, 12, 12);
243
244 store.forceMajor = true;
245 compactEquals(sfCreate(7, 6, 5, 4, 3, 2, 1), 5, 4, 3, 2, 1);
246 store.forceMajor = false;
247
248
249
250 compactEquals(sfCreate(100,50,23,12,12), true, 23, 12, 12);
251 conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 1);
252 conf.setFloat("hbase.hregion.majorcompaction.jitter", 0);
253 store.storeEngine.getCompactionPolicy().setConf(conf);
254 try {
255
256 compactEquals(sfCreate(50,25,12,12), 50, 25, 12, 12);
257
258 compactEquals(sfCreate(100,50,23,12,12), 23, 12, 12);
259 } finally {
260 conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 1000*60*60*24);
261 conf.setFloat("hbase.hregion.majorcompaction.jitter", 0.20F);
262 }
263
264
265
266 compactEquals(sfCreate(true, 100,50,25,12,12), 100, 50, 25, 12, 12);
267
268 compactEquals(sfCreate(true, tooBig, 12,12), tooBig, 12, 12);
269
270 compactEquals(sfCreate(true, 7, 6, 5, 4, 3, 2, 1), 7, 6, 5, 4, 3);
271
272
273 compactEquals(new ArrayList<StoreFile>()
274
275 compactEquals(sfCreate(tooBig, tooBig)
276 }
277
278 public void testOffPeakCompactionRatio() throws IOException {
279
280
281
282
283
284
285 this.conf.setFloat("hbase.hstore.compaction.ratio.offpeak", 5.0F);
286 store.storeEngine.getCompactionPolicy().setConf(this.conf);
287
288 compactEquals(sfCreate(999, 50, 12, 12, 1), false, true, 50, 12, 12, 1);
289 compactEquals(sfCreate(999, 50, 12, 12, 1), 12, 12, 1);
290 }
291
292 public void testStuckStoreCompaction() throws IOException {
293
294 compactEquals(sfCreate(99,99,99,99,99,99, 30,30,30,30), 30, 30, 30);
295
296 compactEquals(sfCreate(99,99,99,99,99, 30,30,30,30), 99, 30, 30, 30, 30);
297
298
299 compactEquals(sfCreate(99,99,99,99,99,99, 30,30,30,15), 30, 30, 30, 15);
300
301 compactEquals(sfCreate(99,99,99,99, 30,26,26,29,25,25), 30, 26, 26);
302
303 compactEquals(sfCreate(99,99,99,99, 27,27,27,20,20,20), 20, 20, 20);
304 }
305
306 public void testCompactionEmptyHFile() throws IOException {
307
308 ScanInfo oldScanInfo = store.getScanInfo();
309 ScanInfo newScanInfo = new ScanInfo(oldScanInfo.getFamily(),
310 oldScanInfo.getMinVersions(), oldScanInfo.getMaxVersions(), 600,
311 oldScanInfo.getKeepDeletedCells(), oldScanInfo.getTimeToPurgeDeletes(),
312 oldScanInfo.getComparator());
313 store.setScanInfo(newScanInfo);
314
315 List<StoreFile> candidates = sfCreate(0);
316 for (StoreFile file : candidates) {
317 if (file instanceof MockStoreFile) {
318 MockStoreFile mockFile = (MockStoreFile) file;
319 mockFile.setTimeRangeTracker(new TimeRangeTracker(-1, -1));
320 mockFile.setEntries(0);
321 }
322 }
323
324 CompactionRequest result = ((RatioBasedCompactionPolicy) store.storeEngine
325 .getCompactionPolicy()).selectCompaction(candidates,
326 new ArrayList<StoreFile>(), false, false, false);
327 assertTrue(result.getFiles().size() == 0);
328 store.setScanInfo(oldScanInfo);
329 }
330 }