1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver.compactions;
20
21 import org.apache.commons.logging.Log;
22 import org.apache.commons.logging.LogFactory;
23 import org.apache.hadoop.conf.Configuration;
24 import org.apache.hadoop.hbase.HBaseConfiguration;
25 import org.apache.hadoop.hbase.MediumTests;
26 import org.apache.hadoop.hbase.regionserver.HStore;
27 import org.apache.hadoop.hbase.regionserver.StoreConfigInformation;
28 import org.apache.hadoop.hbase.regionserver.StoreFile;
29 import org.apache.hadoop.hbase.util.ReflectionUtils;
30 import org.junit.Test;
31 import org.junit.experimental.categories.Category;
32 import org.junit.runner.RunWith;
33 import org.junit.runners.Parameterized;
34
35 import java.io.IOException;
36 import java.util.ArrayList;
37 import java.util.Collection;
38 import java.util.List;
39
40 import static org.mockito.Mockito.mock;
41
42 import static org.mockito.Mockito.when;
43
44 @Category(MediumTests.class)
45 @RunWith(Parameterized.class)
46 public class PerfTestCompactionPolicies extends MockStoreFileGenerator {
47
48
49 private static final Log LOG = LogFactory.getLog(PerfTestCompactionPolicies.class);
50
51 private final RatioBasedCompactionPolicy cp;
52 private final StoreFileListGenerator generator;
53 private final HStore store;
54 private Class<? extends StoreFileListGenerator> fileGenClass;
55 private final int max;
56 private final int min;
57 private final float ratio;
58 private long written = 0;
59
60 @Parameterized.Parameters
61 public static Collection<Object[]> data() {
62
63
64
65 Class[] policyClasses = new Class[]{
66 EverythingPolicy.class,
67 RatioBasedCompactionPolicy.class,
68 ExploringCompactionPolicy.class,
69 };
70
71 Class[] fileListGenClasses = new Class[]{
72 ExplicitFileListGenerator.class,
73 ConstantSizeFileListGenerator.class,
74 SemiConstantSizeFileListGenerator.class,
75 GaussianFileListGenerator.class,
76 SinusoidalFileListGenerator.class,
77 SpikyFileListGenerator.class
78 };
79
80 int[] maxFileValues = new int[] {10};
81 int[] minFilesValues = new int[] {3};
82 float[] ratioValues = new float[] {1.2f};
83
84 List<Object[]> params = new ArrayList<Object[]>(
85 maxFileValues.length
86 * minFilesValues.length
87 * fileListGenClasses.length
88 * policyClasses.length);
89
90
91 for (Class policyClass : policyClasses) {
92 for (Class genClass: fileListGenClasses) {
93 for (int maxFile:maxFileValues) {
94 for (int minFile:minFilesValues) {
95 for (float ratio:ratioValues) {
96 params.add(new Object[] {policyClass, genClass, maxFile, minFile, ratio});
97 }
98 }
99 }
100 }
101 }
102
103 return params;
104 }
105
106
107
108
109
110
111
112
113 public PerfTestCompactionPolicies(
114 final Class<? extends CompactionPolicy> cpClass,
115 final Class<? extends StoreFileListGenerator> fileGenClass,
116 final int inMmax,
117 final int inMin,
118 final float inRatio) throws IllegalAccessException, InstantiationException {
119 super(PerfTestCompactionPolicies.class);
120 this.fileGenClass = fileGenClass;
121 this.max = inMmax;
122 this.min = inMin;
123 this.ratio = inRatio;
124
125
126 org.apache.log4j.Logger.getLogger(CompactionConfiguration.class).
127 setLevel(org.apache.log4j.Level.ERROR);
128 org.apache.log4j.Logger.getLogger(RatioBasedCompactionPolicy.class).
129 setLevel(org.apache.log4j.Level.ERROR);
130
131 org.apache.log4j.Logger.getLogger(cpClass).setLevel(org.apache.log4j.Level.ERROR);
132
133
134 Configuration configuration = HBaseConfiguration.create();
135
136
137 configuration.setInt("hbase.hstore.compaction.max", max);
138 configuration.setInt("hbase.hstore.compaction.min", min);
139 configuration.setFloat("hbase.hstore.compaction.ratio", ratio);
140
141 store = createMockStore();
142 this.cp = ReflectionUtils.instantiateWithCustomCtor(cpClass.getName(),
143 new Class[] {Configuration.class, StoreConfigInformation.class },
144 new Object[] {configuration, store });
145
146 this.generator = fileGenClass.newInstance();
147
148 }
149
150 @Test
151 public final void testSelection() throws Exception {
152 long fileDiff = 0;
153 for (List<StoreFile> storeFileList : generator) {
154 List<StoreFile> currentFiles = new ArrayList<StoreFile>(18);
155 for (StoreFile file : storeFileList) {
156 currentFiles.add(file);
157 currentFiles = runIteration(currentFiles);
158 }
159 fileDiff += (storeFileList.size() - currentFiles.size());
160 }
161
162
163 System.out.println(
164 cp.getClass().getSimpleName()
165 + "\t" + fileGenClass.getSimpleName()
166 + "\t" + max
167 + "\t" + min
168 + "\t" + ratio
169 + "\t" + written
170 + "\t" + fileDiff
171 );
172 }
173
174
175 private List<StoreFile> runIteration(List<StoreFile> startingStoreFiles) throws IOException {
176
177 List<StoreFile> storeFiles = new ArrayList<StoreFile>(startingStoreFiles);
178 CompactionRequest req = cp.selectCompaction(
179 storeFiles, new ArrayList<StoreFile>(), false, false, false);
180 long newFileSize = 0;
181
182 Collection<StoreFile> filesToCompact = req.getFiles();
183
184 if (!filesToCompact.isEmpty()) {
185
186 storeFiles = new ArrayList<StoreFile>(storeFiles);
187 storeFiles.removeAll(filesToCompact);
188
189 for (StoreFile storeFile : filesToCompact) {
190 newFileSize += storeFile.getReader().length();
191 }
192
193 storeFiles.add(createMockStoreFileBytes(newFileSize));
194 }
195
196 written += newFileSize;
197 return storeFiles;
198 }
199
200 private HStore createMockStore() {
201 HStore s = mock(HStore.class);
202 when(s.getStoreFileTtl()).thenReturn(Long.MAX_VALUE);
203 when(s.getBlockingFileCount()).thenReturn(7L);
204 return s;
205 }
206
207 }