1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.io.hfile;
19
20 import java.io.IOException;
21 import java.lang.management.ManagementFactory;
22 import java.lang.management.MemoryUsage;
23
24 import org.apache.commons.logging.Log;
25 import org.apache.commons.logging.LogFactory;
26 import org.apache.hadoop.classification.InterfaceAudience;
27 import org.apache.hadoop.conf.Configuration;
28 import org.apache.hadoop.hbase.HColumnDescriptor;
29 import org.apache.hadoop.hbase.HConstants;
30 import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
31 import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
32 import org.apache.hadoop.hbase.regionserver.StoreFile;
33 import org.apache.hadoop.hbase.util.DirectMemoryUtils;
34 import org.apache.hadoop.util.StringUtils;
35
36
37
38
39 @InterfaceAudience.Private
40 public class CacheConfig {
41 private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName());
42
43
44
45
46
47 public static final String CACHE_BLOCKS_ON_WRITE_KEY =
48 "hbase.rs.cacheblocksonwrite";
49
50
51
52
53
54 public static final String CACHE_INDEX_BLOCKS_ON_WRITE_KEY =
55 "hfile.block.index.cacheonwrite";
56
57
58
59
60 public static final String CACHE_BLOOM_BLOCKS_ON_WRITE_KEY =
61 "hfile.block.bloom.cacheonwrite";
62
63
64
65
66
67 public static final String CACHE_DATA_BLOCKS_COMPRESSED_KEY =
68 "hbase.rs.blockcache.cachedatacompressed";
69
70
71
72
73
74 public static final String EVICT_BLOCKS_ON_CLOSE_KEY =
75 "hbase.rs.evictblocksonclose";
76
77
78
79
80 public static final String BUCKET_CACHE_IOENGINE_KEY = "hbase.bucketcache.ioengine";
81 public static final String BUCKET_CACHE_SIZE_KEY = "hbase.bucketcache.size";
82 public static final String BUCKET_CACHE_PERSISTENT_PATH_KEY =
83 "hbase.bucketcache.persistent.path";
84 public static final String BUCKET_CACHE_COMBINED_KEY =
85 "hbase.bucketcache.combinedcache.enabled";
86 public static final String BUCKET_CACHE_COMBINED_PERCENTAGE_KEY =
87 "hbase.bucketcache.percentage.in.combinedcache";
88 public static final String BUCKET_CACHE_WRITER_THREADS_KEY = "hbase.bucketcache.writer.threads";
89 public static final String BUCKET_CACHE_WRITER_QUEUE_KEY =
90 "hbase.bucketcache.writer.queuelength";
91
92
93
94
95 public static final String BUCKET_CACHE_BUCKETS_KEY = "hbase.bucketcache.bucket.sizes";
96
97
98
99
100 public static final boolean DEFAULT_BUCKET_CACHE_COMBINED = true;
101 public static final int DEFAULT_BUCKET_CACHE_WRITER_THREADS = 3;
102 public static final int DEFAULT_BUCKET_CACHE_WRITER_QUEUE = 64;
103 public static final float DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE = 0.9f;
104
105
106
107
108
109 public static final String PREFETCH_BLOCKS_ON_OPEN_KEY =
110 "hbase.rs.prefetchblocksonopen";
111
112
113
114 public static final boolean DEFAULT_CACHE_DATA_ON_READ = true;
115 public static final boolean DEFAULT_CACHE_DATA_ON_WRITE = false;
116 public static final boolean DEFAULT_IN_MEMORY = false;
117 public static final boolean DEFAULT_CACHE_INDEXES_ON_WRITE = false;
118 public static final boolean DEFAULT_CACHE_BLOOMS_ON_WRITE = false;
119 public static final boolean DEFAULT_EVICT_ON_CLOSE = false;
120 public static final boolean DEFAULT_COMPRESSED_CACHE = false;
121 public static final boolean DEFAULT_PREFETCH_ON_OPEN = false;
122
123
124 private final BlockCache blockCache;
125
126
127
128
129
130 private boolean cacheDataOnRead;
131
132
133 private final boolean inMemory;
134
135
136 private boolean cacheDataOnWrite;
137
138
139 private final boolean cacheIndexesOnWrite;
140
141
142 private final boolean cacheBloomsOnWrite;
143
144
145 private boolean evictOnClose;
146
147
148 private final boolean cacheCompressed;
149
150
151 private final boolean prefetchOnOpen;
152
153
154
155
156
157
158
159 public CacheConfig(Configuration conf, HColumnDescriptor family) {
160 this(CacheConfig.instantiateBlockCache(conf),
161 family.isBlockCacheEnabled(),
162 family.isInMemory(),
163
164
165 conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY,
166 DEFAULT_CACHE_DATA_ON_WRITE) || family.shouldCacheDataOnWrite(),
167 conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
168 DEFAULT_CACHE_INDEXES_ON_WRITE) || family.shouldCacheIndexesOnWrite(),
169 conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
170 DEFAULT_CACHE_BLOOMS_ON_WRITE) || family.shouldCacheBloomsOnWrite(),
171 conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY,
172 DEFAULT_EVICT_ON_CLOSE) || family.shouldEvictBlocksOnClose(),
173 conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_COMPRESSED_CACHE),
174 conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY,
175 DEFAULT_PREFETCH_ON_OPEN) || family.shouldPrefetchBlocksOnOpen()
176 );
177 }
178
179
180
181
182
183
184 public CacheConfig(Configuration conf) {
185 this(CacheConfig.instantiateBlockCache(conf),
186 DEFAULT_CACHE_DATA_ON_READ,
187 DEFAULT_IN_MEMORY,
188
189 conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE),
190 conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
191 DEFAULT_CACHE_INDEXES_ON_WRITE),
192 conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
193 DEFAULT_CACHE_BLOOMS_ON_WRITE),
194 conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE),
195 conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY,
196 DEFAULT_COMPRESSED_CACHE),
197 conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, DEFAULT_PREFETCH_ON_OPEN)
198 );
199 }
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214 CacheConfig(final BlockCache blockCache,
215 final boolean cacheDataOnRead, final boolean inMemory,
216 final boolean cacheDataOnWrite, final boolean cacheIndexesOnWrite,
217 final boolean cacheBloomsOnWrite, final boolean evictOnClose,
218 final boolean cacheCompressed, final boolean prefetchOnOpen) {
219 this.blockCache = blockCache;
220 this.cacheDataOnRead = cacheDataOnRead;
221 this.inMemory = inMemory;
222 this.cacheDataOnWrite = cacheDataOnWrite;
223 this.cacheIndexesOnWrite = cacheIndexesOnWrite;
224 this.cacheBloomsOnWrite = cacheBloomsOnWrite;
225 this.evictOnClose = evictOnClose;
226 this.cacheCompressed = cacheCompressed;
227 this.prefetchOnOpen = prefetchOnOpen;
228 }
229
230
231
232
233
234 public CacheConfig(CacheConfig cacheConf) {
235 this(cacheConf.blockCache, cacheConf.cacheDataOnRead, cacheConf.inMemory,
236 cacheConf.cacheDataOnWrite, cacheConf.cacheIndexesOnWrite,
237 cacheConf.cacheBloomsOnWrite, cacheConf.evictOnClose,
238 cacheConf.cacheCompressed, cacheConf.prefetchOnOpen);
239 }
240
241
242
243
244 public boolean isBlockCacheEnabled() {
245 return this.blockCache != null;
246 }
247
248
249
250
251
252 public BlockCache getBlockCache() {
253 return this.blockCache;
254 }
255
256
257
258
259
260 public boolean shouldCacheDataOnRead() {
261 return isBlockCacheEnabled() && cacheDataOnRead;
262 }
263
264
265
266
267
268
269 public boolean shouldCacheBlockOnRead(BlockCategory category) {
270 boolean shouldCache = isBlockCacheEnabled()
271 && (cacheDataOnRead ||
272 category == BlockCategory.INDEX ||
273 category == BlockCategory.BLOOM ||
274 (prefetchOnOpen &&
275 (category != BlockCategory.META &&
276 category != BlockCategory.UNKNOWN)));
277 return shouldCache;
278 }
279
280
281
282
283 public boolean isInMemory() {
284 return isBlockCacheEnabled() && this.inMemory;
285 }
286
287
288
289
290
291 public boolean shouldCacheDataOnWrite() {
292 return isBlockCacheEnabled() && this.cacheDataOnWrite;
293 }
294
295
296
297
298
299
300 public void setCacheDataOnWrite(boolean cacheDataOnWrite) {
301 this.cacheDataOnWrite = cacheDataOnWrite;
302 }
303
304
305
306
307
308 public boolean shouldCacheIndexesOnWrite() {
309 return isBlockCacheEnabled() && this.cacheIndexesOnWrite;
310 }
311
312
313
314
315
316 public boolean shouldCacheBloomsOnWrite() {
317 return isBlockCacheEnabled() && this.cacheBloomsOnWrite;
318 }
319
320
321
322
323
324 public boolean shouldEvictOnClose() {
325 return isBlockCacheEnabled() && this.evictOnClose;
326 }
327
328
329
330
331
332
333 public void setEvictOnClose(boolean evictOnClose) {
334 this.evictOnClose = evictOnClose;
335 }
336
337
338
339
340 public boolean shouldCacheCompressed() {
341 return isBlockCacheEnabled() && this.cacheCompressed;
342 }
343
344
345
346
347 public boolean shouldPrefetchOnOpen() {
348 return isBlockCacheEnabled() && this.prefetchOnOpen;
349 }
350
351 @Override
352 public String toString() {
353 if (!isBlockCacheEnabled()) {
354 return "CacheConfig:disabled";
355 }
356 return "CacheConfig:enabled " +
357 "[cacheDataOnRead=" + shouldCacheDataOnRead() + "] " +
358 "[cacheDataOnWrite=" + shouldCacheDataOnWrite() + "] " +
359 "[cacheIndexesOnWrite=" + shouldCacheIndexesOnWrite() + "] " +
360 "[cacheBloomsOnWrite=" + shouldCacheBloomsOnWrite() + "] " +
361 "[cacheEvictOnClose=" + shouldEvictOnClose() + "] " +
362 "[cacheCompressed=" + shouldCacheCompressed() + "]" +
363 "[prefetchOnOpen=" + shouldPrefetchOnOpen() + "]";
364 }
365
366
367
368
369
370
371
372 private static BlockCache globalBlockCache;
373
374
375 private static boolean blockCacheDisabled = false;
376
377
378
379
380
381
382
383 private static synchronized BlockCache instantiateBlockCache(Configuration conf) {
384 if (globalBlockCache != null) return globalBlockCache;
385 if (blockCacheDisabled) return null;
386
387 float cachePercentage = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
388 HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
389 if (cachePercentage == 0L) {
390 blockCacheDisabled = true;
391 return null;
392 }
393 if (cachePercentage > 1.0) {
394 throw new IllegalArgumentException(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY +
395 " must be between 0.0 and 1.0, and not > 1.0");
396 }
397
398
399 MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
400 long lruCacheSize = (long) (mu.getMax() * cachePercentage);
401 int blockSize = conf.getInt("hbase.offheapcache.minblocksize", HConstants.DEFAULT_BLOCKSIZE);
402 long offHeapCacheSize =
403 (long) (conf.getFloat("hbase.offheapcache.percentage", (float) 0) *
404 DirectMemoryUtils.getDirectMemorySize());
405 if (offHeapCacheSize <= 0) {
406 String bucketCacheIOEngineName = conf.get(BUCKET_CACHE_IOENGINE_KEY, null);
407 float bucketCachePercentage = conf.getFloat(BUCKET_CACHE_SIZE_KEY, 0F);
408
409 long bucketCacheSize = (long) (bucketCachePercentage < 1 ? mu.getMax()
410 * bucketCachePercentage : bucketCachePercentage * 1024 * 1024);
411
412 boolean combinedWithLru = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY,
413 DEFAULT_BUCKET_CACHE_COMBINED);
414 BucketCache bucketCache = null;
415 if (bucketCacheIOEngineName != null && bucketCacheSize > 0) {
416 int writerThreads = conf.getInt(BUCKET_CACHE_WRITER_THREADS_KEY,
417 DEFAULT_BUCKET_CACHE_WRITER_THREADS);
418 int writerQueueLen = conf.getInt(BUCKET_CACHE_WRITER_QUEUE_KEY,
419 DEFAULT_BUCKET_CACHE_WRITER_QUEUE);
420 String persistentPath = conf.get(BUCKET_CACHE_PERSISTENT_PATH_KEY);
421 float combinedPercentage = conf.getFloat(
422 BUCKET_CACHE_COMBINED_PERCENTAGE_KEY,
423 DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE);
424 String[] configuredBucketSizes = conf.getStrings(BUCKET_CACHE_BUCKETS_KEY);
425 int[] bucketSizes = null;
426 if (configuredBucketSizes != null) {
427 bucketSizes = new int[configuredBucketSizes.length];
428 for (int i = 0; i < configuredBucketSizes.length; i++) {
429 bucketSizes[i] = Integer.parseInt(configuredBucketSizes[i]);
430 }
431 }
432 if (combinedWithLru) {
433 lruCacheSize = (long) ((1 - combinedPercentage) * bucketCacheSize);
434 bucketCacheSize = (long) (combinedPercentage * bucketCacheSize);
435 }
436 try {
437 int ioErrorsTolerationDuration = conf.getInt(
438 "hbase.bucketcache.ioengine.errors.tolerated.duration",
439 BucketCache.DEFAULT_ERROR_TOLERATION_DURATION);
440 bucketCache = new BucketCache(bucketCacheIOEngineName,
441 bucketCacheSize, blockSize, bucketSizes, writerThreads, writerQueueLen, persistentPath,
442 ioErrorsTolerationDuration);
443 } catch (IOException ioex) {
444 LOG.error("Can't instantiate bucket cache", ioex);
445 throw new RuntimeException(ioex);
446 }
447 }
448 LOG.info("Allocating LruBlockCache with maximum size " +
449 StringUtils.humanReadableInt(lruCacheSize));
450 LruBlockCache lruCache = new LruBlockCache(lruCacheSize, blockSize, true, conf);
451 lruCache.setVictimCache(bucketCache);
452 if (bucketCache != null && combinedWithLru) {
453 globalBlockCache = new CombinedBlockCache(lruCache, bucketCache);
454 } else {
455 globalBlockCache = lruCache;
456 }
457 } else {
458 LOG.warn("SlabCache is deprecated. Consider BucketCache as a replacement.");
459 globalBlockCache = new DoubleBlockCache(
460 lruCacheSize, offHeapCacheSize, blockSize, blockSize, conf);
461 }
462 return globalBlockCache;
463 }
464 }