View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.io.hfile;
19  
20  import java.io.IOException;
21  import java.lang.management.ManagementFactory;
22  import java.lang.management.MemoryUsage;
23  
24  import org.apache.commons.logging.Log;
25  import org.apache.commons.logging.LogFactory;
26  import org.apache.hadoop.classification.InterfaceAudience;
27  import org.apache.hadoop.conf.Configuration;
28  import org.apache.hadoop.hbase.HColumnDescriptor;
29  import org.apache.hadoop.hbase.HConstants;
30  import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
31  import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
32  import org.apache.hadoop.hbase.regionserver.StoreFile;
33  import org.apache.hadoop.hbase.util.DirectMemoryUtils;
34  import org.apache.hadoop.util.StringUtils;
35  
36  /**
37   * Stores all of the cache objects and configuration for a single HFile.
38   */
39  @InterfaceAudience.Private
40  public class CacheConfig {
41    private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName());
42  
43    /**
44     * Configuration key to cache data blocks on write. There are separate
45     * switches for bloom blocks and non-root index blocks.
46     */
47    public static final String CACHE_BLOCKS_ON_WRITE_KEY =
48        "hbase.rs.cacheblocksonwrite";
49  
50    /**
51     * Configuration key to cache leaf and intermediate-level index blocks on
52     * write.
53     */
54    public static final String CACHE_INDEX_BLOCKS_ON_WRITE_KEY =
55        "hfile.block.index.cacheonwrite";
56  
57    /**
58     * Configuration key to cache compound bloom filter blocks on write.
59     */
60    public static final String CACHE_BLOOM_BLOCKS_ON_WRITE_KEY =
61        "hfile.block.bloom.cacheonwrite";
62  
63    /**
64     * TODO: Implement this (jgray)
65     * Configuration key to cache data blocks in compressed format.
66     */
67    public static final String CACHE_DATA_BLOCKS_COMPRESSED_KEY =
68        "hbase.rs.blockcache.cachedatacompressed";
69  
70    /**
71     * Configuration key to evict all blocks of a given file from the block cache
72     * when the file is closed.
73     */
74    public static final String EVICT_BLOCKS_ON_CLOSE_KEY =
75        "hbase.rs.evictblocksonclose";
76  
77    /**
78     * Configuration keys for Bucket cache
79     */
80    public static final String BUCKET_CACHE_IOENGINE_KEY = "hbase.bucketcache.ioengine";
81    public static final String BUCKET_CACHE_SIZE_KEY = "hbase.bucketcache.size";
82    public static final String BUCKET_CACHE_PERSISTENT_PATH_KEY = 
83        "hbase.bucketcache.persistent.path";
84    public static final String BUCKET_CACHE_COMBINED_KEY = 
85        "hbase.bucketcache.combinedcache.enabled";
86    public static final String BUCKET_CACHE_COMBINED_PERCENTAGE_KEY = 
87        "hbase.bucketcache.percentage.in.combinedcache";
88    public static final String BUCKET_CACHE_WRITER_THREADS_KEY = "hbase.bucketcache.writer.threads";
89    public static final String BUCKET_CACHE_WRITER_QUEUE_KEY = 
90        "hbase.bucketcache.writer.queuelength";
91  
92    /**
93     * A comma-delimited array of values for use as bucket sizes.
94     */
95    public static final String BUCKET_CACHE_BUCKETS_KEY = "hbase.bucketcache.bucket.sizes";
96  
97    /**
98     * Defaults for Bucket cache
99     */
100   public static final boolean DEFAULT_BUCKET_CACHE_COMBINED = true;
101   public static final int DEFAULT_BUCKET_CACHE_WRITER_THREADS = 3;
102   public static final int DEFAULT_BUCKET_CACHE_WRITER_QUEUE = 64;
103   public static final float DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE = 0.9f;
104 
105  /**
106    * Configuration key to prefetch all blocks of a given file into the block cache
107    * when the file is opened.
108    */
109   public static final String PREFETCH_BLOCKS_ON_OPEN_KEY =
110       "hbase.rs.prefetchblocksonopen";
111 
112   // Defaults
113 
114   public static final boolean DEFAULT_CACHE_DATA_ON_READ = true;
115   public static final boolean DEFAULT_CACHE_DATA_ON_WRITE = false;
116   public static final boolean DEFAULT_IN_MEMORY = false;
117   public static final boolean DEFAULT_CACHE_INDEXES_ON_WRITE = false;
118   public static final boolean DEFAULT_CACHE_BLOOMS_ON_WRITE = false;
119   public static final boolean DEFAULT_EVICT_ON_CLOSE = false;
120   public static final boolean DEFAULT_COMPRESSED_CACHE = false;
121   public static final boolean DEFAULT_PREFETCH_ON_OPEN = false;
122 
123   /** Local reference to the block cache, null if completely disabled */
124   private final BlockCache blockCache;
125 
126   /**
127    * Whether blocks should be cached on read (default is on if there is a
128    * cache but this can be turned off on a per-family or per-request basis)
129    */
130   private boolean cacheDataOnRead;
131 
132   /** Whether blocks should be flagged as in-memory when being cached */
133   private final boolean inMemory;
134 
135   /** Whether data blocks should be cached when new files are written */
136   private boolean cacheDataOnWrite;
137 
138   /** Whether index blocks should be cached when new files are written */
139   private final boolean cacheIndexesOnWrite;
140 
141   /** Whether compound bloom filter blocks should be cached on write */
142   private final boolean cacheBloomsOnWrite;
143 
144   /** Whether blocks of a file should be evicted when the file is closed */
145   private boolean evictOnClose;
146 
147   /** Whether data blocks should be stored in compressed form in the cache */
148   private final boolean cacheCompressed;
149 
150   /** Whether data blocks should be prefetched into the cache */
151   private final boolean prefetchOnOpen;
152 
153   /**
154    * Create a cache configuration using the specified configuration object and
155    * family descriptor.
156    * @param conf hbase configuration
157    * @param family column family configuration
158    */
159   public CacheConfig(Configuration conf, HColumnDescriptor family) {
160     this(CacheConfig.instantiateBlockCache(conf),
161         family.isBlockCacheEnabled(),
162         family.isInMemory(),
163         // For the following flags we enable them regardless of per-schema settings
164         // if they are enabled in the global configuration.
165         conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY,
166             DEFAULT_CACHE_DATA_ON_WRITE) || family.shouldCacheDataOnWrite(),
167         conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
168             DEFAULT_CACHE_INDEXES_ON_WRITE) || family.shouldCacheIndexesOnWrite(),
169         conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
170             DEFAULT_CACHE_BLOOMS_ON_WRITE) || family.shouldCacheBloomsOnWrite(),
171         conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY,
172             DEFAULT_EVICT_ON_CLOSE) || family.shouldEvictBlocksOnClose(),
173         conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_COMPRESSED_CACHE),
174         conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY,
175             DEFAULT_PREFETCH_ON_OPEN) || family.shouldPrefetchBlocksOnOpen()
176      );
177   }
178 
179   /**
180    * Create a cache configuration using the specified configuration object and
181    * defaults for family level settings.
182    * @param conf hbase configuration
183    */
184   public CacheConfig(Configuration conf) {
185     this(CacheConfig.instantiateBlockCache(conf),
186         DEFAULT_CACHE_DATA_ON_READ,
187         DEFAULT_IN_MEMORY, // This is a family-level setting so can't be set
188                            // strictly from conf
189         conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE),
190         conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
191             DEFAULT_CACHE_INDEXES_ON_WRITE),
192             conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
193                 DEFAULT_CACHE_BLOOMS_ON_WRITE),
194         conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE),
195         conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY,
196             DEFAULT_COMPRESSED_CACHE),
197         conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, DEFAULT_PREFETCH_ON_OPEN)
198      );
199   }
200 
201   /**
202    * Create a block cache configuration with the specified cache and
203    * configuration parameters.
204    * @param blockCache reference to block cache, null if completely disabled
205    * @param cacheDataOnRead whether data blocks should be cached on read
206    * @param inMemory whether blocks should be flagged as in-memory
207    * @param cacheDataOnWrite whether data blocks should be cached on write
208    * @param cacheIndexesOnWrite whether index blocks should be cached on write
209    * @param cacheBloomsOnWrite whether blooms should be cached on write
210    * @param evictOnClose whether blocks should be evicted when HFile is closed
211    * @param cacheCompressed whether to store blocks as compressed in the cache
212    * @param prefetchOnOpen whether to prefetch blocks upon open
213    */
214   CacheConfig(final BlockCache blockCache,
215       final boolean cacheDataOnRead, final boolean inMemory,
216       final boolean cacheDataOnWrite, final boolean cacheIndexesOnWrite,
217       final boolean cacheBloomsOnWrite, final boolean evictOnClose,
218       final boolean cacheCompressed, final boolean prefetchOnOpen) {
219     this.blockCache = blockCache;
220     this.cacheDataOnRead = cacheDataOnRead;
221     this.inMemory = inMemory;
222     this.cacheDataOnWrite = cacheDataOnWrite;
223     this.cacheIndexesOnWrite = cacheIndexesOnWrite;
224     this.cacheBloomsOnWrite = cacheBloomsOnWrite;
225     this.evictOnClose = evictOnClose;
226     this.cacheCompressed = cacheCompressed;
227     this.prefetchOnOpen = prefetchOnOpen;
228   }
229 
230   /**
231    * Constructs a cache configuration copied from the specified configuration.
232    * @param cacheConf
233    */
234   public CacheConfig(CacheConfig cacheConf) {
235     this(cacheConf.blockCache, cacheConf.cacheDataOnRead, cacheConf.inMemory,
236         cacheConf.cacheDataOnWrite, cacheConf.cacheIndexesOnWrite,
237         cacheConf.cacheBloomsOnWrite, cacheConf.evictOnClose,
238         cacheConf.cacheCompressed, cacheConf.prefetchOnOpen);
239   }
240 
241   /**
242    * Checks whether the block cache is enabled.
243    */
244   public boolean isBlockCacheEnabled() {
245     return this.blockCache != null;
246   }
247 
248   /**
249    * Returns the block cache.
250    * @return the block cache, or null if caching is completely disabled
251    */
252   public BlockCache getBlockCache() {
253     return this.blockCache;
254   }
255 
256   /**
257    * Returns whether the blocks of this HFile should be cached on read or not.
258    * @return true if blocks should be cached on read, false if not
259    */
260   public boolean shouldCacheDataOnRead() {
261     return isBlockCacheEnabled() && cacheDataOnRead;
262   }
263 
264   /**
265    * Should we cache a block of a particular category? We always cache
266    * important blocks such as index blocks, as long as the block cache is
267    * available.
268    */
269   public boolean shouldCacheBlockOnRead(BlockCategory category) {
270     boolean shouldCache = isBlockCacheEnabled()
271         && (cacheDataOnRead ||
272             category == BlockCategory.INDEX ||
273             category == BlockCategory.BLOOM ||
274             (prefetchOnOpen &&
275                 (category != BlockCategory.META &&
276                  category != BlockCategory.UNKNOWN)));
277     return shouldCache;
278   }
279 
280   /**
281    * @return true if blocks in this file should be flagged as in-memory
282    */
283   public boolean isInMemory() {
284     return isBlockCacheEnabled() && this.inMemory;
285   }
286 
287   /**
288    * @return true if data blocks should be written to the cache when an HFile is
289    *         written, false if not
290    */
291   public boolean shouldCacheDataOnWrite() {
292     return isBlockCacheEnabled() && this.cacheDataOnWrite;
293   }
294 
295   /**
296    * Only used for testing.
297    * @param cacheDataOnWrite whether data blocks should be written to the cache
298    *                         when an HFile is written
299    */
300   public void setCacheDataOnWrite(boolean cacheDataOnWrite) {
301     this.cacheDataOnWrite = cacheDataOnWrite;
302   }
303 
304   /**
305    * @return true if index blocks should be written to the cache when an HFile
306    *         is written, false if not
307    */
308   public boolean shouldCacheIndexesOnWrite() {
309     return isBlockCacheEnabled() && this.cacheIndexesOnWrite;
310   }
311 
312   /**
313    * @return true if bloom blocks should be written to the cache when an HFile
314    *         is written, false if not
315    */
316   public boolean shouldCacheBloomsOnWrite() {
317     return isBlockCacheEnabled() && this.cacheBloomsOnWrite;
318   }
319 
320   /**
321    * @return true if blocks should be evicted from the cache when an HFile
322    *         reader is closed, false if not
323    */
324   public boolean shouldEvictOnClose() {
325     return isBlockCacheEnabled() && this.evictOnClose;
326   }
327 
328   /**
329    * Only used for testing.
330    * @param evictOnClose whether blocks should be evicted from the cache when an
331    *                     HFile reader is closed
332    */
333   public void setEvictOnClose(boolean evictOnClose) {
334     this.evictOnClose = evictOnClose;
335   }
336 
337   /**
338    * @return true if blocks should be compressed in the cache, false if not
339    */
340   public boolean shouldCacheCompressed() {
341     return isBlockCacheEnabled() && this.cacheCompressed;
342   }
343 
344   /**
345    * @return true if blocks should be prefetched into the cache on open, false if not
346    */
347   public boolean shouldPrefetchOnOpen() {
348     return isBlockCacheEnabled() && this.prefetchOnOpen;
349   }
350 
351   @Override
352   public String toString() {
353     if (!isBlockCacheEnabled()) {
354       return "CacheConfig:disabled";
355     }
356     return "CacheConfig:enabled " +
357       "[cacheDataOnRead=" + shouldCacheDataOnRead() + "] " +
358       "[cacheDataOnWrite=" + shouldCacheDataOnWrite() + "] " +
359       "[cacheIndexesOnWrite=" + shouldCacheIndexesOnWrite() + "] " +
360       "[cacheBloomsOnWrite=" + shouldCacheBloomsOnWrite() + "] " +
361       "[cacheEvictOnClose=" + shouldEvictOnClose() + "] " +
362       "[cacheCompressed=" + shouldCacheCompressed() + "]" +
363       "[prefetchOnOpen=" + shouldPrefetchOnOpen() + "]";
364   }
365 
366   // Static block cache reference and methods
367 
368   /**
369    * Static reference to the block cache, or null if no caching should be used
370    * at all.
371    */
372   private static BlockCache globalBlockCache;
373 
374   /** Boolean whether we have disabled the block cache entirely. */
375   private static boolean blockCacheDisabled = false;
376 
377   /**
378    * Returns the block cache or <code>null</code> in case none should be used.
379    *
380    * @param conf  The current configuration.
381    * @return The block cache or <code>null</code>.
382    */
383   private static synchronized BlockCache instantiateBlockCache(Configuration conf) {
384     if (globalBlockCache != null) return globalBlockCache;
385     if (blockCacheDisabled) return null;
386 
387     float cachePercentage = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
388       HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
389     if (cachePercentage == 0L) {
390       blockCacheDisabled = true;
391       return null;
392     }
393     if (cachePercentage > 1.0) {
394       throw new IllegalArgumentException(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY +
395         " must be between 0.0 and 1.0, and not > 1.0");
396     }
397 
398     // Calculate the amount of heap to give the heap.
399     MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
400     long lruCacheSize = (long) (mu.getMax() * cachePercentage);
401     int blockSize = conf.getInt("hbase.offheapcache.minblocksize", HConstants.DEFAULT_BLOCKSIZE);
402     long offHeapCacheSize =
403       (long) (conf.getFloat("hbase.offheapcache.percentage", (float) 0) *
404           DirectMemoryUtils.getDirectMemorySize());
405     if (offHeapCacheSize <= 0) {
406       String bucketCacheIOEngineName = conf.get(BUCKET_CACHE_IOENGINE_KEY, null);
407       float bucketCachePercentage = conf.getFloat(BUCKET_CACHE_SIZE_KEY, 0F);
408       // A percentage of max heap size or a absolute value with unit megabytes
409       long bucketCacheSize = (long) (bucketCachePercentage < 1 ? mu.getMax()
410           * bucketCachePercentage : bucketCachePercentage * 1024 * 1024);
411 
412       boolean combinedWithLru = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY,
413           DEFAULT_BUCKET_CACHE_COMBINED);
414       BucketCache bucketCache = null;
415       if (bucketCacheIOEngineName != null && bucketCacheSize > 0) {
416         int writerThreads = conf.getInt(BUCKET_CACHE_WRITER_THREADS_KEY,
417             DEFAULT_BUCKET_CACHE_WRITER_THREADS);
418         int writerQueueLen = conf.getInt(BUCKET_CACHE_WRITER_QUEUE_KEY,
419             DEFAULT_BUCKET_CACHE_WRITER_QUEUE);
420         String persistentPath = conf.get(BUCKET_CACHE_PERSISTENT_PATH_KEY);
421         float combinedPercentage = conf.getFloat(
422             BUCKET_CACHE_COMBINED_PERCENTAGE_KEY,
423             DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE);
424         String[] configuredBucketSizes = conf.getStrings(BUCKET_CACHE_BUCKETS_KEY);
425         int[] bucketSizes = null;
426         if (configuredBucketSizes != null) {
427           bucketSizes = new int[configuredBucketSizes.length];
428           for (int i = 0; i < configuredBucketSizes.length; i++) {
429             bucketSizes[i] = Integer.parseInt(configuredBucketSizes[i]);
430           }
431         }
432         if (combinedWithLru) {
433           lruCacheSize = (long) ((1 - combinedPercentage) * bucketCacheSize);
434           bucketCacheSize = (long) (combinedPercentage * bucketCacheSize);
435         }
436         try {
437           int ioErrorsTolerationDuration = conf.getInt(
438               "hbase.bucketcache.ioengine.errors.tolerated.duration",
439               BucketCache.DEFAULT_ERROR_TOLERATION_DURATION);
440           bucketCache = new BucketCache(bucketCacheIOEngineName,
441               bucketCacheSize, blockSize, bucketSizes, writerThreads, writerQueueLen, persistentPath,
442               ioErrorsTolerationDuration);
443         } catch (IOException ioex) {
444           LOG.error("Can't instantiate bucket cache", ioex);
445           throw new RuntimeException(ioex);
446         }
447       }
448       LOG.info("Allocating LruBlockCache with maximum size " +
449         StringUtils.humanReadableInt(lruCacheSize));
450       LruBlockCache lruCache = new LruBlockCache(lruCacheSize, blockSize, true, conf);
451       lruCache.setVictimCache(bucketCache);
452       if (bucketCache != null && combinedWithLru) {
453         globalBlockCache = new CombinedBlockCache(lruCache, bucketCache);
454       } else {
455         globalBlockCache = lruCache;
456       }
457     } else {
458       LOG.warn("SlabCache is deprecated. Consider BucketCache as a replacement.");
459       globalBlockCache = new DoubleBlockCache(
460           lruCacheSize, offHeapCacheSize, blockSize, blockSize, conf);
461     }
462     return globalBlockCache;
463   }
464 }