View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.io.hfile;
19  
20  import java.io.IOException;
21  import java.util.NavigableMap;
22  import java.util.NavigableSet;
23  import java.util.concurrent.ConcurrentSkipListMap;
24  import java.util.concurrent.ConcurrentSkipListSet;
25  
26  import org.apache.hadoop.classification.InterfaceAudience;
27  import org.apache.hadoop.conf.Configuration;
28  import org.codehaus.jackson.JsonGenerationException;
29  import org.codehaus.jackson.annotate.JsonIgnoreProperties;
30  import org.codehaus.jackson.map.JsonMappingException;
31  import org.codehaus.jackson.map.ObjectMapper;
32  import org.codehaus.jackson.map.SerializationConfig;
33  
34  import com.yammer.metrics.core.Histogram;
35  import com.yammer.metrics.core.MetricsRegistry;
36  import com.yammer.metrics.stats.Snapshot;
37  
38  /**
39   * Utilty for aggregating counts in CachedBlocks and toString/toJSON CachedBlocks and BlockCaches.
40   * No attempt has been made at making this thread safe.
41   */
42  @InterfaceAudience.Private
43  public class BlockCacheUtil {
44    /**
45     * Needed making histograms.
46     */
47    private static final MetricsRegistry METRICS = new MetricsRegistry();
48  
49    /**
50     * Needed generating JSON.
51     */
52    private static final ObjectMapper MAPPER = new ObjectMapper();
53    static {
54      MAPPER.configure(SerializationConfig.Feature.FAIL_ON_EMPTY_BEANS, false);
55      MAPPER.configure(SerializationConfig.Feature.FLUSH_AFTER_WRITE_VALUE, true);
56      MAPPER.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
57    }
58  
59    /**
60     * @param cb
61     * @return The block content as String.
62     */
63    public static String toString(final CachedBlock cb, final long now) {
64      return "filename=" + cb.getFilename() + ", " + toStringMinusFileName(cb, now);
65    }
66  
67    /**
68     * Little data structure to hold counts for a file.
69     * Used doing a toJSON.
70     */
71    static class CachedBlockCountsPerFile {
72      private int count = 0;
73      private long size = 0;
74      private int countData = 0;
75      private long sizeData = 0;
76      private final String filename;
77  
78      CachedBlockCountsPerFile(final String filename) {
79        this.filename = filename;
80      }
81  
82      public int getCount() {
83        return count;
84      }
85  
86      public long getSize() {
87        return size;
88      }
89  
90      public int getCountData() {
91        return countData;
92      }
93  
94      public long getSizeData() {
95        return sizeData;
96      }
97  
98      public String getFilename() {
99        return filename;
100     }
101   }
102 
103   /**
104    * @param filename
105    * @param blocks
106    * @return A JSON String of <code>filename</code> and counts of <code>blocks</code>
107    * @throws JsonGenerationException
108    * @throws JsonMappingException
109    * @throws IOException
110    */
111   public static String toJSON(final String filename, final NavigableSet<CachedBlock> blocks)
112   throws JsonGenerationException, JsonMappingException, IOException {
113     CachedBlockCountsPerFile counts = new CachedBlockCountsPerFile(filename);
114     for (CachedBlock cb: blocks) {
115       counts.count++;
116       counts.size += cb.getSize();
117       BlockType bt = cb.getBlockType();
118       if (bt != null && bt.isData()) {
119         counts.countData++;
120         counts.sizeData += cb.getSize();
121       }
122     }
123     return MAPPER.writeValueAsString(counts);
124   }
125 
126   /**
127    * @param cbsbf
128    * @return JSON string of <code>cbsf</code> aggregated
129    * @throws JsonGenerationException
130    * @throws JsonMappingException
131    * @throws IOException
132    */
133   public static String toJSON(final CachedBlocksByFile cbsbf)
134   throws JsonGenerationException, JsonMappingException, IOException {
135     return MAPPER.writeValueAsString(cbsbf);
136   }
137 
138   /**
139    * @param bc
140    * @return JSON string of <code>bc</code> content.
141    * @throws JsonGenerationException
142    * @throws JsonMappingException
143    * @throws IOException
144    */
145   public static String toJSON(final BlockCache bc)
146   throws JsonGenerationException, JsonMappingException, IOException {
147     return MAPPER.writeValueAsString(bc);
148   }
149 
150   /**
151    * @param cb
152    * @return The block content of <code>bc</code> as a String minus the filename.
153    */
154   public static String toStringMinusFileName(final CachedBlock cb, final long now) {
155     return "offset=" + cb.getOffset() +
156       ", size=" + cb.getSize() +
157       ", age=" + (now - cb.getCachedTime()) +
158       ", type=" + cb.getBlockType() +
159       ", priority=" + cb.getBlockPriority();
160   }
161 
162   /**
163    * Snapshot of block cache age in cache.
164    * This object is preferred because we can control how it is serialized out when JSON'ing.
165    */
166   @JsonIgnoreProperties({"ageHistogram", "snapshot"})
167   public static class AgeSnapshot {
168     private final Histogram ageHistogram;
169     private final Snapshot snapshot;
170 
171     AgeSnapshot(final Histogram ageHistogram) {
172       this.ageHistogram = ageHistogram;
173       this.snapshot = ageHistogram.getSnapshot();
174     }
175 
176     public double get75thPercentile() {
177       return snapshot.get75thPercentile();
178     }
179 
180     public double get95thPercentile() {
181       return snapshot.get95thPercentile();
182     }
183 
184     public double get98thPercentile() {
185       return snapshot.get98thPercentile();
186     }
187 
188     public double get999thPercentile() {
189       return snapshot.get999thPercentile();
190     }
191 
192     public double get99thPercentile() {
193       return snapshot.get99thPercentile();
194     }
195 
196     public double getMean() {
197       return this.ageHistogram.mean();
198     }
199 
200     public double getMax() {
201       return ageHistogram.max();
202     }
203 
204     public double getMin() {
205       return ageHistogram.min();
206     }
207 
208     public double getStdDev() {
209       return ageHistogram.stdDev();
210     }
211   }
212 
213   /**
214    * Get a {@link CachedBlocksByFile} instance and load it up by iterating content in
215    * {@link BlockCache}.
216    * @param conf Used to read configurations
217    * @param bc Block Cache to iterate.
218    * @return Laoded up instance of CachedBlocksByFile
219    */
220   public static CachedBlocksByFile getLoadedCachedBlocksByFile(final Configuration conf,
221       final BlockCache bc) {
222     CachedBlocksByFile cbsbf = new CachedBlocksByFile(conf);
223     for (CachedBlock cb: bc) {
224       if (cbsbf.update(cb)) break;
225     }
226     return cbsbf;
227   }
228 
229   /**
230    * Use one of these to keep a running account of cached blocks by file.  Throw it away when done.
231    * This is different than metrics in that it is stats on current state of a cache.
232    * @see getLoadedCachedBlocksByFile
233    */
234   @JsonIgnoreProperties({"cachedBlockStatsByFile"})
235   public static class CachedBlocksByFile {
236     private int count;
237     private int dataBlockCount;
238     private long size;
239     private long dataSize;
240     private final long now = System.nanoTime();
241     private final int max;
242     public static final int DEFAULT_MAX = 100000;
243  
244     CachedBlocksByFile() {
245       this(null);
246     }
247 
248     CachedBlocksByFile(final Configuration c) {
249       this.max = c == null? DEFAULT_MAX:
250         c.getInt("hbase.ui.blockcache.by.file.max", DEFAULT_MAX);
251     }
252 
253     /**
254      * Map by filename. use concurent utils because we want our Map and contained blocks sorted.
255      */
256     private NavigableMap<String, NavigableSet<CachedBlock>> cachedBlockByFile =
257       new ConcurrentSkipListMap<String, NavigableSet<CachedBlock>>();
258     Histogram age = METRICS.newHistogram(CachedBlocksByFile.class, "age");
259 
260     /**
261      * @param cb
262      * @return True if full.... if we won't be adding any more.
263      */
264     public boolean update(final CachedBlock cb) {
265       if (isFull()) return true;
266       NavigableSet<CachedBlock> set = this.cachedBlockByFile.get(cb.getFilename());
267       if (set == null) {
268         set = new ConcurrentSkipListSet<CachedBlock>();
269         this.cachedBlockByFile.put(cb.getFilename(), set);
270       }
271       set.add(cb);
272       this.size += cb.getSize();
273       this.count++;
274       BlockType bt = cb.getBlockType();
275       if (bt != null && bt.isData()) {
276         this.dataBlockCount++;
277         this.dataSize += cb.getSize();
278       }
279       long age = this.now - cb.getCachedTime();
280       this.age.update(age);
281       return false;
282     }
283 
284     /**
285      * @return True if full; i.e. there are more items in the cache but we only loaded up
286      * the maximum set in configuration <code>hbase.ui.blockcache.by.file.max</code>
287      * (Default: DEFAULT_MAX).
288      */
289     public boolean isFull() {
290       return this.count >= this.max;
291     }
292  
293     public NavigableMap<String, NavigableSet<CachedBlock>> getCachedBlockStatsByFile() {
294       return this.cachedBlockByFile;
295     }
296 
297     /**
298      * @return count of blocks in the cache
299      */
300     public int getCount() {
301       return count;
302     }
303 
304     public int getDataCount() {
305       return dataBlockCount;
306     }
307 
308     /**
309      * @return size of blocks in the cache
310      */
311     public long getSize() {
312       return size;
313     }
314 
315     /**
316      * @return Size of data.
317      */
318     public long getDataSize() {
319       return dataSize;
320     }
321 
322     public AgeSnapshot getAgeSnapshot() {
323       return new AgeSnapshot(this.age);
324     }
325 
326     @Override
327     public String toString() {
328       Snapshot snapshot = this.age.getSnapshot();
329       return "count=" + count + ", dataBlockCount=" + this.dataBlockCount + ", size=" + size +
330           ", dataSize=" + getDataSize() +
331           ", mean age=" + this.age.mean() + ", stddev age=" + this.age.stdDev() +
332           ", min age=" + this.age.min() + ", max age=" + this.age.max() +
333           ", 95th percentile age=" + snapshot.get95thPercentile() +
334           ", 99th percentile age=" + snapshot.get99thPercentile();
335     }
336   }
337 }