View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.ipc;
19  
20  import static org.junit.Assert.assertEquals;
21  
22  import java.io.IOException;
23  import java.nio.ByteBuffer;
24  import java.util.Arrays;
25  
26  import org.apache.commons.lang.time.StopWatch;
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.commons.logging.impl.Log4JLogger;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.hbase.Cell;
32  import org.apache.hadoop.hbase.CellScanner;
33  import org.apache.hadoop.hbase.CellUtil;
34  import org.apache.hadoop.hbase.HBaseConfiguration;
35  import org.apache.hadoop.hbase.KeyValue;
36  import org.apache.hadoop.hbase.SmallTests;
37  import org.apache.hadoop.hbase.codec.Codec;
38  import org.apache.hadoop.hbase.codec.KeyValueCodec;
39  import org.apache.hadoop.hbase.io.SizedCellScanner;
40  import org.apache.hadoop.hbase.util.Bytes;
41  import org.apache.hadoop.hbase.util.ClassSize;
42  import org.apache.hadoop.io.compress.CompressionCodec;
43  import org.apache.hadoop.io.compress.DefaultCodec;
44  import org.apache.hadoop.io.compress.GzipCodec;
45  import org.apache.log4j.Level;
46  import org.junit.Before;
47  import org.junit.Test;
48  import org.junit.experimental.categories.Category;
49  
50  @Category(SmallTests.class) 
51  public class TestIPCUtil {
52  
53    public static final Log LOG = LogFactory.getLog(IPCUtil.class);
54  
55    IPCUtil util;
56    @Before
57    public void before() {
58      this.util = new IPCUtil(new Configuration());
59    }
60    
61    @Test
62    public void testBuildCellBlock() throws IOException {
63      doBuildCellBlockUndoCellBlock(this.util, new KeyValueCodec(), null);
64      doBuildCellBlockUndoCellBlock(this.util, new KeyValueCodec(), new DefaultCodec());
65      doBuildCellBlockUndoCellBlock(this.util, new KeyValueCodec(), new GzipCodec());
66    }
67  
68    static void doBuildCellBlockUndoCellBlock(final IPCUtil util,
69        final Codec codec, final CompressionCodec compressor)
70    throws IOException {
71      doBuildCellBlockUndoCellBlock(util, codec, compressor, 10, 1, false);
72    }
73  
74    static void doBuildCellBlockUndoCellBlock(final IPCUtil util, final Codec codec,
75      final CompressionCodec compressor, final int count, final int size, final boolean sized)
76    throws IOException {
77      Cell [] cells = getCells(count, size);
78      CellScanner cellScanner = sized? getSizedCellScanner(cells):
79        CellUtil.createCellScanner(Arrays.asList(cells).iterator());
80      ByteBuffer bb = util.buildCellBlock(codec, compressor, cellScanner);
81      cellScanner = util.createCellScanner(codec, compressor, bb.array(), 0, bb.limit());
82      int i = 0;
83      while (cellScanner.advance()) {
84        i++;
85      }
86      assertEquals(count, i);
87    }
88  
89    static CellScanner getSizedCellScanner(final Cell [] cells) {
90      int size = -1;
91      for (Cell cell: cells) {
92        size += CellUtil.estimatedSizeOf(cell);
93      }
94      final int totalSize = ClassSize.align(size);
95      final CellScanner cellScanner = CellUtil.createCellScanner(cells);
96      return new SizedCellScanner() {
97        @Override
98        public long heapSize() {
99          return totalSize;
100       }
101 
102       @Override
103       public Cell current() {
104         return cellScanner.current();
105       }
106 
107       @Override
108       public boolean advance() throws IOException {
109         return cellScanner.advance();
110       }
111     };
112   }
113 
114   static Cell [] getCells(final int howMany) {
115     return getCells(howMany, 1024);
116   }
117 
118   static Cell [] getCells(final int howMany, final int valueSize) {
119     Cell [] cells = new Cell[howMany];
120     byte [] value = new byte[valueSize];
121     for (int i = 0; i < howMany; i++) {
122       byte [] index = Bytes.toBytes(i);
123       KeyValue kv = new KeyValue(index, Bytes.toBytes("f"), index, value);
124       cells[i] = kv;
125     }
126     return cells;
127   }
128 
129   private static final String COUNT = "--count=";
130   private static final String SIZE = "--size=";
131 
132   /**
133    * Prints usage and then exits w/ passed <code>errCode</code>
134    * @param errCode
135    */
136   private static void usage(final int errCode) {
137     System.out.println("Usage: IPCUtil [options]");
138     System.out.println("Micro-benchmarking how changed sizes and counts work with buffer resizing");
139     System.out.println(" --count  Count of Cells");
140     System.out.println(" --size   Size of Cell values");
141     System.out.println("Example: IPCUtil --count=1024 --size=1024");
142     System.exit(errCode);
143   }
144 
145   private static void timerTests(final IPCUtil util, final int count, final int size,
146       final Codec codec, final CompressionCodec compressor)
147   throws IOException {
148     final int cycles = 1000;
149     StopWatch timer = new StopWatch();
150     timer.start();
151     for (int i = 0; i < cycles; i++) {
152       timerTest(util, timer, count, size, codec, compressor, false);
153     }
154     timer.stop();
155     LOG.info("Codec=" + codec + ", compression=" + compressor + ", sized=" + false +
156         ", count=" + count + ", size=" + size + ", + took=" + timer.getTime() + "ms");
157     timer.reset();
158     timer.start();
159     for (int i = 0; i < cycles; i++) {
160       timerTest(util, timer, count, size, codec, compressor, true);
161     }
162     timer.stop();
163     LOG.info("Codec=" + codec + ", compression=" + compressor + ", sized=" + true +
164       ", count=" + count + ", size=" + size + ", + took=" + timer.getTime() + "ms");
165   }
166 
167   private static void timerTest(final IPCUtil util, final StopWatch timer, final int count,
168       final int size, final Codec codec, final CompressionCodec compressor, final boolean sized)
169   throws IOException {
170     doBuildCellBlockUndoCellBlock(util, codec, compressor, count, size, sized);
171   }
172 
173   /**
174    * For running a few tests of methods herein.
175    * @param args
176    * @throws IOException
177    */
178   public static void main(String[] args) throws IOException {
179     int count = 1024;
180     int size = 10240;
181     for (String arg: args) {
182       if (arg.startsWith(COUNT)) {
183         count = Integer.parseInt(arg.replace(COUNT, ""));
184       } else if (arg.startsWith(SIZE)) {
185         size = Integer.parseInt(arg.replace(SIZE, ""));
186       } else {
187         usage(1);
188       }
189     }
190     IPCUtil util = new IPCUtil(HBaseConfiguration.create());
191     ((Log4JLogger)IPCUtil.LOG).getLogger().setLevel(Level.ALL);
192     timerTests(util, count, size,  new KeyValueCodec(), null);
193     timerTests(util, count, size,  new KeyValueCodec(), new DefaultCodec());
194     timerTests(util, count, size,  new KeyValueCodec(), new GzipCodec());
195   }
196 }