View Javadoc

1   /**
2    * Copyright The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one or more
5    * contributor license agreements. See the NOTICE file distributed with this
6    * work for additional information regarding copyright ownership. The ASF
7    * licenses this file to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance with the License.
9    * You may obtain a copy of the License at
10   *
11   * http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
15   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
16   * License for the specific language governing permissions and limitations
17   * under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertTrue;
23  
24  import java.rmi.UnexpectedException;
25  import java.util.List;
26  import java.util.Random;
27  
28  import org.apache.hadoop.conf.Configuration;
29  import org.apache.hadoop.hbase.KeyValue;
30  import org.apache.hadoop.hbase.SmallTests;
31  import org.apache.hadoop.hbase.regionserver.MemStoreLAB.Allocation;
32  import org.apache.hadoop.hbase.util.Bytes;
33  import org.junit.AfterClass;
34  import org.junit.Before;
35  import org.junit.BeforeClass;
36  import org.junit.Test;
37  import org.junit.experimental.categories.Category;
38  
39  /**
40   * Test the {@link MemStoreChunkPool} class
41   */
42  @Category(SmallTests.class)
43  public class TestMemStoreChunkPool {
44    private final static Configuration conf = new Configuration();
45    private static MemStoreChunkPool chunkPool;
46    private static boolean chunkPoolDisabledBeforeTest;
47  
48    @BeforeClass
49    public static void setUpBeforeClass() throws Exception {
50      conf.setBoolean(MemStore.USEMSLAB_KEY, true);
51      conf.setFloat(MemStoreChunkPool.CHUNK_POOL_MAXSIZE_KEY, 0.2f);
52      chunkPoolDisabledBeforeTest = MemStoreChunkPool.chunkPoolDisabled;
53      MemStoreChunkPool.chunkPoolDisabled = false;
54      chunkPool = MemStoreChunkPool.getPool(conf);
55      assertTrue(chunkPool != null);
56    }
57  
58    @AfterClass
59    public static void tearDownAfterClass() throws Exception {
60      MemStoreChunkPool.chunkPoolDisabled = chunkPoolDisabledBeforeTest;
61    }
62  
63    @Before
64    public void tearDown() throws Exception {
65      chunkPool.clearChunks();
66    }
67  
68    @Test
69    public void testReusingChunks() {
70      Random rand = new Random();
71      MemStoreLAB mslab = new MemStoreLAB(conf, chunkPool);
72      int expectedOff = 0;
73      byte[] lastBuffer = null;
74      // Randomly allocate some bytes
75      for (int i = 0; i < 100; i++) {
76        int size = rand.nextInt(1000);
77        Allocation alloc = mslab.allocateBytes(size);
78  
79        if (alloc.getData() != lastBuffer) {
80          expectedOff = 0;
81          lastBuffer = alloc.getData();
82        }
83        assertEquals(expectedOff, alloc.getOffset());
84        assertTrue("Allocation " + alloc + " overruns buffer", alloc.getOffset()
85            + size <= alloc.getData().length);
86        expectedOff += size;
87      }
88      // chunks will be put back to pool after close
89      mslab.close();
90      int chunkCount = chunkPool.getPoolSize();
91      assertTrue(chunkCount > 0);
92      // reconstruct mslab
93      mslab = new MemStoreLAB(conf, chunkPool);
94      // chunk should be got from the pool, so we can reuse it.
95      mslab.allocateBytes(1000);
96      assertEquals(chunkCount - 1, chunkPool.getPoolSize());
97    }
98  
99    @Test
100   public void testPuttingBackChunksAfterFlushing() throws UnexpectedException {
101     byte[] row = Bytes.toBytes("testrow");
102     byte[] fam = Bytes.toBytes("testfamily");
103     byte[] qf1 = Bytes.toBytes("testqualifier1");
104     byte[] qf2 = Bytes.toBytes("testqualifier2");
105     byte[] qf3 = Bytes.toBytes("testqualifier3");
106     byte[] qf4 = Bytes.toBytes("testqualifier4");
107     byte[] qf5 = Bytes.toBytes("testqualifier5");
108     byte[] val = Bytes.toBytes("testval");
109 
110     MemStore memstore = new MemStore();
111 
112     // Setting up memstore
113     memstore.add(new KeyValue(row, fam, qf1, val));
114     memstore.add(new KeyValue(row, fam, qf2, val));
115     memstore.add(new KeyValue(row, fam, qf3, val));
116 
117     // Creating a snapshot
118     memstore.snapshot();
119     KeyValueSkipListSet snapshot = memstore.getSnapshot();
120     assertEquals(3, memstore.snapshot.size());
121 
122     // Adding value to "new" memstore
123     assertEquals(0, memstore.kvset.size());
124     memstore.add(new KeyValue(row, fam, qf4, val));
125     memstore.add(new KeyValue(row, fam, qf5, val));
126     assertEquals(2, memstore.kvset.size());
127     memstore.clearSnapshot(snapshot);
128 
129     int chunkCount = chunkPool.getPoolSize();
130     assertTrue(chunkCount > 0);
131 
132   }
133 
134   @Test
135   public void testPuttingBackChunksWithOpeningScanner()
136       throws UnexpectedException {
137     byte[] row = Bytes.toBytes("testrow");
138     byte[] fam = Bytes.toBytes("testfamily");
139     byte[] qf1 = Bytes.toBytes("testqualifier1");
140     byte[] qf2 = Bytes.toBytes("testqualifier2");
141     byte[] qf3 = Bytes.toBytes("testqualifier3");
142     byte[] qf4 = Bytes.toBytes("testqualifier4");
143     byte[] qf5 = Bytes.toBytes("testqualifier5");
144     byte[] qf6 = Bytes.toBytes("testqualifier6");
145     byte[] qf7 = Bytes.toBytes("testqualifier7");
146     byte[] val = Bytes.toBytes("testval");
147 
148     MemStore memstore = new MemStore();
149 
150     // Setting up memstore
151     memstore.add(new KeyValue(row, fam, qf1, val));
152     memstore.add(new KeyValue(row, fam, qf2, val));
153     memstore.add(new KeyValue(row, fam, qf3, val));
154 
155     // Creating a snapshot
156     memstore.snapshot();
157     KeyValueSkipListSet snapshot = memstore.getSnapshot();
158     assertEquals(3, memstore.snapshot.size());
159 
160     // Adding value to "new" memstore
161     assertEquals(0, memstore.kvset.size());
162     memstore.add(new KeyValue(row, fam, qf4, val));
163     memstore.add(new KeyValue(row, fam, qf5, val));
164     assertEquals(2, memstore.kvset.size());
165 
166     // opening scanner before clear the snapshot
167     List<KeyValueScanner> scanners = memstore.getScanners(0);
168     // Shouldn't putting back the chunks to pool,since some scanners are opening
169     // based on their data
170     memstore.clearSnapshot(snapshot);
171 
172     assertTrue(chunkPool.getPoolSize() == 0);
173 
174     // Chunks will be put back to pool after close scanners;
175     for (KeyValueScanner scanner : scanners) {
176       scanner.close();
177     }
178     assertTrue(chunkPool.getPoolSize() > 0);
179 
180     // clear chunks
181     chunkPool.clearChunks();
182 
183     // Creating another snapshot
184     memstore.snapshot();
185     snapshot = memstore.getSnapshot();
186     // Adding more value
187     memstore.add(new KeyValue(row, fam, qf6, val));
188     memstore.add(new KeyValue(row, fam, qf7, val));
189     // opening scanners
190     scanners = memstore.getScanners(0);
191     // close scanners before clear the snapshot
192     for (KeyValueScanner scanner : scanners) {
193       scanner.close();
194     }
195     // Since no opening scanner, the chunks of snapshot should be put back to
196     // pool
197     memstore.clearSnapshot(snapshot);
198     assertTrue(chunkPool.getPoolSize() > 0);
199   }
200 
201 }