View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import static org.junit.Assert.*;
22  
23  import java.util.List;
24  import java.util.Map;
25  import java.util.Random;
26  import java.util.concurrent.atomic.AtomicInteger;
27  
28  import org.apache.hadoop.conf.Configuration;
29  import org.apache.hadoop.hbase.MultithreadedTestUtil;
30  import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
31  import org.apache.hadoop.hbase.SmallTests;
32  import org.apache.hadoop.hbase.regionserver.MemStoreLAB.Allocation;
33  import org.junit.Test;
34  
35  import com.google.common.collect.Iterables;
36  import com.google.common.collect.Lists;
37  import com.google.common.collect.Maps;
38  import com.google.common.primitives.Ints;
39  import org.junit.experimental.categories.Category;
40  
41  @Category(SmallTests.class)
42  public class TestMemStoreLAB {
43  
44    /**
45     * Test a bunch of random allocations
46     */
47    @Test
48    public void testLABRandomAllocation() {
49      Random rand = new Random();
50      MemStoreLAB mslab = new MemStoreLAB();
51      int expectedOff = 0;
52      byte[] lastBuffer = null;
53      // 100K iterations by 0-1K alloc -> 50MB expected
54      // should be reasonable for unit test and also cover wraparound
55      // behavior
56      for (int i = 0; i < 100000; i++) {
57        int size = rand.nextInt(1000);
58        Allocation alloc = mslab.allocateBytes(size);
59        
60        if (alloc.getData() != lastBuffer) {
61          expectedOff = 0;
62          lastBuffer = alloc.getData();
63        }
64        assertEquals(expectedOff, alloc.getOffset());
65        assertTrue("Allocation " + alloc + " overruns buffer",
66            alloc.getOffset() + size <= alloc.getData().length);
67        expectedOff += size;
68      }
69    }
70  
71    @Test
72    public void testLABLargeAllocation() {
73      MemStoreLAB mslab = new MemStoreLAB();
74      Allocation alloc = mslab.allocateBytes(2*1024*1024);
75      assertNull("2MB allocation shouldn't be satisfied by LAB.",
76        alloc);
77    } 
78  
79    /**
80     * Test allocation from lots of threads, making sure the results don't
81     * overlap in any way
82     */
83    @Test
84    public void testLABThreading() throws Exception {
85      Configuration conf = new Configuration();
86      MultithreadedTestUtil.TestContext ctx =
87        new MultithreadedTestUtil.TestContext(conf);
88      
89      final AtomicInteger totalAllocated = new AtomicInteger();
90      
91      final MemStoreLAB mslab = new MemStoreLAB();
92      List<List<AllocRecord>> allocations = Lists.newArrayList();
93      
94      for (int i = 0; i < 10; i++) {
95        final List<AllocRecord> allocsByThisThread = Lists.newLinkedList();
96        allocations.add(allocsByThisThread);
97        
98        TestThread t = new MultithreadedTestUtil.RepeatingTestThread(ctx) {
99          private Random r = new Random();
100         @Override
101         public void doAnAction() throws Exception {
102           int size = r.nextInt(1000);
103           Allocation alloc = mslab.allocateBytes(size);
104           totalAllocated.addAndGet(size);
105           allocsByThisThread.add(new AllocRecord(alloc, size));
106         }
107       };
108       ctx.addThread(t);
109     }
110     
111     ctx.startThreads();
112     while (totalAllocated.get() < 50*1024*1024 && ctx.shouldRun()) {
113       Thread.sleep(10);
114     }
115     ctx.stop();
116     
117     // Partition the allocations by the actual byte[] they point into,
118     // make sure offsets are unique for each chunk
119     Map<byte[], Map<Integer, AllocRecord>> mapsByChunk =
120       Maps.newHashMap();
121     
122     int sizeCounted = 0;
123     for (AllocRecord rec : Iterables.concat(allocations)) {
124       sizeCounted += rec.size;
125       if (rec.size == 0) continue;
126       
127       Map<Integer, AllocRecord> mapForThisByteArray =
128         mapsByChunk.get(rec.alloc.getData());
129       if (mapForThisByteArray == null) {
130         mapForThisByteArray = Maps.newTreeMap();
131         mapsByChunk.put(rec.alloc.getData(), mapForThisByteArray);
132       }
133       AllocRecord oldVal = mapForThisByteArray.put(rec.alloc.getOffset(), rec);
134       assertNull("Already had an entry " + oldVal + " for allocation " + rec,
135           oldVal);
136     }
137     assertEquals("Sanity check test", sizeCounted, totalAllocated.get());
138     
139     // Now check each byte array to make sure allocations don't overlap
140     for (Map<Integer, AllocRecord> allocsInChunk : mapsByChunk.values()) {
141       int expectedOff = 0;
142       for (AllocRecord alloc : allocsInChunk.values()) {
143         assertEquals(expectedOff, alloc.alloc.getOffset());
144         assertTrue("Allocation " + alloc + " overruns buffer",
145             alloc.alloc.getOffset() + alloc.size <= alloc.alloc.getData().length);
146         expectedOff += alloc.size;
147       }
148     }
149 
150   }
151   
152   private static class AllocRecord implements Comparable<AllocRecord>{
153     private final Allocation alloc;
154     private final int size;
155     public AllocRecord(Allocation alloc, int size) {
156       super();
157       this.alloc = alloc;
158       this.size = size;
159     }
160 
161     @Override
162     public int compareTo(AllocRecord e) {
163       if (alloc.getData() != e.alloc.getData()) {
164         throw new RuntimeException("Can only compare within a particular array");
165       }
166       return Ints.compare(alloc.getOffset(), e.alloc.getOffset());
167     }
168     
169     @Override
170     public String toString() {
171       return "AllocRecord(alloc=" + alloc + ", size=" + size + ")";
172     }
173     
174   }
175 
176 }
177