View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertTrue;
23  
24  import java.io.IOException;
25  import java.util.ArrayList;
26  import java.util.List;
27  import java.util.Random;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.hbase.TableName;
32  import org.apache.hadoop.hbase.HBaseTestingUtility;
33  import org.apache.hadoop.hbase.LargeTests;
34  import org.apache.hadoop.hbase.client.HBaseAdmin;
35  import org.apache.hadoop.hbase.client.HTable;
36  import org.apache.hadoop.hbase.client.Put;
37  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
38  import org.apache.hadoop.hbase.util.Bytes;
39  import org.junit.AfterClass;
40  import org.junit.BeforeClass;
41  import org.junit.Test;
42  import org.junit.experimental.categories.Category;
43  
44  /** Unit tests to test retrieving table/region compaction state*/
45  @Category(LargeTests.class)
46  public class TestCompactionState {
47    final static Log LOG = LogFactory.getLog(TestCompactionState.class);
48    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
49    private final static Random random = new Random();
50  
51    @BeforeClass
52    public static void setUpBeforeClass() throws Exception {
53      TEST_UTIL.startMiniCluster();
54    }
55  
56    @AfterClass
57    public static void tearDownAfterClass() throws Exception {
58      TEST_UTIL.shutdownMiniCluster();
59    }
60  
61    @Test(timeout=600000)
62    public void testMajorCompaction() throws IOException, InterruptedException {
63      compaction("testMajorCompaction", 8, CompactionState.MAJOR, false);
64    }
65  
66    @Test(timeout=600000)
67    public void testMinorCompaction() throws IOException, InterruptedException {
68      compaction("testMinorCompaction", 15, CompactionState.MINOR, false);
69    }
70  
71    @Test(timeout=600000)
72    public void testMajorCompactionOnFamily() throws IOException, InterruptedException {
73      compaction("testMajorCompactionOnFamily", 8, CompactionState.MAJOR, true);
74    }
75  
76    @Test(timeout=600000)
77    public void testMinorCompactionOnFamily() throws IOException, InterruptedException {
78      compaction("testMinorCompactionOnFamily", 15, CompactionState.MINOR, true);
79    }
80  
81    @Test
82    public void testInvalidColumnFamily() throws IOException, InterruptedException {
83      byte [] table = Bytes.toBytes("testInvalidColumnFamily");
84      byte [] family = Bytes.toBytes("family");
85      byte [] fakecf = Bytes.toBytes("fakecf");
86      boolean caughtMinorCompact = false;
87      boolean caughtMajorCompact = false;
88      HTable ht = null;
89      try {
90        ht = TEST_UTIL.createTable(table, family);
91        HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
92        try {
93          admin.compact(table, fakecf);
94        } catch (IOException ioe) {
95          caughtMinorCompact = true;
96        }
97        try {
98          admin.majorCompact(table, fakecf);
99        } catch (IOException ioe) {
100         caughtMajorCompact = true;
101       }
102     } finally {
103       if (ht != null) {
104         TEST_UTIL.deleteTable(table);
105       }
106       assertTrue(caughtMinorCompact);
107       assertTrue(caughtMajorCompact);
108     }
109   }
110 
111   /**
112    * Load data to a table, flush it to disk, trigger compaction,
113    * confirm the compaction state is right and wait till it is done.
114    *
115    * @param tableName
116    * @param flushes
117    * @param expectedState
118    * @param singleFamily otherwise, run compaction on all cfs
119    * @throws IOException
120    * @throws InterruptedException
121    */
122   private void compaction(final String tableName, final int flushes,
123       final CompactionState expectedState, boolean singleFamily)
124       throws IOException, InterruptedException {
125     // Create a table with regions
126     TableName table =
127         TableName.valueOf(tableName);
128     byte [] family = Bytes.toBytes("family");
129     byte [][] families =
130       {family, Bytes.add(family, Bytes.toBytes("2")), Bytes.add(family, Bytes.toBytes("3"))};
131     HTable ht = null;
132     try {
133       ht = TEST_UTIL.createTable(table, families);
134       loadData(ht, families, 3000, flushes);
135       HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
136       List<HRegion> regions = rs.getOnlineRegions(table);
137       int countBefore = countStoreFilesInFamilies(regions, families);
138       int countBeforeSingleFamily = countStoreFilesInFamily(regions, family);
139       assertTrue(countBefore > 0); // there should be some data files
140       HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
141       if (expectedState == CompactionState.MINOR) {
142         if (singleFamily) {
143           admin.compact(table.getName(), family);
144         } else {
145           admin.compact(table.getName());
146         }
147       } else {
148         if (singleFamily) {
149           admin.majorCompact(table.getName(), family);
150         } else {
151           admin.majorCompact(table.getName());
152         }
153       }
154       long curt = System.currentTimeMillis();
155       long waitTime = 5000;
156       long endt = curt + waitTime;
157       CompactionState state = admin.getCompactionState(table.getName());
158       while (state == CompactionState.NONE && curt < endt) {
159         Thread.sleep(10);
160         state = admin.getCompactionState(table.getName());
161         curt = System.currentTimeMillis();
162       }
163       // Now, should have the right compaction state,
164       // otherwise, the compaction should have already been done
165       if (expectedState != state) {
166         for (HRegion region: regions) {
167           state = region.getCompactionState();
168           assertEquals(CompactionState.NONE, state);
169         }
170       } else {
171         // Wait until the compaction is done
172         state = admin.getCompactionState(table.getName());
173         while (state != CompactionState.NONE && curt < endt) {
174           Thread.sleep(10);
175           state = admin.getCompactionState(table.getName());
176         }
177         // Now, compaction should be done.
178         assertEquals(CompactionState.NONE, state);
179       }
180       int countAfter = countStoreFilesInFamilies(regions, families);
181       int countAfterSingleFamily = countStoreFilesInFamily(regions, family);
182       assertTrue(countAfter < countBefore);
183       if (!singleFamily) {
184         if (expectedState == CompactionState.MAJOR) assertTrue(families.length == countAfter);
185         else assertTrue(families.length < countAfter);
186       } else {
187         int singleFamDiff = countBeforeSingleFamily - countAfterSingleFamily;
188         // assert only change was to single column family
189         assertTrue(singleFamDiff == (countBefore - countAfter));
190         if (expectedState == CompactionState.MAJOR) {
191           assertTrue(1 == countAfterSingleFamily);
192         } else {
193           assertTrue(1 < countAfterSingleFamily);
194         }
195       }
196     } finally {
197       if (ht != null) {
198         TEST_UTIL.deleteTable(table);
199       }
200     }
201   }
202 
203   private static int countStoreFilesInFamily(
204       List<HRegion> regions, final byte[] family) {
205     return countStoreFilesInFamilies(regions, new byte[][]{family});
206   }
207 
208   private static int countStoreFilesInFamilies(List<HRegion> regions, final byte[][] families) {
209     int count = 0;
210     for (HRegion region: regions) {
211       count += region.getStoreFileList(families).size();
212     }
213     return count;
214   }
215 
216   private static void loadData(final HTable ht, final byte[][] families,
217       final int rows, final int flushes) throws IOException {
218     List<Put> puts = new ArrayList<Put>(rows);
219     byte[] qualifier = Bytes.toBytes("val");
220     for (int i = 0; i < flushes; i++) {
221       for (int k = 0; k < rows; k++) {
222         byte[] row = Bytes.toBytes(random.nextLong());
223         Put p = new Put(row);
224         for (int j = 0; j < families.length; ++j) {
225           p.add(families[ j ], qualifier, row);
226         }
227         puts.add(p);
228       }
229       ht.put(puts);
230       ht.flushCommits();
231       TEST_UTIL.flush();
232       puts.clear();
233     }
234   } 
235 }