View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.master.procedure;
20  
21  import static org.junit.Assert.assertTrue;
22  
23  import org.apache.commons.logging.Log;
24  import org.apache.commons.logging.LogFactory;
25  import org.apache.hadoop.conf.Configuration;
26  import org.apache.hadoop.hbase.HBaseTestingUtility;
27  import org.apache.hadoop.hbase.HColumnDescriptor;
28  import org.apache.hadoop.hbase.HConstants;
29  import org.apache.hadoop.hbase.HTableDescriptor;
30  import org.apache.hadoop.hbase.InvalidFamilyOperationException;
31  import org.apache.hadoop.hbase.ProcedureInfo;
32  import org.apache.hadoop.hbase.TableName;
33  import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
34  import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
35  import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ModifyColumnFamilyState;
36  import org.apache.hadoop.hbase.testclassification.MediumTests;
37  import org.junit.After;
38  import org.junit.AfterClass;
39  import org.junit.Before;
40  import org.junit.BeforeClass;
41  import org.junit.Test;
42  import org.junit.experimental.categories.Category;
43  
44  @Category(MediumTests.class)
45  public class TestModifyColumnFamilyProcedure {
46    private static final Log LOG = LogFactory.getLog(TestModifyColumnFamilyProcedure.class);
47  
48    protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
49  
50    private static long nonceGroup = HConstants.NO_NONCE;
51    private static long nonce = HConstants.NO_NONCE;
52  
53    private static void setupConf(Configuration conf) {
54      conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
55    }
56  
57    @BeforeClass
58    public static void setupCluster() throws Exception {
59      setupConf(UTIL.getConfiguration());
60      UTIL.startMiniCluster(1);
61    }
62  
63    @AfterClass
64    public static void cleanupTest() throws Exception {
65      try {
66        UTIL.shutdownMiniCluster();
67      } catch (Exception e) {
68        LOG.warn("failure shutting down cluster", e);
69      }
70    }
71  
72    @Before
73    public void setup() throws Exception {
74      ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
75      nonceGroup =
76          MasterProcedureTestingUtility.generateNonceGroup(UTIL.getHBaseCluster().getMaster());
77      nonce = MasterProcedureTestingUtility.generateNonce(UTIL.getHBaseCluster().getMaster());
78    }
79  
80    @After
81    public void tearDown() throws Exception {
82      ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
83      for (HTableDescriptor htd: UTIL.getHBaseAdmin().listTables()) {
84        LOG.info("Tear down, remove table=" + htd.getTableName());
85        UTIL.deleteTable(htd.getTableName());
86      }
87    }
88  
89    @Test(timeout = 60000)
90    public void testModifyColumnFamily() throws Exception {
91      final TableName tableName = TableName.valueOf("testModifyColumnFamily");
92      final String cf1 = "cf1";
93      final HColumnDescriptor columnDescriptor = new HColumnDescriptor(cf1);
94      int oldBlockSize = columnDescriptor.getBlocksize();
95      int newBlockSize = 3 * oldBlockSize;
96  
97      final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
98  
99      MasterProcedureTestingUtility.createTable(procExec, tableName, null, cf1, "f2");
100 
101     // Test 1: modify the column family online
102     columnDescriptor.setBlocksize(newBlockSize);
103     long procId1 = procExec.submitProcedure(
104       new ModifyColumnFamilyProcedure(procExec.getEnvironment(), tableName, columnDescriptor),
105       nonceGroup,
106       nonce);
107     // Wait the completion
108     ProcedureTestingUtility.waitProcedure(procExec, procId1);
109     ProcedureTestingUtility.assertProcNotFailed(procExec, procId1);
110     MasterProcedureTestingUtility.validateColumnFamilyModification(UTIL.getHBaseCluster()
111         .getMaster(), tableName, cf1, columnDescriptor);
112 
113     // Test 2: modify the column family offline
114     UTIL.getHBaseAdmin().disableTable(tableName);
115     columnDescriptor.setBlocksize(newBlockSize * 2);
116     long procId2 = procExec.submitProcedure(
117       new ModifyColumnFamilyProcedure(procExec.getEnvironment(), tableName, columnDescriptor),
118       nonceGroup + 1,
119       nonce + 1);
120     // Wait the completion
121     ProcedureTestingUtility.waitProcedure(procExec, procId2);
122     ProcedureTestingUtility.assertProcNotFailed(procExec, procId2);
123     MasterProcedureTestingUtility.validateColumnFamilyModification(UTIL.getHBaseCluster()
124         .getMaster(), tableName, cf1, columnDescriptor);
125   }
126 
127   @Test(timeout=60000)
128   public void testModifyNonExistingColumnFamily() throws Exception {
129     final TableName tableName = TableName.valueOf("testModifyExistingColumnFamily");
130     final String cf2 = "cf2";
131     final HColumnDescriptor columnDescriptor = new HColumnDescriptor(cf2);
132     int oldBlockSize = columnDescriptor.getBlocksize();
133     int newBlockSize = 2 * oldBlockSize;
134 
135     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
136 
137     MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f1");
138 
139     // Modify the column family that does not exist
140     columnDescriptor.setBlocksize(newBlockSize);
141     long procId1 = procExec.submitProcedure(
142       new ModifyColumnFamilyProcedure(procExec.getEnvironment(), tableName, columnDescriptor),
143       nonceGroup,
144       nonce);
145     // Wait the completion
146     ProcedureTestingUtility.waitProcedure(procExec, procId1);
147 
148     ProcedureInfo result = procExec.getResult(procId1);
149     assertTrue(result.isFailed());
150     LOG.debug("Modify failed with exception: " + result.getExceptionFullMessage());
151     assertTrue(
152       ProcedureTestingUtility.getExceptionCause(result) instanceof InvalidFamilyOperationException);
153   }
154 
155   @Test(timeout=60000)
156   public void testRecoveryAndDoubleExecutionOffline() throws Exception {
157     final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecutionOffline");
158     final String cf3 = "cf3";
159     final HColumnDescriptor columnDescriptor = new HColumnDescriptor(cf3);
160     int oldBlockSize = columnDescriptor.getBlocksize();
161     int newBlockSize = 4 * oldBlockSize;
162 
163     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
164 
165     // create the table
166     MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f1", "f2", cf3);
167     UTIL.getHBaseAdmin().disableTable(tableName);
168     ProcedureTestingUtility.waitNoProcedureRunning(procExec);
169     ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
170 
171     // Start the Modify procedure && kill the executor
172     columnDescriptor.setBlocksize(newBlockSize);
173     long procId = procExec.submitProcedure(
174       new ModifyColumnFamilyProcedure(procExec.getEnvironment(), tableName, columnDescriptor),
175       nonceGroup,
176       nonce);
177 
178     // Restart the executor and execute the step twice
179     int numberOfSteps = ModifyColumnFamilyState.values().length;
180     MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(
181       procExec,
182       procId,
183       numberOfSteps,
184       ModifyColumnFamilyState.values());
185 
186     MasterProcedureTestingUtility.validateColumnFamilyModification(UTIL.getHBaseCluster()
187         .getMaster(), tableName, cf3, columnDescriptor);
188   }
189 
190   @Test(timeout = 60000)
191   public void testRecoveryAndDoubleExecutionOnline() throws Exception {
192     final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecutionOnline");
193     final String cf4 = "cf4";
194     final HColumnDescriptor columnDescriptor = new HColumnDescriptor(cf4);
195     int oldBlockSize = columnDescriptor.getBlocksize();
196     int newBlockSize = 4 * oldBlockSize;
197 
198     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
199 
200     // create the table
201     MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f1", "f2", cf4);
202     ProcedureTestingUtility.waitNoProcedureRunning(procExec);
203     ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
204 
205     // Start the Modify procedure && kill the executor
206     columnDescriptor.setBlocksize(newBlockSize);
207     long procId = procExec.submitProcedure(
208       new ModifyColumnFamilyProcedure(procExec.getEnvironment(), tableName, columnDescriptor),
209       nonceGroup,
210       nonce);
211 
212     // Restart the executor and execute the step twice
213     int numberOfSteps = ModifyColumnFamilyState.values().length;
214     MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, numberOfSteps,
215       ModifyColumnFamilyState.values());
216 
217     MasterProcedureTestingUtility.validateColumnFamilyModification(UTIL.getHBaseCluster()
218         .getMaster(), tableName, cf4, columnDescriptor);
219   }
220 
221   @Test(timeout = 60000)
222   public void testRollbackAndDoubleExecution() throws Exception {
223     final TableName tableName = TableName.valueOf("testRollbackAndDoubleExecution");
224     final String cf3 = "cf3";
225     final HColumnDescriptor columnDescriptor = new HColumnDescriptor(cf3);
226     int oldBlockSize = columnDescriptor.getBlocksize();
227     int newBlockSize = 4 * oldBlockSize;
228 
229     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
230 
231     // create the table
232     MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f1", "f2", cf3);
233     ProcedureTestingUtility.waitNoProcedureRunning(procExec);
234     ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
235 
236     // Start the Modify procedure && kill the executor
237     columnDescriptor.setBlocksize(newBlockSize);
238     long procId = procExec.submitProcedure(
239       new ModifyColumnFamilyProcedure(procExec.getEnvironment(), tableName, columnDescriptor),
240       nonceGroup,
241       nonce);
242 
243     // Failing in the middle of proc
244     int numberOfSteps = ModifyColumnFamilyState.values().length - 2;
245     MasterProcedureTestingUtility.testRollbackAndDoubleExecution(
246       procExec,
247       procId,
248       numberOfSteps,
249       ModifyColumnFamilyState.values());
250   }
251 
252   private ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
253     return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
254   }
255 }