View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.master.procedure;
20  
21  import static org.junit.Assert.assertTrue;
22  
23  import org.apache.commons.logging.Log;
24  import org.apache.commons.logging.LogFactory;
25  import org.apache.hadoop.conf.Configuration;
26  import org.apache.hadoop.hbase.HBaseTestingUtility;
27  import org.apache.hadoop.hbase.HConstants;
28  import org.apache.hadoop.hbase.HRegionInfo;
29  import org.apache.hadoop.hbase.HTableDescriptor;
30  import org.apache.hadoop.hbase.InvalidFamilyOperationException;
31  import org.apache.hadoop.hbase.ProcedureInfo;
32  import org.apache.hadoop.hbase.TableName;
33  import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
34  import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
35  import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.DeleteColumnFamilyState;
36  import org.apache.hadoop.hbase.testclassification.MediumTests;
37  import org.junit.After;
38  import org.junit.AfterClass;
39  import org.junit.Before;
40  import org.junit.BeforeClass;
41  import org.junit.Test;
42  import org.junit.experimental.categories.Category;
43  
44  @Category(MediumTests.class)
45  public class TestDeleteColumnFamilyProcedure {
46    private static final Log LOG = LogFactory.getLog(TestDeleteColumnFamilyProcedure.class);
47  
48    protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
49  
50    private static long nonceGroup = HConstants.NO_NONCE;
51    private static long nonce = HConstants.NO_NONCE;
52  
53    private static void setupConf(Configuration conf) {
54      conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
55    }
56  
57    @BeforeClass
58    public static void setupCluster() throws Exception {
59      setupConf(UTIL.getConfiguration());
60      UTIL.startMiniCluster(1);
61    }
62  
63    @AfterClass
64    public static void cleanupTest() throws Exception {
65      try {
66        UTIL.shutdownMiniCluster();
67      } catch (Exception e) {
68        LOG.warn("failure shutting down cluster", e);
69      }
70    }
71  
72    @Before
73    public void setup() throws Exception {
74      ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
75      nonceGroup =
76          MasterProcedureTestingUtility.generateNonceGroup(UTIL.getHBaseCluster().getMaster());
77      nonce = MasterProcedureTestingUtility.generateNonce(UTIL.getHBaseCluster().getMaster());
78    }
79  
80    @After
81    public void tearDown() throws Exception {
82      ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
83      for (HTableDescriptor htd: UTIL.getHBaseAdmin().listTables()) {
84        LOG.info("Tear down, remove table=" + htd.getTableName());
85        UTIL.deleteTable(htd.getTableName());
86      }
87    }
88  
89    @Test(timeout = 60000)
90    public void testDeleteColumnFamily() throws Exception {
91      final TableName tableName = TableName.valueOf("testDeleteColumnFamily");
92      final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
93      final String cf1 = "cf1";
94      final String cf2 = "cf2";
95  
96      MasterProcedureTestingUtility.createTable(procExec, tableName, null, cf1, cf2, "f3");
97  
98      // Test 1: delete the column family that exists online
99      long procId1 = procExec.submitProcedure(
100       new DeleteColumnFamilyProcedure(procExec.getEnvironment(), tableName, cf1.getBytes()),
101       nonceGroup,
102       nonce);
103     // Wait the completion
104     ProcedureTestingUtility.waitProcedure(procExec, procId1);
105     ProcedureTestingUtility.assertProcNotFailed(procExec, procId1);
106 
107     MasterProcedureTestingUtility.validateColumnFamilyDeletion(UTIL.getHBaseCluster().getMaster(),
108       tableName, cf1);
109 
110     // Test 2: delete the column family that exists offline
111     UTIL.getHBaseAdmin().disableTable(tableName);
112     long procId2 = procExec.submitProcedure(
113       new DeleteColumnFamilyProcedure(procExec.getEnvironment(), tableName, cf2.getBytes()),
114       nonceGroup,
115       nonce);
116     // Wait the completion
117     ProcedureTestingUtility.waitProcedure(procExec, procId2);
118     ProcedureTestingUtility.assertProcNotFailed(procExec, procId2);
119   }
120 
121   @Test(timeout=60000)
122   public void testDeleteColumnFamilyTwice() throws Exception {
123     final TableName tableName = TableName.valueOf("testDeleteColumnFamilyTwice");
124     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
125 
126     final String cf2 = "cf2";
127 
128     MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f1", cf2);
129 
130     // delete the column family that exists
131     long procId1 = procExec.submitProcedure(
132       new DeleteColumnFamilyProcedure(procExec.getEnvironment(), tableName, cf2.getBytes()),
133       nonceGroup,
134       nonce);
135     // Wait the completion
136     ProcedureTestingUtility.waitProcedure(procExec, procId1);
137     // First delete should succeed
138     ProcedureTestingUtility.assertProcNotFailed(procExec, procId1);
139 
140     MasterProcedureTestingUtility.validateColumnFamilyDeletion(UTIL.getHBaseCluster().getMaster(),
141       tableName, cf2);
142 
143     // delete the column family that does not exist
144     long procId2 = procExec.submitProcedure(
145       new DeleteColumnFamilyProcedure(procExec.getEnvironment(), tableName, cf2.getBytes()),
146       nonceGroup + 1,
147       nonce + 1);
148 
149     // Wait the completion
150     ProcedureTestingUtility.waitProcedure(procExec, procId2);
151 
152     // Second delete should fail with InvalidFamilyOperationException
153     ProcedureInfo result = procExec.getResult(procId2);
154     assertTrue(result.isFailed());
155     LOG.debug("Delete online failed with exception: " + result.getExceptionFullMessage());
156     assertTrue(
157       ProcedureTestingUtility.getExceptionCause(result) instanceof InvalidFamilyOperationException);
158 
159     // Try again, this time with table disabled.
160     UTIL.getHBaseAdmin().disableTable(tableName);
161     long procId3 = procExec.submitProcedure(
162       new DeleteColumnFamilyProcedure(procExec.getEnvironment(), tableName, cf2.getBytes()),
163       nonceGroup + 2,
164       nonce + 2);
165     // Wait the completion
166     ProcedureTestingUtility.waitProcedure(procExec, procId3);
167     // Expect fail with InvalidFamilyOperationException
168     result = procExec.getResult(procId2);
169     assertTrue(result.isFailed());
170     LOG.debug("Delete offline failed with exception: " + result.getExceptionFullMessage());
171     assertTrue(
172       ProcedureTestingUtility.getExceptionCause(result) instanceof InvalidFamilyOperationException);
173   }
174 
175   @Test(timeout=60000)
176   public void testDeleteColumnFamilyTwiceWithSameNonce() throws Exception {
177     final TableName tableName = TableName.valueOf("testDeleteColumnFamilyTwiceWithSameNonce");
178     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
179 
180     final String cf2 = "cf2";
181 
182     MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f1", cf2);
183 
184     // delete the column family that exists
185     long procId1 = procExec.submitProcedure(
186       new DeleteColumnFamilyProcedure(procExec.getEnvironment(), tableName, cf2.getBytes()),
187       nonceGroup,
188       nonce);
189     long procId2 = procExec.submitProcedure(
190       new DeleteColumnFamilyProcedure(procExec.getEnvironment(), tableName, cf2.getBytes()),
191       nonceGroup,
192       nonce);
193 
194     // Wait the completion
195     ProcedureTestingUtility.waitProcedure(procExec, procId1);
196     ProcedureTestingUtility.assertProcNotFailed(procExec, procId1);
197     MasterProcedureTestingUtility.validateColumnFamilyDeletion(UTIL.getHBaseCluster().getMaster(),
198       tableName, cf2);
199 
200     // Wait the completion and expect not fail - because it is the same proc
201     ProcedureTestingUtility.waitProcedure(procExec, procId2);
202     ProcedureTestingUtility.assertProcNotFailed(procExec, procId2);
203     assertTrue(procId1 == procId2);
204   }
205 
206   @Test(timeout=60000)
207   public void testDeleteNonExistingColumnFamily() throws Exception {
208     final TableName tableName = TableName.valueOf("testDeleteNonExistingColumnFamily");
209     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
210 
211     final String cf3 = "cf3";
212 
213     MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f1", "f2");
214 
215     // delete the column family that does not exist
216     long procId1 = procExec.submitProcedure(
217       new DeleteColumnFamilyProcedure(procExec.getEnvironment(), tableName, cf3.getBytes()),
218       nonceGroup,
219       nonce);
220     // Wait the completion
221     ProcedureTestingUtility.waitProcedure(procExec, procId1);
222 
223     ProcedureInfo result = procExec.getResult(procId1);
224     assertTrue(result.isFailed());
225     LOG.debug("Delete failed with exception: " + result.getExceptionFullMessage());
226     assertTrue(
227       ProcedureTestingUtility.getExceptionCause(result) instanceof InvalidFamilyOperationException);
228   }
229 
230   @Test(timeout=60000)
231   public void testRecoveryAndDoubleExecutionOffline() throws Exception {
232     final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecutionOffline");
233     final String cf4 = "cf4";
234 
235     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
236 
237     // create the table
238     MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f1", "f2", "f3", cf4);
239     UTIL.getHBaseAdmin().disableTable(tableName);
240     ProcedureTestingUtility.waitNoProcedureRunning(procExec);
241     ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
242 
243     // Start the Delete procedure && kill the executor
244     long procId = procExec.submitProcedure(
245       new DeleteColumnFamilyProcedure(procExec.getEnvironment(), tableName, cf4.getBytes()),
246       nonceGroup,
247       nonce);
248 
249     // Restart the executor and execute the step twice
250     int numberOfSteps = DeleteColumnFamilyState.values().length;
251     MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, numberOfSteps,
252       DeleteColumnFamilyState.values());
253 
254     MasterProcedureTestingUtility.validateColumnFamilyDeletion(UTIL.getHBaseCluster().getMaster(),
255       tableName, cf4);
256   }
257 
258   @Test(timeout = 60000)
259   public void testRecoveryAndDoubleExecutionOnline() throws Exception {
260     final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecutionOnline");
261     final String cf5 = "cf5";
262 
263     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
264 
265     // create the table
266     MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f1", "f2", "f3", cf5);
267     ProcedureTestingUtility.waitNoProcedureRunning(procExec);
268     ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
269 
270     // Start the Delete procedure && kill the executor
271     long procId = procExec.submitProcedure(
272       new DeleteColumnFamilyProcedure(procExec.getEnvironment(), tableName, cf5.getBytes()),
273       nonceGroup,
274       nonce);
275 
276     // Restart the executor and execute the step twice
277     int numberOfSteps = DeleteColumnFamilyState.values().length;
278     MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, numberOfSteps,
279       DeleteColumnFamilyState.values());
280 
281     MasterProcedureTestingUtility.validateColumnFamilyDeletion(UTIL.getHBaseCluster().getMaster(),
282       tableName, cf5);
283   }
284 
285   @Test(timeout = 60000)
286   public void testRollbackAndDoubleExecution() throws Exception {
287     final TableName tableName = TableName.valueOf("testRollbackAndDoubleExecution");
288     final String cf5 = "cf5";
289 
290     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
291 
292     // create the table
293     HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
294       procExec, tableName, null, "f1", "f2", "f3", cf5);
295     ProcedureTestingUtility.waitNoProcedureRunning(procExec);
296     ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
297 
298     // Start the Delete procedure && kill the executor
299     long procId = procExec.submitProcedure(
300       new DeleteColumnFamilyProcedure(procExec.getEnvironment(), tableName, cf5.getBytes()),
301       nonceGroup,
302       nonce);
303 
304     // Failing before DELETE_COLUMN_FAMILY_DELETE_FS_LAYOUT we should trigger the rollback
305     // NOTE: the 1 (number before DELETE_COLUMN_FAMILY_DELETE_FS_LAYOUT step) is hardcoded,
306     //       so you have to look at this test at least once when you add a new step.
307     int numberOfSteps = 1;
308     MasterProcedureTestingUtility.testRollbackAndDoubleExecution(
309       procExec,
310       procId,
311       numberOfSteps,
312       DeleteColumnFamilyState.values());
313 
314     MasterProcedureTestingUtility.validateTableCreation(
315       UTIL.getHBaseCluster().getMaster(), tableName, regions, "f1", "f2", "f3", cf5);
316   }
317 
318   @Test(timeout = 60000)
319   public void testRollbackAndDoubleExecutionAfterPONR() throws Exception {
320     final TableName tableName = TableName.valueOf("testRollbackAndDoubleExecutionAfterPONR");
321     final String cf5 = "cf5";
322 
323     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
324 
325     // create the table
326     HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
327       procExec, tableName, null, "f1", "f2", "f3", cf5);
328     ProcedureTestingUtility.waitNoProcedureRunning(procExec);
329     ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
330 
331     // Start the Delete procedure && kill the executor
332     long procId = procExec.submitProcedure(
333       new DeleteColumnFamilyProcedure(procExec.getEnvironment(), tableName, cf5.getBytes()),
334       nonceGroup,
335       nonce);
336 
337     // Failing after DELETE_COLUMN_FAMILY_DELETE_FS_LAYOUT we should not trigger the rollback.
338     // NOTE: the 4 (number of DELETE_COLUMN_FAMILY_DELETE_FS_LAYOUT + 1 step) is hardcoded,
339     //       so you have to look at this test at least once when you add a new step.
340     int numberOfSteps = 4;
341     MasterProcedureTestingUtility.testRollbackAndDoubleExecutionAfterPONR(
342       procExec,
343       procId,
344       numberOfSteps,
345       DeleteColumnFamilyState.values());
346 
347     MasterProcedureTestingUtility.validateColumnFamilyDeletion(
348       UTIL.getHBaseCluster().getMaster(), tableName, cf5);
349   }
350 
351   private ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
352     return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
353   }
354 }