View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.master.procedure;
20  
21  import java.io.IOException;
22  import java.io.InputStream;
23  import java.io.OutputStream;
24  import java.security.PrivilegedExceptionAction;
25  import java.util.HashSet;
26  import java.util.List;
27  import java.util.Set;
28  import java.util.concurrent.atomic.AtomicBoolean;
29  
30  import org.apache.commons.logging.Log;
31  import org.apache.commons.logging.LogFactory;
32  import org.apache.hadoop.hbase.HConstants;
33  import org.apache.hadoop.hbase.HRegionInfo;
34  import org.apache.hadoop.hbase.HTableDescriptor;
35  import org.apache.hadoop.hbase.MetaTableAccessor;
36  import org.apache.hadoop.hbase.TableName;
37  import org.apache.hadoop.hbase.TableNotDisabledException;
38  import org.apache.hadoop.hbase.TableNotFoundException;
39  import org.apache.hadoop.hbase.classification.InterfaceAudience;
40  import org.apache.hadoop.hbase.client.Connection;
41  import org.apache.hadoop.hbase.client.Result;
42  import org.apache.hadoop.hbase.client.ResultScanner;
43  import org.apache.hadoop.hbase.client.Scan;
44  import org.apache.hadoop.hbase.client.Table;
45  import org.apache.hadoop.hbase.executor.EventType;
46  import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
47  import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
48  import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos;
49  import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ModifyTableState;
50  import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
51  import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
52  import org.apache.hadoop.security.UserGroupInformation;
53  
54  @InterfaceAudience.Private
55  public class ModifyTableProcedure
56      extends StateMachineProcedure<MasterProcedureEnv, ModifyTableState>
57      implements TableProcedureInterface {
58    private static final Log LOG = LogFactory.getLog(ModifyTableProcedure.class);
59  
60    private final AtomicBoolean aborted = new AtomicBoolean(false);
61  
62    private HTableDescriptor unmodifiedHTableDescriptor = null;
63    private HTableDescriptor modifiedHTableDescriptor;
64    private UserGroupInformation user;
65    private boolean deleteColumnFamilyInModify;
66  
67    private List<HRegionInfo> regionInfoList;
68    private Boolean traceEnabled = null;
69  
70    public ModifyTableProcedure() {
71      initilize();
72    }
73  
74    public ModifyTableProcedure(
75      final MasterProcedureEnv env,
76      final HTableDescriptor htd) throws IOException {
77      initilize();
78      this.modifiedHTableDescriptor = htd;
79      this.user = env.getRequestUser().getUGI();
80      this.setOwner(this.user.getShortUserName());
81    }
82  
83    private void initilize() {
84      this.unmodifiedHTableDescriptor = null;
85      this.regionInfoList = null;
86      this.traceEnabled = null;
87      this.deleteColumnFamilyInModify = false;
88    }
89  
90    @Override
91    protected Flow executeFromState(final MasterProcedureEnv env, final ModifyTableState state)
92        throws InterruptedException {
93      if (isTraceEnabled()) {
94        LOG.trace(this + " execute state=" + state);
95      }
96  
97      try {
98        switch (state) {
99        case MODIFY_TABLE_PREPARE:
100         prepareModify(env);
101         setNextState(ModifyTableState.MODIFY_TABLE_PRE_OPERATION);
102         break;
103       case MODIFY_TABLE_PRE_OPERATION:
104         preModify(env, state);
105         setNextState(ModifyTableState.MODIFY_TABLE_UPDATE_TABLE_DESCRIPTOR);
106         break;
107       case MODIFY_TABLE_UPDATE_TABLE_DESCRIPTOR:
108         updateTableDescriptor(env);
109         setNextState(ModifyTableState.MODIFY_TABLE_REMOVE_REPLICA_COLUMN);
110         break;
111       case MODIFY_TABLE_REMOVE_REPLICA_COLUMN:
112         updateReplicaColumnsIfNeeded(env, unmodifiedHTableDescriptor, modifiedHTableDescriptor);
113         if (deleteColumnFamilyInModify) {
114           setNextState(ModifyTableState.MODIFY_TABLE_DELETE_FS_LAYOUT);
115         } else {
116           setNextState(ModifyTableState.MODIFY_TABLE_POST_OPERATION);
117         }
118         break;
119       case MODIFY_TABLE_DELETE_FS_LAYOUT:
120         deleteFromFs(env, unmodifiedHTableDescriptor, modifiedHTableDescriptor);
121         setNextState(ModifyTableState.MODIFY_TABLE_POST_OPERATION);
122         break;
123       case MODIFY_TABLE_POST_OPERATION:
124         postModify(env, state);
125         setNextState(ModifyTableState.MODIFY_TABLE_REOPEN_ALL_REGIONS);
126         break;
127       case MODIFY_TABLE_REOPEN_ALL_REGIONS:
128         reOpenAllRegionsIfTableIsOnline(env);
129         return Flow.NO_MORE_STATE;
130       default:
131         throw new UnsupportedOperationException("unhandled state=" + state);
132       }
133     } catch (IOException e) {
134       if (!isRollbackSupported(state)) {
135         // We reach a state that cannot be rolled back. We just need to keep retry.
136         LOG.warn("Error trying to modify table=" + getTableName() + " state=" + state, e);
137       } else {
138         LOG.error("Error trying to modify table=" + getTableName() + " state=" + state, e);
139         setFailure("master-modify-table", e);
140       }
141     }
142     return Flow.HAS_MORE_STATE;
143   }
144 
145   @Override
146   protected void rollbackState(final MasterProcedureEnv env, final ModifyTableState state)
147       throws IOException {
148     if (isTraceEnabled()) {
149       LOG.trace(this + " rollback state=" + state);
150     }
151     try {
152       switch (state) {
153       case MODIFY_TABLE_REOPEN_ALL_REGIONS:
154         break; // Nothing to undo.
155       case MODIFY_TABLE_POST_OPERATION:
156         // TODO-MAYBE: call the coprocessor event to un-modify?
157         break;
158       case MODIFY_TABLE_DELETE_FS_LAYOUT:
159         // Once we reach to this state - we could NOT rollback - as it is tricky to undelete
160         // the deleted files. We are not suppose to reach here, throw exception so that we know
161         // there is a code bug to investigate.
162         assert deleteColumnFamilyInModify;
163         throw new UnsupportedOperationException(this + " rollback of state=" + state
164             + " is unsupported.");
165       case MODIFY_TABLE_REMOVE_REPLICA_COLUMN:
166         // Undo the replica column update.
167         updateReplicaColumnsIfNeeded(env, modifiedHTableDescriptor, unmodifiedHTableDescriptor);
168         break;
169       case MODIFY_TABLE_UPDATE_TABLE_DESCRIPTOR:
170         restoreTableDescriptor(env);
171         break;
172       case MODIFY_TABLE_PRE_OPERATION:
173         // TODO-MAYBE: call the coprocessor event to un-modify?
174         break;
175       case MODIFY_TABLE_PREPARE:
176         break; // Nothing to undo.
177       default:
178         throw new UnsupportedOperationException("unhandled state=" + state);
179       }
180     } catch (IOException e) {
181       LOG.warn("Fail trying to rollback modify table=" + getTableName() + " state=" + state, e);
182       throw e;
183     }
184   }
185 
186   @Override
187   protected ModifyTableState getState(final int stateId) {
188     return ModifyTableState.valueOf(stateId);
189   }
190 
191   @Override
192   protected int getStateId(final ModifyTableState state) {
193     return state.getNumber();
194   }
195 
196   @Override
197   protected ModifyTableState getInitialState() {
198     return ModifyTableState.MODIFY_TABLE_PREPARE;
199   }
200 
201   @Override
202   protected void setNextState(final ModifyTableState state) {
203     if (aborted.get() && isRollbackSupported(state)) {
204       setAbortFailure("modify-table", "abort requested");
205     } else {
206       super.setNextState(state);
207     }
208   }
209 
210   @Override
211   public boolean abort(final MasterProcedureEnv env) {
212     aborted.set(true);
213     return true;
214   }
215 
216   @Override
217   protected boolean acquireLock(final MasterProcedureEnv env) {
218     if (env.waitInitialized(this)) return false;
219     return env.getProcedureQueue().tryAcquireTableExclusiveLock(this, getTableName());
220   }
221 
222   @Override
223   protected void releaseLock(final MasterProcedureEnv env) {
224     env.getProcedureQueue().releaseTableExclusiveLock(this, getTableName());
225   }
226 
227   @Override
228   public void serializeStateData(final OutputStream stream) throws IOException {
229     super.serializeStateData(stream);
230 
231     MasterProcedureProtos.ModifyTableStateData.Builder modifyTableMsg =
232         MasterProcedureProtos.ModifyTableStateData.newBuilder()
233             .setUserInfo(MasterProcedureUtil.toProtoUserInfo(user))
234             .setModifiedTableSchema(modifiedHTableDescriptor.convert())
235             .setDeleteColumnFamilyInModify(deleteColumnFamilyInModify);
236 
237     if (unmodifiedHTableDescriptor != null) {
238       modifyTableMsg.setUnmodifiedTableSchema(unmodifiedHTableDescriptor.convert());
239     }
240 
241     modifyTableMsg.build().writeDelimitedTo(stream);
242   }
243 
244   @Override
245   public void deserializeStateData(final InputStream stream) throws IOException {
246     super.deserializeStateData(stream);
247 
248     MasterProcedureProtos.ModifyTableStateData modifyTableMsg =
249         MasterProcedureProtos.ModifyTableStateData.parseDelimitedFrom(stream);
250     user = MasterProcedureUtil.toUserInfo(modifyTableMsg.getUserInfo());
251     modifiedHTableDescriptor = HTableDescriptor.convert(modifyTableMsg.getModifiedTableSchema());
252     deleteColumnFamilyInModify = modifyTableMsg.getDeleteColumnFamilyInModify();
253 
254     if (modifyTableMsg.hasUnmodifiedTableSchema()) {
255       unmodifiedHTableDescriptor =
256           HTableDescriptor.convert(modifyTableMsg.getUnmodifiedTableSchema());
257     }
258   }
259 
260   @Override
261   public void toStringClassDetails(StringBuilder sb) {
262     sb.append(getClass().getSimpleName());
263     sb.append(" (table=");
264     sb.append(getTableName());
265     sb.append(")");
266   }
267 
268   @Override
269   public TableName getTableName() {
270     return modifiedHTableDescriptor.getTableName();
271   }
272 
273   @Override
274   public TableOperationType getTableOperationType() {
275     return TableOperationType.EDIT;
276   }
277 
278   /**
279    * Check conditions before any real action of modifying a table.
280    * @param env MasterProcedureEnv
281    * @throws IOException
282    */
283   private void prepareModify(final MasterProcedureEnv env) throws IOException {
284     // Checks whether the table exists
285     if (!MetaTableAccessor.tableExists(env.getMasterServices().getConnection(), getTableName())) {
286       throw new TableNotFoundException(getTableName());
287     }
288 
289     // In order to update the descriptor, we need to retrieve the old descriptor for comparison.
290     this.unmodifiedHTableDescriptor =
291         env.getMasterServices().getTableDescriptors().get(getTableName());
292 
293     if (env.getMasterServices().getAssignmentManager().getTableStateManager()
294         .isTableState(getTableName(), ZooKeeperProtos.Table.State.ENABLED)) {
295       // We only execute this procedure with table online if online schema change config is set.
296       if (!MasterDDLOperationHelper.isOnlineSchemaChangeAllowed(env)) {
297         throw new TableNotDisabledException(getTableName());
298       }
299 
300       if (modifiedHTableDescriptor.getRegionReplication() != unmodifiedHTableDescriptor
301           .getRegionReplication()) {
302         throw new IOException("REGION_REPLICATION change is not supported for enabled tables");
303       }
304     }
305 
306     // Find out whether all column families in unmodifiedHTableDescriptor also exists in
307     // the modifiedHTableDescriptor. This is to determine whether we are safe to rollback.
308     final Set<byte[]> oldFamilies = unmodifiedHTableDescriptor.getFamiliesKeys();
309     final Set<byte[]> newFamilies = modifiedHTableDescriptor.getFamiliesKeys();
310     for (byte[] familyName : oldFamilies) {
311       if (!newFamilies.contains(familyName)) {
312         this.deleteColumnFamilyInModify = true;
313         break;
314       }
315     }
316   }
317 
318   /**
319    * Action before modifying table.
320    * @param env MasterProcedureEnv
321    * @param state the procedure state
322    * @throws IOException
323    * @throws InterruptedException
324    */
325   private void preModify(final MasterProcedureEnv env, final ModifyTableState state)
326       throws IOException, InterruptedException {
327     runCoprocessorAction(env, state);
328   }
329 
330   /**
331    * Update descriptor
332    * @param env MasterProcedureEnv
333    * @throws IOException
334    **/
335   private void updateTableDescriptor(final MasterProcedureEnv env) throws IOException {
336     env.getMasterServices().getTableDescriptors().add(modifiedHTableDescriptor);
337   }
338 
339   /**
340    * Undo the descriptor change (for rollback)
341    * @param env MasterProcedureEnv
342    * @throws IOException
343    **/
344   private void restoreTableDescriptor(final MasterProcedureEnv env) throws IOException {
345     env.getMasterServices().getTableDescriptors().add(unmodifiedHTableDescriptor);
346 
347     // delete any new column families from the modifiedHTableDescriptor.
348     deleteFromFs(env, modifiedHTableDescriptor, unmodifiedHTableDescriptor);
349 
350     // Make sure regions are opened after table descriptor is updated.
351     reOpenAllRegionsIfTableIsOnline(env);
352   }
353 
354   /**
355    * Removes from hdfs the families that are not longer present in the new table descriptor.
356    * @param env MasterProcedureEnv
357    * @throws IOException
358    */
359   private void deleteFromFs(final MasterProcedureEnv env,
360       final HTableDescriptor oldHTableDescriptor, final HTableDescriptor newHTableDescriptor)
361       throws IOException {
362     final Set<byte[]> oldFamilies = oldHTableDescriptor.getFamiliesKeys();
363     final Set<byte[]> newFamilies = newHTableDescriptor.getFamiliesKeys();
364     for (byte[] familyName : oldFamilies) {
365       if (!newFamilies.contains(familyName)) {
366         MasterDDLOperationHelper.deleteColumnFamilyFromFileSystem(
367           env,
368           getTableName(),
369           getRegionInfoList(env),
370           familyName);
371       }
372     }
373   }
374 
375   /**
376    * update replica column families if necessary.
377    * @param env MasterProcedureEnv
378    * @throws IOException
379    */
380   private void updateReplicaColumnsIfNeeded(
381     final MasterProcedureEnv env,
382     final HTableDescriptor oldHTableDescriptor,
383     final HTableDescriptor newHTableDescriptor) throws IOException {
384     final int oldReplicaCount = oldHTableDescriptor.getRegionReplication();
385     final int newReplicaCount = newHTableDescriptor.getRegionReplication();
386 
387     if (newReplicaCount < oldReplicaCount) {
388       Set<byte[]> tableRows = new HashSet<byte[]>();
389       Connection connection = env.getMasterServices().getConnection();
390       Scan scan = MetaTableAccessor.getScanForTableName(getTableName());
391       scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
392 
393       try (Table metaTable = connection.getTable(TableName.META_TABLE_NAME)) {
394         ResultScanner resScanner = metaTable.getScanner(scan);
395         for (Result result : resScanner) {
396           tableRows.add(result.getRow());
397         }
398         MetaTableAccessor.removeRegionReplicasFromMeta(
399           tableRows,
400           newReplicaCount,
401           oldReplicaCount - newReplicaCount,
402           connection);
403       }
404     }
405 
406     // Setup replication for region replicas if needed
407     if (newReplicaCount > 1 && oldReplicaCount <= 1) {
408       ServerRegionReplicaUtil.setupRegionReplicaReplication(env.getMasterConfiguration());
409     }
410   }
411 
412   /**
413    * Action after modifying table.
414    * @param env MasterProcedureEnv
415    * @param state the procedure state
416    * @throws IOException
417    * @throws InterruptedException
418    */
419   private void postModify(final MasterProcedureEnv env, final ModifyTableState state)
420       throws IOException, InterruptedException {
421     runCoprocessorAction(env, state);
422   }
423 
424   /**
425    * Last action from the procedure - executed when online schema change is supported.
426    * @param env MasterProcedureEnv
427    * @throws IOException
428    */
429   private void reOpenAllRegionsIfTableIsOnline(final MasterProcedureEnv env) throws IOException {
430     // This operation only run when the table is enabled.
431     if (!env.getMasterServices().getAssignmentManager().getTableStateManager()
432         .isTableState(getTableName(), ZooKeeperProtos.Table.State.ENABLED)) {
433       return;
434     }
435 
436     if (MasterDDLOperationHelper.reOpenAllRegions(env, getTableName(), getRegionInfoList(env))) {
437       LOG.info("Completed modify table operation on table " + getTableName());
438     } else {
439       LOG.warn("Error on reopening the regions on table " + getTableName());
440     }
441   }
442 
443   /**
444    * The procedure could be restarted from a different machine. If the variable is null, we need to
445    * retrieve it.
446    * @return traceEnabled whether the trace is enabled
447    */
448   private Boolean isTraceEnabled() {
449     if (traceEnabled == null) {
450       traceEnabled = LOG.isTraceEnabled();
451     }
452     return traceEnabled;
453   }
454 
455   /**
456    * Coprocessor Action.
457    * @param env MasterProcedureEnv
458    * @param state the procedure state
459    * @throws IOException
460    * @throws InterruptedException
461    */
462   private void runCoprocessorAction(final MasterProcedureEnv env, final ModifyTableState state)
463       throws IOException, InterruptedException {
464     final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
465     if (cpHost != null) {
466       user.doAs(new PrivilegedExceptionAction<Void>() {
467         @Override
468         public Void run() throws Exception {
469           switch (state) {
470           case MODIFY_TABLE_PRE_OPERATION:
471             cpHost.preModifyTableHandler(getTableName(), modifiedHTableDescriptor);
472             break;
473           case MODIFY_TABLE_POST_OPERATION:
474             cpHost.postModifyTableHandler(getTableName(), modifiedHTableDescriptor);
475             break;
476           default:
477             throw new UnsupportedOperationException(this + " unhandled state=" + state);
478           }
479           return null;
480         }
481       });
482     }
483   }
484 
485   /*
486    * Check whether we are in the state that can be rollback
487    */
488   private boolean isRollbackSupported(final ModifyTableState state) {
489     if (deleteColumnFamilyInModify) {
490       switch (state) {
491       case MODIFY_TABLE_DELETE_FS_LAYOUT:
492       case MODIFY_TABLE_POST_OPERATION:
493       case MODIFY_TABLE_REOPEN_ALL_REGIONS:
494         // It is not safe to rollback if we reach to these states.
495         return false;
496       default:
497         break;
498       }
499     }
500     return true;
501   }
502 
503   private List<HRegionInfo> getRegionInfoList(final MasterProcedureEnv env) throws IOException {
504     if (regionInfoList == null) {
505       regionInfoList = ProcedureSyncWait.getRegionsFromMeta(env, getTableName());
506     }
507     return regionInfoList;
508   }
509 }