View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.master.procedure;
20  
21  import java.io.IOException;
22  import java.io.InputStream;
23  import java.io.OutputStream;
24  import java.security.PrivilegedExceptionAction;
25  import java.util.ArrayList;
26  import java.util.List;
27  import java.util.concurrent.atomic.AtomicBoolean;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.fs.FileSystem;
32  import org.apache.hadoop.fs.Path;
33  import org.apache.hadoop.hbase.HRegionInfo;
34  import org.apache.hadoop.hbase.HTableDescriptor;
35  import org.apache.hadoop.hbase.MetaTableAccessor;
36  import org.apache.hadoop.hbase.TableExistsException;
37  import org.apache.hadoop.hbase.TableName;
38  import org.apache.hadoop.hbase.TableStateManager;
39  import org.apache.hadoop.hbase.classification.InterfaceAudience;
40  import org.apache.hadoop.hbase.client.RegionReplicaUtil;
41  import org.apache.hadoop.hbase.exceptions.HBaseException;
42  import org.apache.hadoop.hbase.master.AssignmentManager;
43  import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
44  import org.apache.hadoop.hbase.master.MasterFileSystem;
45  import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
46  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
47  import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos;
48  import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.CreateTableState;
49  import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
50  import org.apache.hadoop.hbase.util.FSTableDescriptors;
51  import org.apache.hadoop.hbase.util.FSUtils;
52  import org.apache.hadoop.hbase.util.ModifyRegionUtils;
53  import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
54  import org.apache.hadoop.security.UserGroupInformation;
55  
56  import com.google.common.collect.Lists;
57  
58  @InterfaceAudience.Private
59  public class CreateTableProcedure
60      extends StateMachineProcedure<MasterProcedureEnv, CreateTableState>
61      implements TableProcedureInterface {
62    private static final Log LOG = LogFactory.getLog(CreateTableProcedure.class);
63  
64    private final AtomicBoolean aborted = new AtomicBoolean(false);
65  
66    // used for compatibility with old clients
67    private final ProcedurePrepareLatch syncLatch;
68  
69    private HTableDescriptor hTableDescriptor;
70    private List<HRegionInfo> newRegions;
71    private UserGroupInformation user;
72  
73    public CreateTableProcedure() {
74      // Required by the Procedure framework to create the procedure on replay
75      syncLatch = null;
76    }
77  
78    public CreateTableProcedure(final MasterProcedureEnv env,
79        final HTableDescriptor hTableDescriptor, final HRegionInfo[] newRegions)
80        throws IOException {
81      this(env, hTableDescriptor, newRegions, null);
82    }
83  
84    public CreateTableProcedure(final MasterProcedureEnv env,
85        final HTableDescriptor hTableDescriptor, final HRegionInfo[] newRegions,
86        final ProcedurePrepareLatch syncLatch)
87        throws IOException {
88      this.hTableDescriptor = hTableDescriptor;
89      this.newRegions = newRegions != null ? Lists.newArrayList(newRegions) : null;
90      this.user = env.getRequestUser().getUGI();
91      this.setOwner(this.user.getShortUserName());
92  
93      // used for compatibility with clients without procedures
94      // they need a sync TableExistsException
95      this.syncLatch = syncLatch;
96    }
97  
98    @Override
99    protected Flow executeFromState(final MasterProcedureEnv env, final CreateTableState state)
100       throws InterruptedException {
101     if (LOG.isTraceEnabled()) {
102       LOG.trace(this + " execute state=" + state);
103     }
104     try {
105       switch (state) {
106         case CREATE_TABLE_PRE_OPERATION:
107           // Verify if we can create the table
108           boolean exists = !prepareCreate(env);
109           ProcedurePrepareLatch.releaseLatch(syncLatch, this);
110 
111           if (exists) {
112             assert isFailed() : "the delete should have an exception here";
113             return Flow.NO_MORE_STATE;
114           }
115 
116           preCreate(env);
117           setNextState(CreateTableState.CREATE_TABLE_WRITE_FS_LAYOUT);
118           break;
119         case CREATE_TABLE_WRITE_FS_LAYOUT:
120           newRegions = createFsLayout(env, hTableDescriptor, newRegions);
121           setNextState(CreateTableState.CREATE_TABLE_ADD_TO_META);
122           break;
123         case CREATE_TABLE_ADD_TO_META:
124           newRegions = addTableToMeta(env, hTableDescriptor, newRegions);
125           setNextState(CreateTableState.CREATE_TABLE_ASSIGN_REGIONS);
126           break;
127         case CREATE_TABLE_ASSIGN_REGIONS:
128           assignRegions(env, getTableName(), newRegions);
129           setNextState(CreateTableState.CREATE_TABLE_UPDATE_DESC_CACHE);
130           break;
131         case CREATE_TABLE_UPDATE_DESC_CACHE:
132           updateTableDescCache(env, getTableName());
133           setNextState(CreateTableState.CREATE_TABLE_POST_OPERATION);
134           break;
135         case CREATE_TABLE_POST_OPERATION:
136           postCreate(env);
137           return Flow.NO_MORE_STATE;
138         default:
139           throw new UnsupportedOperationException("unhandled state=" + state);
140       }
141     } catch (HBaseException|IOException e) {
142       LOG.error("Error trying to create table=" + getTableName() + " state=" + state, e);
143       setFailure("master-create-table", e);
144     }
145     return Flow.HAS_MORE_STATE;
146   }
147 
148   @Override
149   protected void rollbackState(final MasterProcedureEnv env, final CreateTableState state)
150       throws IOException {
151     if (LOG.isTraceEnabled()) {
152       LOG.trace(this + " rollback state=" + state);
153     }
154     try {
155       switch (state) {
156         case CREATE_TABLE_POST_OPERATION:
157           break;
158         case CREATE_TABLE_UPDATE_DESC_CACHE:
159           DeleteTableProcedure.deleteTableDescriptorCache(env, getTableName());
160           break;
161         case CREATE_TABLE_ASSIGN_REGIONS:
162           DeleteTableProcedure.deleteAssignmentState(env, getTableName());
163           break;
164         case CREATE_TABLE_ADD_TO_META:
165           DeleteTableProcedure.deleteFromMeta(env, getTableName(), newRegions);
166           break;
167         case CREATE_TABLE_WRITE_FS_LAYOUT:
168           DeleteTableProcedure.deleteFromFs(env, getTableName(), newRegions, false);
169           break;
170         case CREATE_TABLE_PRE_OPERATION:
171           DeleteTableProcedure.deleteTableStates(env, getTableName());
172           // TODO-MAYBE: call the deleteTable coprocessor event?
173           ProcedurePrepareLatch.releaseLatch(syncLatch, this);
174           break;
175         default:
176           throw new UnsupportedOperationException("unhandled state=" + state);
177       }
178     } catch (HBaseException e) {
179       LOG.warn("Failed rollback attempt step=" + state + " table=" + getTableName(), e);
180       throw new IOException(e);
181     } catch (IOException e) {
182       // This will be retried. Unless there is a bug in the code,
183       // this should be just a "temporary error" (e.g. network down)
184       LOG.warn("Failed rollback attempt step=" + state + " table=" + getTableName(), e);
185       throw e;
186     }
187   }
188 
189   @Override
190   protected CreateTableState getState(final int stateId) {
191     return CreateTableState.valueOf(stateId);
192   }
193 
194   @Override
195   protected int getStateId(final CreateTableState state) {
196     return state.getNumber();
197   }
198 
199   @Override
200   protected CreateTableState getInitialState() {
201     return CreateTableState.CREATE_TABLE_PRE_OPERATION;
202   }
203 
204   @Override
205   protected void setNextState(final CreateTableState state) {
206     if (aborted.get()) {
207       setAbortFailure("create-table", "abort requested");
208     } else {
209       super.setNextState(state);
210     }
211   }
212 
213   @Override
214   public TableName getTableName() {
215     return hTableDescriptor.getTableName();
216   }
217 
218   @Override
219   public TableOperationType getTableOperationType() {
220     return TableOperationType.CREATE;
221   }
222 
223   @Override
224   public boolean abort(final MasterProcedureEnv env) {
225     aborted.set(true);
226     return true;
227   }
228 
229   @Override
230   public void toStringClassDetails(StringBuilder sb) {
231     sb.append(getClass().getSimpleName());
232     sb.append(" (table=");
233     sb.append(getTableName());
234     sb.append(")");
235   }
236 
237   @Override
238   public void serializeStateData(final OutputStream stream) throws IOException {
239     super.serializeStateData(stream);
240 
241     MasterProcedureProtos.CreateTableStateData.Builder state =
242       MasterProcedureProtos.CreateTableStateData.newBuilder()
243         .setUserInfo(MasterProcedureUtil.toProtoUserInfo(this.user))
244         .setTableSchema(hTableDescriptor.convert());
245     if (newRegions != null) {
246       for (HRegionInfo hri: newRegions) {
247         state.addRegionInfo(HRegionInfo.convert(hri));
248       }
249     }
250     state.build().writeDelimitedTo(stream);
251   }
252 
253   @Override
254   public void deserializeStateData(final InputStream stream) throws IOException {
255     super.deserializeStateData(stream);
256 
257     MasterProcedureProtos.CreateTableStateData state =
258       MasterProcedureProtos.CreateTableStateData.parseDelimitedFrom(stream);
259     user = MasterProcedureUtil.toUserInfo(state.getUserInfo());
260     hTableDescriptor = HTableDescriptor.convert(state.getTableSchema());
261     if (state.getRegionInfoCount() == 0) {
262       newRegions = null;
263     } else {
264       newRegions = new ArrayList<HRegionInfo>(state.getRegionInfoCount());
265       for (HBaseProtos.RegionInfo hri: state.getRegionInfoList()) {
266         newRegions.add(HRegionInfo.convert(hri));
267       }
268     }
269   }
270 
271   @Override
272   protected boolean acquireLock(final MasterProcedureEnv env) {
273     if (!getTableName().isSystemTable() && env.waitInitialized(this)) {
274       return false;
275     }
276     return env.getProcedureQueue().tryAcquireTableExclusiveLock(this, getTableName());
277   }
278 
279   @Override
280   protected void releaseLock(final MasterProcedureEnv env) {
281     env.getProcedureQueue().releaseTableExclusiveLock(this, getTableName());
282   }
283 
284   private boolean prepareCreate(final MasterProcedureEnv env) throws IOException {
285     final TableName tableName = getTableName();
286     if (MetaTableAccessor.tableExists(env.getMasterServices().getConnection(), tableName)) {
287       setFailure("master-create-table", new TableExistsException(getTableName()));
288       return false;
289     }
290     // During master initialization, the ZK state could be inconsistent from failed DDL
291     // in the past. If we fail here, it would prevent master to start.  We should force
292     // setting the system table state regardless the table state.
293     boolean skipTableStateCheck =
294         !(env.getMasterServices().isInitialized()) && tableName.isSystemTable();
295     if (!skipTableStateCheck) {
296       TableStateManager tsm = env.getMasterServices().getAssignmentManager().getTableStateManager();
297       if (tsm.isTableState(tableName, true, ZooKeeperProtos.Table.State.ENABLING,
298           ZooKeeperProtos.Table.State.ENABLED)) {
299         LOG.warn("The table " + tableName + " does not exist in meta but has a znode. " +
300                "run hbck to fix inconsistencies.");
301         setFailure("master-create-table", new TableExistsException(getTableName()));
302         return false;
303       }
304     }
305     return true;
306   }
307 
308   private void preCreate(final MasterProcedureEnv env)
309       throws IOException, InterruptedException {
310     if (!getTableName().isSystemTable()) {
311       ProcedureSyncWait.getMasterQuotaManager(env)
312         .checkNamespaceTableAndRegionQuota(getTableName(), newRegions.size());
313     }
314 
315     final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
316     if (cpHost != null) {
317       final HRegionInfo[] regions = newRegions == null ? null :
318         newRegions.toArray(new HRegionInfo[newRegions.size()]);
319       user.doAs(new PrivilegedExceptionAction<Void>() {
320         @Override
321         public Void run() throws Exception {
322           cpHost.preCreateTableHandler(hTableDescriptor, regions);
323           return null;
324         }
325       });
326     }
327   }
328 
329   private void postCreate(final MasterProcedureEnv env)
330       throws IOException, InterruptedException {
331     final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
332     if (cpHost != null) {
333       final HRegionInfo[] regions = (newRegions == null) ? null :
334         newRegions.toArray(new HRegionInfo[newRegions.size()]);
335       user.doAs(new PrivilegedExceptionAction<Void>() {
336         @Override
337         public Void run() throws Exception {
338           cpHost.postCreateTableHandler(hTableDescriptor, regions);
339           return null;
340         }
341       });
342     }
343   }
344 
345   protected interface CreateHdfsRegions {
346     List<HRegionInfo> createHdfsRegions(final MasterProcedureEnv env,
347       final Path tableRootDir, final TableName tableName,
348       final List<HRegionInfo> newRegions) throws IOException;
349   }
350 
351   protected static List<HRegionInfo> createFsLayout(final MasterProcedureEnv env,
352       final HTableDescriptor hTableDescriptor, final List<HRegionInfo> newRegions)
353       throws IOException {
354     return createFsLayout(env, hTableDescriptor, newRegions, new CreateHdfsRegions() {
355       @Override
356       public List<HRegionInfo> createHdfsRegions(final MasterProcedureEnv env,
357           final Path tableRootDir, final TableName tableName,
358           final List<HRegionInfo> newRegions) throws IOException {
359         HRegionInfo[] regions = newRegions != null ?
360           newRegions.toArray(new HRegionInfo[newRegions.size()]) : null;
361         return ModifyRegionUtils.createRegions(env.getMasterConfiguration(),
362             tableRootDir, hTableDescriptor, regions, null);
363       }
364     });
365   }
366 
367   protected static List<HRegionInfo> createFsLayout(final MasterProcedureEnv env,
368       final HTableDescriptor hTableDescriptor, List<HRegionInfo> newRegions,
369       final CreateHdfsRegions hdfsRegionHandler) throws IOException {
370     final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
371     final Path tempdir = mfs.getTempDir();
372 
373     // 1. Create Table Descriptor
374     // using a copy of descriptor, table will be created enabling first
375     final Path tempTableDir = FSUtils.getTableDir(tempdir, hTableDescriptor.getTableName());
376     new FSTableDescriptors(env.getMasterConfiguration()).createTableDescriptorForTableDirectory(
377       tempTableDir, hTableDescriptor, false);
378 
379     // 2. Create Regions
380     newRegions = hdfsRegionHandler.createHdfsRegions(env, tempdir,
381       hTableDescriptor.getTableName(), newRegions);
382 
383     // 3. Move Table temp directory to the hbase root location
384     final Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), hTableDescriptor.getTableName());
385     FileSystem fs = mfs.getFileSystem();
386     if (!fs.delete(tableDir, true) && fs.exists(tableDir)) {
387       throw new IOException("Couldn't delete " + tableDir);
388     }
389     if (!fs.rename(tempTableDir, tableDir)) {
390       throw new IOException("Unable to move table from temp=" + tempTableDir +
391         " to hbase root=" + tableDir);
392     }
393     return newRegions;
394   }
395 
396   protected static List<HRegionInfo> addTableToMeta(final MasterProcedureEnv env,
397       final HTableDescriptor hTableDescriptor,
398       final List<HRegionInfo> regions) throws IOException {
399     if (regions != null && regions.size() > 0) {
400       ProcedureSyncWait.waitMetaRegions(env);
401 
402       // Add regions to META
403       addRegionsToMeta(env, hTableDescriptor, regions);
404       // Add replicas if needed
405       List<HRegionInfo> newRegions = addReplicas(env, hTableDescriptor, regions);
406 
407       // Setup replication for region replicas if needed
408       if (hTableDescriptor.getRegionReplication() > 1) {
409         ServerRegionReplicaUtil.setupRegionReplicaReplication(env.getMasterConfiguration());
410       }
411       return newRegions;
412     }
413     return regions;
414   }
415 
416   /**
417    * Create any replicas for the regions (the default replicas that was
418    * already created is passed to the method)
419    * @param hTableDescriptor descriptor to use
420    * @param regions default replicas
421    * @return the combined list of default and non-default replicas
422    */
423   private static List<HRegionInfo> addReplicas(final MasterProcedureEnv env,
424       final HTableDescriptor hTableDescriptor,
425       final List<HRegionInfo> regions) {
426     int numRegionReplicas = hTableDescriptor.getRegionReplication() - 1;
427     if (numRegionReplicas <= 0) {
428       return regions;
429     }
430     List<HRegionInfo> hRegionInfos =
431         new ArrayList<HRegionInfo>((numRegionReplicas+1)*regions.size());
432     for (int i = 0; i < regions.size(); i++) {
433       for (int j = 1; j <= numRegionReplicas; j++) {
434         hRegionInfos.add(RegionReplicaUtil.getRegionInfoForReplica(regions.get(i), j));
435       }
436     }
437     hRegionInfos.addAll(regions);
438     return hRegionInfos;
439   }
440 
441   protected static void assignRegions(final MasterProcedureEnv env,
442       final TableName tableName, final List<HRegionInfo> regions)
443       throws HBaseException, IOException {
444     ProcedureSyncWait.waitRegionServers(env);
445 
446     final AssignmentManager assignmentManager = env.getMasterServices().getAssignmentManager();
447 
448     // Mark the table as Enabling
449     assignmentManager.getTableStateManager().setTableState(tableName,
450         ZooKeeperProtos.Table.State.ENABLING);
451 
452     // Trigger immediate assignment of the regions in round-robin fashion
453     ModifyRegionUtils.assignRegions(assignmentManager, regions);
454 
455     // Enable table
456     assignmentManager.getTableStateManager()
457       .setTableState(tableName, ZooKeeperProtos.Table.State.ENABLED);
458   }
459 
460   /**
461    * Add the specified set of regions to the hbase:meta table.
462    */
463   protected static void addRegionsToMeta(final MasterProcedureEnv env,
464       final HTableDescriptor hTableDescriptor,
465       final List<HRegionInfo> regionInfos) throws IOException {
466     MetaTableAccessor.addRegionsToMeta(env.getMasterServices().getConnection(),
467       regionInfos, hTableDescriptor.getRegionReplication());
468   }
469 
470   protected static void updateTableDescCache(final MasterProcedureEnv env,
471       final TableName tableName) throws IOException {
472     env.getMasterServices().getTableDescriptors().get(tableName);
473   }
474 
475   @Override
476   protected boolean shouldWaitClientAck(MasterProcedureEnv env) {
477     // system tables are created on bootstrap internally by the system
478     // the client does not know about this procedures.
479     return !getTableName().isSystemTable();
480   }
481 }