View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master;
20  
21  import com.google.common.collect.Sets;
22  import org.apache.commons.logging.Log;
23  import org.apache.commons.logging.LogFactory;
24  import org.apache.hadoop.classification.InterfaceAudience;
25  import org.apache.hadoop.fs.FileStatus;
26  import org.apache.hadoop.fs.FileSystem;
27  import org.apache.hadoop.fs.Path;
28  import org.apache.hadoop.hbase.Chore;
29  import org.apache.hadoop.hbase.HConstants;
30  import org.apache.hadoop.hbase.NamespaceDescriptor;
31  import org.apache.hadoop.hbase.util.FSUtils;
32  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
33  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
34  import org.apache.zookeeper.KeeperException;
35  
36  import java.io.IOException;
37  import java.util.Set;
38  import java.util.concurrent.atomic.AtomicBoolean;
39  
40  /**
41   * A janitor for the namespace artifacts.
42   * Traverses hdfs and zk to remove orphaned directories/znodes
43   */
44  @InterfaceAudience.Private
45  public class NamespaceJanitor extends Chore {
46    private static final Log LOG = LogFactory.getLog(NamespaceJanitor.class.getName());
47    private final MasterServices services;
48    private AtomicBoolean enabled = new AtomicBoolean(true);
49  
50    public NamespaceJanitor(final MasterServices services) {
51      super("NamespaceJanitor-" + services.getServerName().toShortString(),
52        services.getConfiguration().getInt("hbase.namespacejanitor.interval", 300000),
53        services);
54      this.services = services;
55    }
56  
57    @Override
58    protected boolean initialChore() {
59      try {
60        if (this.enabled.get()) removeOrphans();
61      } catch (IOException e) {
62        LOG.warn("Failed NamespaceJanitor chore", e);
63        return false;
64      } catch (KeeperException e) {
65        LOG.warn("Failed NamespaceJanitor chore", e);
66        return false;
67      }
68      return true;
69    }
70  
71    /**
72     * @param enabled
73     */
74    public boolean setEnabled(final boolean enabled) {
75      return this.enabled.getAndSet(enabled);
76    }
77  
78    boolean getEnabled() {
79      return this.enabled.get();
80    }
81  
82    @Override
83    protected void chore() {
84      try {
85        if (this.enabled.get()) {
86          removeOrphans();
87        } else {
88          LOG.warn("NamepsaceJanitor disabled! Not running scan.");
89        }
90      } catch (IOException e) {
91        LOG.warn("Failed NamespaceJanitor chore", e);
92      } catch (KeeperException e) {
93        LOG.warn("Failed NamespaceJanitor chore", e);
94      }
95    }
96  
97    private void removeOrphans() throws IOException, KeeperException {
98      //cache the info so we don't need to keep the master nsLock for long
99      //and not be wasteful with rpc calls
100     FileSystem fs = services.getMasterFileSystem().getFileSystem();
101     Set<String> descs = Sets.newHashSet();
102     for(NamespaceDescriptor ns : services.listNamespaceDescriptors()) {
103       descs.add(ns.getName());
104     }
105 
106     //cleanup hdfs orphans
107     for (FileStatus nsStatus : FSUtils.listStatus(fs,
108         new Path(FSUtils.getRootDir(services.getConfiguration()), HConstants.BASE_NAMESPACE_DIR))) {
109       if (!descs.contains(nsStatus.getPath().getName()) &&
110           !NamespaceDescriptor.RESERVED_NAMESPACES.contains(nsStatus.getPath().getName())) {
111         boolean isEmpty = true;
112         for(FileStatus status : fs.listStatus(nsStatus.getPath())) {
113           if (!HConstants.HBASE_NON_TABLE_DIRS.contains(status.getPath().getName())) {
114             isEmpty = false;
115             break;
116           }
117         }
118         if(isEmpty) {
119           try {
120             if (!fs.delete(nsStatus.getPath(), true)) {
121               LOG.error("Failed to remove namespace directory: " + nsStatus.getPath());
122             }
123           } catch (IOException ex) {
124             LOG.error("Failed to remove namespace directory: " + nsStatus.getPath(),
125                 ex);
126           }
127           LOG.debug("Removed namespace directory: "+nsStatus.getPath());
128         } else {
129           LOG.debug("Skipping non-empty namespace directory: " + nsStatus.getPath());
130         }
131       }
132     }
133 
134     String baseZnode = ZooKeeperWatcher.namespaceZNode;
135     for(String child : ZKUtil.listChildrenNoWatch(services.getZooKeeper(), baseZnode)) {
136       if (!descs.contains(child) &&
137           !NamespaceDescriptor.RESERVED_NAMESPACES.contains(child)) {
138         String znode = ZKUtil.joinZNode(baseZnode, child);
139         try {
140           ZKUtil.deleteNode(services.getZooKeeper(), znode);
141           LOG.debug("Removed namespace znode: " + znode);
142         } catch (KeeperException ex) {
143           LOG.debug("Failed to remove namespace znode: " + znode, ex);
144         }
145       }
146     }
147 
148   }
149 }