1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.util;
20
21 import java.io.IOException;
22 import java.util.List;
23 import java.util.Map;
24
25 import org.apache.commons.logging.Log;
26 import org.apache.commons.logging.LogFactory;
27 import org.apache.hadoop.classification.InterfaceAudience;
28 import org.apache.hadoop.conf.Configuration;
29 import org.apache.hadoop.fs.Path;
30 import org.apache.hadoop.hbase.HRegionInfo;
31 import org.apache.hadoop.hbase.HTableDescriptor;
32 import org.apache.hadoop.hbase.ServerName;
33 import org.apache.hadoop.hbase.TableName;
34 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
35 import org.apache.hadoop.hbase.catalog.MetaEditor;
36 import org.apache.hadoop.hbase.client.HBaseAdmin;
37 import org.apache.hadoop.hbase.client.HConnection;
38 import org.apache.hadoop.hbase.client.HTable;
39 import org.apache.hadoop.hbase.master.RegionState;
40 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
41 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
42 import org.apache.hadoop.hbase.regionserver.HRegion;
43 import org.apache.hadoop.hbase.regionserver.wal.HLog;
44 import org.apache.zookeeper.KeeperException;
45
46
47
48
49
50 @InterfaceAudience.Private
51 public class HBaseFsckRepair {
52 public static final Log LOG = LogFactory.getLog(HBaseFsckRepair.class);
53
54
55
56
57
58
59
60
61
62
63 public static void fixMultiAssignment(HBaseAdmin admin, HRegionInfo region,
64 List<ServerName> servers)
65 throws IOException, KeeperException, InterruptedException {
66 HRegionInfo actualRegion = new HRegionInfo(region);
67
68
69 for(ServerName server : servers) {
70 closeRegionSilentlyAndWait(admin, server, actualRegion);
71 }
72
73
74 forceOfflineInZK(admin, actualRegion);
75 }
76
77
78
79
80
81
82
83
84
85
86
87
88
89 public static void fixUnassigned(HBaseAdmin admin, HRegionInfo region)
90 throws IOException, KeeperException {
91 HRegionInfo actualRegion = new HRegionInfo(region);
92
93
94 forceOfflineInZK(admin, actualRegion);
95 }
96
97
98
99
100
101
102
103
104
105
106
107
108
109 private static void forceOfflineInZK(HBaseAdmin admin, final HRegionInfo region)
110 throws ZooKeeperConnectionException, KeeperException, IOException {
111 admin.assign(region.getRegionName());
112 }
113
114
115
116
117 public static void waitUntilAssigned(HBaseAdmin admin,
118 HRegionInfo region) throws IOException, InterruptedException {
119 long timeout = admin.getConfiguration().getLong("hbase.hbck.assign.timeout", 120000);
120 long expiration = timeout + System.currentTimeMillis();
121 while (System.currentTimeMillis() < expiration) {
122 try {
123 Map<String, RegionState> rits=
124 admin.getClusterStatus().getRegionsInTransition();
125
126 if (rits.keySet() != null && !rits.keySet().contains(region.getEncodedName())) {
127
128 return;
129 }
130
131 LOG.info("Region still in transition, waiting for "
132 + "it to become assigned: " + region);
133 } catch (IOException e) {
134 LOG.warn("Exception when waiting for region to become assigned,"
135 + " retrying", e);
136 }
137 Thread.sleep(1000);
138 }
139 throw new IOException("Region " + region + " failed to move out of " +
140 "transition within timeout " + timeout + "ms");
141 }
142
143
144
145
146
147 public static void closeRegionSilentlyAndWait(HBaseAdmin admin,
148 ServerName server, HRegionInfo region) throws IOException, InterruptedException {
149 HConnection connection = admin.getConnection();
150 AdminService.BlockingInterface rs = connection.getAdmin(server);
151 try {
152 ProtobufUtil.closeRegion(rs, server, region.getRegionName(), false);
153 } catch (IOException e) {
154 LOG.warn("Exception when closing region: " + region.getRegionNameAsString(), e);
155 }
156 long timeout = admin.getConfiguration()
157 .getLong("hbase.hbck.close.timeout", 120000);
158 long expiration = timeout + System.currentTimeMillis();
159 while (System.currentTimeMillis() < expiration) {
160 try {
161 HRegionInfo rsRegion =
162 ProtobufUtil.getRegionInfo(rs, region.getRegionName());
163 if (rsRegion == null) return;
164 } catch (IOException ioe) {
165 return;
166 }
167 Thread.sleep(1000);
168 }
169 throw new IOException("Region " + region + " failed to close within"
170 + " timeout " + timeout);
171 }
172
173
174
175
176 public static void fixMetaHoleOnline(Configuration conf,
177 HRegionInfo hri) throws IOException {
178 HTable meta = new HTable(conf, TableName.META_TABLE_NAME);
179 MetaEditor.addRegionToMeta(meta, hri);
180 meta.close();
181 }
182
183
184
185
186 public static HRegion createHDFSRegionDir(Configuration conf,
187 HRegionInfo hri, HTableDescriptor htd) throws IOException {
188
189 Path root = FSUtils.getRootDir(conf);
190 HRegion region = HRegion.createHRegion(hri, root, conf, htd);
191 HLog hlog = region.getLog();
192
193
194 region.close();
195 hlog.closeAndDelete();
196 return region;
197 }
198 }