1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.util.hbck;
19
20 import static org.junit.Assert.assertEquals;
21
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.List;
25 import java.util.Map;
26 import java.util.Map.Entry;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.fs.FSDataOutputStream;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.hbase.TableName;
35 import org.apache.hadoop.hbase.HBaseTestingUtility;
36 import org.apache.hadoop.hbase.HColumnDescriptor;
37 import org.apache.hadoop.hbase.HRegionInfo;
38 import org.apache.hadoop.hbase.HTableDescriptor;
39 import org.apache.hadoop.hbase.LargeTests;
40 import org.apache.hadoop.hbase.NamespaceDescriptor;
41 import org.apache.hadoop.hbase.ServerName;
42 import org.apache.hadoop.hbase.catalog.MetaEditor;
43 import org.apache.hadoop.hbase.client.Delete;
44 import org.apache.hadoop.hbase.client.HBaseAdmin;
45 import org.apache.hadoop.hbase.client.HConnectionManager;
46 import org.apache.hadoop.hbase.client.HTable;
47 import org.apache.hadoop.hbase.client.Put;
48 import org.apache.hadoop.hbase.client.Result;
49 import org.apache.hadoop.hbase.client.ResultScanner;
50 import org.apache.hadoop.hbase.client.Scan;
51 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
52 import org.apache.hadoop.hbase.util.Bytes;
53 import org.apache.hadoop.hbase.util.FSUtils;
54 import org.apache.zookeeper.KeeperException;
55 import org.junit.After;
56 import org.junit.Before;
57 import org.junit.experimental.categories.Category;
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72 @Category(LargeTests.class)
73 public class OfflineMetaRebuildTestCore {
74 protected final static Log LOG = LogFactory
75 .getLog(OfflineMetaRebuildTestCore.class);
76 protected HBaseTestingUtility TEST_UTIL;
77 protected Configuration conf;
78 private final static byte[] FAM = Bytes.toBytes("fam");
79
80
81 protected HTable htbl;
82 protected final static byte[][] splits = new byte[][] { Bytes.toBytes("A"),
83 Bytes.toBytes("B"), Bytes.toBytes("C") };
84
85 private final static String TABLE_BASE = "tableMetaRebuild";
86 private static int tableIdx = 0;
87 protected TableName table = TableName.valueOf("tableMetaRebuild");
88
89 @Before
90 public void setUpBefore() throws Exception {
91 TEST_UTIL = new HBaseTestingUtility();
92 TEST_UTIL.getConfiguration().setInt("dfs.datanode.max.xceivers", 9192);
93 TEST_UTIL.startMiniCluster(3);
94 conf = TEST_UTIL.getConfiguration();
95 assertEquals(0, TEST_UTIL.getHBaseAdmin().listTables().length);
96
97
98 table = TableName.valueOf(TABLE_BASE + "-" + tableIdx);
99 tableIdx++;
100 htbl = setupTable(table);
101 populateTable(htbl);
102 assertEquals(5, scanMeta());
103 LOG.info("Table " + table + " has " + tableRowCount(conf, table)
104 + " entries.");
105 assertEquals(16, tableRowCount(conf, table));
106 TEST_UTIL.getHBaseAdmin().disableTable(table);
107 assertEquals(1, TEST_UTIL.getHBaseAdmin().listTables().length);
108 }
109
110 @After
111 public void tearDownAfter() throws Exception {
112 TEST_UTIL.shutdownMiniCluster();
113 HConnectionManager.deleteConnection(conf);
114 }
115
116
117
118
119
120
121
122
123 private HTable setupTable(TableName tablename) throws Exception {
124 HTableDescriptor desc = new HTableDescriptor(tablename);
125 HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
126 desc.addFamily(hcd);
127 TEST_UTIL.getHBaseAdmin().createTable(desc, splits);
128 return new HTable(TEST_UTIL.getConfiguration(), tablename);
129 }
130
131 private void dumpMeta(HTableDescriptor htd) throws IOException {
132 List<byte[]> metaRows = TEST_UTIL.getMetaTableRows(htd.getTableName());
133 for (byte[] row : metaRows) {
134 LOG.info(Bytes.toString(row));
135 }
136 }
137
138 private void populateTable(HTable tbl) throws IOException {
139 byte[] values = { 'A', 'B', 'C', 'D' };
140 for (int i = 0; i < values.length; i++) {
141 for (int j = 0; j < values.length; j++) {
142 Put put = new Put(new byte[] { values[i], values[j] });
143 put.add(Bytes.toBytes("fam"), new byte[] {}, new byte[] { values[i],
144 values[j] });
145 tbl.put(put);
146 }
147 }
148 tbl.flushCommits();
149 }
150
151
152
153
154
155
156
157 void deleteTable(HBaseAdmin admin, String tablename) throws IOException {
158 try {
159 byte[] tbytes = Bytes.toBytes(tablename);
160 admin.disableTable(tbytes);
161 admin.deleteTable(tbytes);
162 } catch (Exception e) {
163
164 }
165 }
166
167 protected void deleteRegion(Configuration conf, final HTable tbl,
168 byte[] startKey, byte[] endKey) throws IOException {
169
170 LOG.info("Before delete:");
171 HTableDescriptor htd = tbl.getTableDescriptor();
172 dumpMeta(htd);
173
174 Map<HRegionInfo, ServerName> hris = tbl.getRegionLocations();
175 for (Entry<HRegionInfo, ServerName> e : hris.entrySet()) {
176 HRegionInfo hri = e.getKey();
177 ServerName hsa = e.getValue();
178 if (Bytes.compareTo(hri.getStartKey(), startKey) == 0
179 && Bytes.compareTo(hri.getEndKey(), endKey) == 0) {
180
181 LOG.info("RegionName: " + hri.getRegionNameAsString());
182 byte[] deleteRow = hri.getRegionName();
183 TEST_UTIL.getHBaseAdmin().unassign(deleteRow, true);
184
185 LOG.info("deleting hdfs data: " + hri.toString() + hsa.toString());
186 Path rootDir = FSUtils.getRootDir(conf);
187 FileSystem fs = rootDir.getFileSystem(conf);
188 Path p = new Path(FSUtils.getTableDir(rootDir, htd.getTableName()),
189 hri.getEncodedName());
190 fs.delete(p, true);
191
192 HTable meta = new HTable(conf, TableName.META_TABLE_NAME);
193 Delete delete = new Delete(deleteRow);
194 meta.delete(delete);
195 meta.close();
196 }
197 LOG.info(hri.toString() + hsa.toString());
198 }
199
200 TEST_UTIL.getMetaTableRows(htd.getTableName());
201 LOG.info("After delete:");
202 dumpMeta(htd);
203 }
204
205 protected HRegionInfo createRegion(Configuration conf, final HTable htbl,
206 byte[] startKey, byte[] endKey) throws IOException {
207 HTable meta = new HTable(conf, TableName.META_TABLE_NAME);
208 HTableDescriptor htd = htbl.getTableDescriptor();
209 HRegionInfo hri = new HRegionInfo(htbl.getName(), startKey, endKey);
210
211 LOG.info("manually adding regioninfo and hdfs data: " + hri.toString());
212 Path rootDir = FSUtils.getRootDir(conf);
213 FileSystem fs = rootDir.getFileSystem(conf);
214 Path p = new Path(FSUtils.getTableDir(rootDir, htbl.getName()),
215 hri.getEncodedName());
216 fs.mkdirs(p);
217 Path riPath = new Path(p, HRegionFileSystem.REGION_INFO_FILE);
218 FSDataOutputStream out = fs.create(riPath);
219 out.write(hri.toDelimitedByteArray());
220 out.close();
221
222
223 MetaEditor.addRegionToMeta(meta, hri);
224 meta.close();
225 return hri;
226 }
227
228 protected void wipeOutMeta() throws IOException {
229
230 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
231 Scan s = new Scan();
232 HTable meta = new HTable(conf, TableName.META_TABLE_NAME);
233 ResultScanner scanner = meta.getScanner(s);
234 List<Delete> dels = new ArrayList<Delete>();
235 for (Result r : scanner) {
236 HRegionInfo info =
237 HRegionInfo.getHRegionInfo(r);
238 if(info != null && !info.getTable().getNamespaceAsString()
239 .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
240 Delete d = new Delete(r.getRow());
241 dels.add(d);
242 admin.unassign(r.getRow(), true);
243 }
244 }
245 meta.delete(dels);
246 meta.flushCommits();
247 scanner.close();
248 meta.close();
249 }
250
251
252
253
254
255
256
257 protected int tableRowCount(Configuration conf, TableName table)
258 throws IOException {
259 HTable t = new HTable(conf, table);
260 Scan st = new Scan();
261
262 ResultScanner rst = t.getScanner(st);
263 int count = 0;
264 for (@SuppressWarnings("unused")
265 Result rt : rst) {
266 count++;
267 }
268 t.close();
269 return count;
270 }
271
272
273
274
275
276
277 protected int scanMeta() throws IOException {
278 int count = 0;
279 HTable meta = new HTable(conf, HTableDescriptor.META_TABLEDESC.getTableName());
280 ResultScanner scanner = meta.getScanner(new Scan());
281 LOG.info("Table: " + Bytes.toString(meta.getTableName()));
282 for (Result res : scanner) {
283 LOG.info(Bytes.toString(res.getRow()));
284 count++;
285 }
286 meta.close();
287 return count;
288 }
289 }