1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.catalog;
19
20 import java.io.IOException;
21 import java.io.InterruptedIOException;
22 import java.net.ConnectException;
23 import java.util.ArrayList;
24 import java.util.List;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.classification.InterfaceAudience;
29 import org.apache.hadoop.hbase.DoNotRetryIOException;
30 import org.apache.hadoop.hbase.HConstants;
31 import org.apache.hadoop.hbase.HRegionInfo;
32 import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
33 import org.apache.hadoop.hbase.ServerName;
34 import org.apache.hadoop.hbase.client.Delete;
35 import org.apache.hadoop.hbase.client.HTable;
36 import org.apache.hadoop.hbase.client.Mutation;
37 import org.apache.hadoop.hbase.client.Put;
38 import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
39 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
40 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
41 import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MultiRowMutationService;
42 import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MutateRowsRequest;
43 import org.apache.hadoop.hbase.util.Bytes;
44 import org.apache.hadoop.hbase.util.Threads;
45
46 import com.google.protobuf.ServiceException;
47
48
49
50
51
52
53 @InterfaceAudience.Private
54 public class MetaEditor {
55
56
57
58 private static final Log LOG = LogFactory.getLog(MetaEditor.class);
59
60
61
62
63 public static Put makePutFromRegionInfo(HRegionInfo regionInfo)
64 throws IOException {
65 Put put = new Put(regionInfo.getRegionName());
66 addRegionInfo(put, regionInfo);
67 return put;
68 }
69
70
71
72
73
74 public static Delete makeDeleteFromRegionInfo(HRegionInfo regionInfo) {
75 if (regionInfo == null) {
76 throw new IllegalArgumentException("Can't make a delete for null region");
77 }
78 Delete delete = new Delete(regionInfo.getRegionName());
79 return delete;
80 }
81
82
83
84
85 public static Put addDaughtersToPut(Put put, HRegionInfo splitA, HRegionInfo splitB) {
86 if (splitA != null) {
87 put.addImmutable(
88 HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER, splitA.toByteArray());
89 }
90 if (splitB != null) {
91 put.addImmutable(
92 HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER, splitB.toByteArray());
93 }
94 return put;
95 }
96
97
98
99
100
101
102
103 static void putToMetaTable(final CatalogTracker ct, final Put p)
104 throws IOException {
105 put(MetaReader.getMetaHTable(ct), p);
106 }
107
108
109
110
111
112
113
114 static void putToCatalogTable(final CatalogTracker ct, final Put p)
115 throws IOException {
116 put(MetaReader.getCatalogHTable(ct), p);
117 }
118
119
120
121
122
123
124 private static void put(final HTable t, final Put p) throws IOException {
125 try {
126 t.put(p);
127 } finally {
128 t.close();
129 }
130 }
131
132
133
134
135
136
137
138 public static void putsToMetaTable(final CatalogTracker ct, final List<Put> ps)
139 throws IOException {
140 HTable t = MetaReader.getMetaHTable(ct);
141 try {
142 t.put(ps);
143 } finally {
144 t.close();
145 }
146 }
147
148
149
150
151
152
153
154 static void deleteFromMetaTable(final CatalogTracker ct, final Delete d)
155 throws IOException {
156 List<Delete> dels = new ArrayList<Delete>(1);
157 dels.add(d);
158 deleteFromMetaTable(ct, dels);
159 }
160
161
162
163
164
165
166
167 public static void deleteFromMetaTable(final CatalogTracker ct, final List<Delete> deletes)
168 throws IOException {
169 HTable t = MetaReader.getMetaHTable(ct);
170 try {
171 t.delete(deletes);
172 } finally {
173 t.close();
174 }
175 }
176
177
178
179
180
181
182
183 public static void mutateMetaTable(final CatalogTracker ct, final List<Mutation> mutations)
184 throws IOException {
185 HTable t = MetaReader.getMetaHTable(ct);
186 try {
187 t.batch(mutations);
188 } catch (InterruptedException e) {
189 InterruptedIOException ie = new InterruptedIOException(e.getMessage());
190 ie.initCause(e);
191 throw ie;
192 } finally {
193 t.close();
194 }
195 }
196
197
198
199
200
201
202 public static void addRegionToMeta(CatalogTracker catalogTracker,
203 HRegionInfo regionInfo)
204 throws IOException {
205 putToMetaTable(catalogTracker, makePutFromRegionInfo(regionInfo));
206 LOG.info("Added " + regionInfo.getRegionNameAsString());
207 }
208
209
210
211
212
213
214
215
216 public static void addRegionToMeta(HTable meta, HRegionInfo regionInfo) throws IOException {
217 addRegionToMeta(meta, regionInfo, null, null);
218 }
219
220
221
222
223
224
225
226
227
228
229
230
231
232 public static void addRegionToMeta(HTable meta, HRegionInfo regionInfo,
233 HRegionInfo splitA, HRegionInfo splitB) throws IOException {
234 Put put = makePutFromRegionInfo(regionInfo);
235 addDaughtersToPut(put, splitA, splitB);
236 meta.put(put);
237 if (LOG.isDebugEnabled()) {
238 LOG.debug("Added " + regionInfo.getRegionNameAsString());
239 }
240 }
241
242
243
244
245
246
247
248
249
250
251
252
253
254 public static void addRegionToMeta(CatalogTracker catalogTracker, HRegionInfo regionInfo,
255 HRegionInfo splitA, HRegionInfo splitB) throws IOException {
256 HTable meta = MetaReader.getMetaHTable(catalogTracker);
257 try {
258 addRegionToMeta(meta, regionInfo, splitA, splitB);
259 } finally {
260 meta.close();
261 }
262 }
263
264
265
266
267
268
269
270 public static void addRegionsToMeta(CatalogTracker catalogTracker,
271 List<HRegionInfo> regionInfos)
272 throws IOException {
273 List<Put> puts = new ArrayList<Put>();
274 for (HRegionInfo regionInfo : regionInfos) {
275 puts.add(makePutFromRegionInfo(regionInfo));
276 }
277 putsToMetaTable(catalogTracker, puts);
278 LOG.info("Added " + puts.size());
279 }
280
281
282
283
284
285
286
287 public static void addDaughter(final CatalogTracker catalogTracker,
288 final HRegionInfo regionInfo, final ServerName sn, final long openSeqNum)
289 throws NotAllMetaRegionsOnlineException, IOException {
290 Put put = new Put(regionInfo.getRegionName());
291 addRegionInfo(put, regionInfo);
292 if (sn != null) {
293 addLocation(put, sn, openSeqNum);
294 }
295 putToMetaTable(catalogTracker, put);
296 LOG.info("Added daughter " + regionInfo.getEncodedName() +
297 (sn == null? ", serverName=null": ", serverName=" + sn.toString()));
298 }
299
300
301
302
303
304
305
306
307
308
309
310
311 public static void mergeRegions(final CatalogTracker catalogTracker,
312 HRegionInfo mergedRegion, HRegionInfo regionA, HRegionInfo regionB,
313 ServerName sn) throws IOException {
314 HTable meta = MetaReader.getMetaHTable(catalogTracker);
315 try {
316 HRegionInfo copyOfMerged = new HRegionInfo(mergedRegion);
317
318
319 Put putOfMerged = makePutFromRegionInfo(copyOfMerged);
320 putOfMerged.addImmutable(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER,
321 regionA.toByteArray());
322 putOfMerged.addImmutable(HConstants.CATALOG_FAMILY, HConstants.MERGEB_QUALIFIER,
323 regionB.toByteArray());
324
325
326 Delete deleteA = makeDeleteFromRegionInfo(regionA);
327 Delete deleteB = makeDeleteFromRegionInfo(regionB);
328
329
330 addLocation(putOfMerged, sn, 1);
331
332 byte[] tableRow = Bytes.toBytes(mergedRegion.getRegionNameAsString()
333 + HConstants.DELIMITER);
334 multiMutate(meta, tableRow, putOfMerged, deleteA, deleteB);
335 } finally {
336 meta.close();
337 }
338 }
339
340
341
342
343
344
345
346
347
348
349
350
351 public static void splitRegion(final CatalogTracker catalogTracker,
352 HRegionInfo parent, HRegionInfo splitA, HRegionInfo splitB,
353 ServerName sn) throws IOException {
354 HTable meta = MetaReader.getMetaHTable(catalogTracker);
355 try {
356 HRegionInfo copyOfParent = new HRegionInfo(parent);
357 copyOfParent.setOffline(true);
358 copyOfParent.setSplit(true);
359
360
361 Put putParent = makePutFromRegionInfo(copyOfParent);
362 addDaughtersToPut(putParent, splitA, splitB);
363
364
365 Put putA = makePutFromRegionInfo(splitA);
366 Put putB = makePutFromRegionInfo(splitB);
367
368 addLocation(putA, sn, 1);
369 addLocation(putB, sn, 1);
370
371 byte[] tableRow = Bytes.toBytes(parent.getRegionNameAsString() + HConstants.DELIMITER);
372 multiMutate(meta, tableRow, putParent, putA, putB);
373 } finally {
374 meta.close();
375 }
376 }
377
378
379
380
381 private static void multiMutate(HTable table, byte[] row, Mutation... mutations) throws IOException {
382 CoprocessorRpcChannel channel = table.coprocessorService(row);
383 MutateRowsRequest.Builder mmrBuilder = MutateRowsRequest.newBuilder();
384 for (Mutation mutation : mutations) {
385 if (mutation instanceof Put) {
386 mmrBuilder.addMutationRequest(ProtobufUtil.toMutation(MutationType.PUT, mutation));
387 } else if (mutation instanceof Delete) {
388 mmrBuilder.addMutationRequest(ProtobufUtil.toMutation(MutationType.DELETE, mutation));
389 } else {
390 throw new DoNotRetryIOException("multi in MetaEditor doesn't support "
391 + mutation.getClass().getName());
392 }
393 }
394
395 MultiRowMutationService.BlockingInterface service =
396 MultiRowMutationService.newBlockingStub(channel);
397 try {
398 service.mutateRows(null, mmrBuilder.build());
399 } catch (ServiceException ex) {
400 ProtobufUtil.toIOException(ex);
401 }
402 }
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421 public static void updateMetaLocation(CatalogTracker catalogTracker,
422 HRegionInfo regionInfo, ServerName sn, long openSeqNum)
423 throws IOException, ConnectException {
424 updateLocation(catalogTracker, regionInfo, sn, openSeqNum);
425 }
426
427
428
429
430
431
432
433
434
435
436
437
438
439 public static void updateRegionLocation(CatalogTracker catalogTracker,
440 HRegionInfo regionInfo, ServerName sn, long updateSeqNum)
441 throws IOException {
442 updateLocation(catalogTracker, regionInfo, sn, updateSeqNum);
443 }
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458 private static void updateLocation(final CatalogTracker catalogTracker,
459 HRegionInfo regionInfo, ServerName sn, long openSeqNum)
460 throws IOException {
461 Put put = new Put(regionInfo.getRegionName());
462 addLocation(put, sn, openSeqNum);
463 putToCatalogTable(catalogTracker, put);
464 LOG.info("Updated row " + regionInfo.getRegionNameAsString() +
465 " with server=" + sn);
466 }
467
468
469
470
471
472
473
474 public static void deleteRegion(CatalogTracker catalogTracker,
475 HRegionInfo regionInfo)
476 throws IOException {
477 Delete delete = new Delete(regionInfo.getRegionName());
478 deleteFromMetaTable(catalogTracker, delete);
479 LOG.info("Deleted " + regionInfo.getRegionNameAsString());
480 }
481
482
483
484
485
486
487
488 public static void deleteRegions(CatalogTracker catalogTracker,
489 List<HRegionInfo> regionsInfo) throws IOException {
490 List<Delete> deletes = new ArrayList<Delete>(regionsInfo.size());
491 for (HRegionInfo hri: regionsInfo) {
492 deletes.add(new Delete(hri.getRegionName()));
493 }
494 deleteFromMetaTable(catalogTracker, deletes);
495 LOG.info("Deleted " + regionsInfo);
496 }
497
498
499
500
501
502
503
504
505 public static void mutateRegions(CatalogTracker catalogTracker,
506 final List<HRegionInfo> regionsToRemove, final List<HRegionInfo> regionsToAdd)
507 throws IOException {
508 List<Mutation> mutation = new ArrayList<Mutation>();
509 if (regionsToRemove != null) {
510 for (HRegionInfo hri: regionsToRemove) {
511 mutation.add(new Delete(hri.getRegionName()));
512 }
513 }
514 if (regionsToAdd != null) {
515 for (HRegionInfo hri: regionsToAdd) {
516 mutation.add(makePutFromRegionInfo(hri));
517 }
518 }
519 mutateMetaTable(catalogTracker, mutation);
520 if (regionsToRemove != null && regionsToRemove.size() > 0) {
521 LOG.debug("Deleted " + regionsToRemove);
522 }
523 if (regionsToAdd != null && regionsToAdd.size() > 0) {
524 LOG.debug("Added " + regionsToAdd);
525 }
526 }
527
528
529
530
531
532
533
534 public static void overwriteRegions(CatalogTracker catalogTracker,
535 List<HRegionInfo> regionInfos) throws IOException {
536 deleteRegions(catalogTracker, regionInfos);
537
538
539
540
541 Threads.sleep(20);
542 addRegionsToMeta(catalogTracker, regionInfos);
543 LOG.info("Overwritten " + regionInfos);
544 }
545
546
547
548
549
550
551
552 public static void deleteMergeQualifiers(CatalogTracker catalogTracker,
553 final HRegionInfo mergedRegion) throws IOException {
554 Delete delete = new Delete(mergedRegion.getRegionName());
555 delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER);
556 delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.MERGEB_QUALIFIER);
557 deleteFromMetaTable(catalogTracker, delete);
558 LOG.info("Deleted references in merged region "
559 + mergedRegion.getRegionNameAsString() + ", qualifier="
560 + Bytes.toStringBinary(HConstants.MERGEA_QUALIFIER) + " and qualifier="
561 + Bytes.toStringBinary(HConstants.MERGEB_QUALIFIER));
562 }
563
564 private static Put addRegionInfo(final Put p, final HRegionInfo hri)
565 throws IOException {
566 p.addImmutable(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
567 hri.toByteArray());
568 return p;
569 }
570
571 private static Put addLocation(final Put p, final ServerName sn, long openSeqNum) {
572 p.addImmutable(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
573 Bytes.toBytes(sn.getHostAndPort()));
574 p.addImmutable(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,
575 Bytes.toBytes(sn.getStartcode()));
576 p.addImmutable(HConstants.CATALOG_FAMILY, HConstants.SEQNUM_QUALIFIER,
577 Bytes.toBytes(openSeqNum));
578 return p;
579 }
580 }