1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.snapshot;
20
21 import java.io.IOException;
22 import java.io.InputStream;
23 import java.io.OutputStream;
24 import java.util.Arrays;
25 import java.util.HashMap;
26 import java.util.HashSet;
27 import java.util.LinkedList;
28 import java.util.List;
29 import java.util.Map;
30 import java.util.Set;
31 import java.util.TreeMap;
32
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.classification.InterfaceAudience;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FileStatus;
38 import org.apache.hadoop.fs.FileSystem;
39 import org.apache.hadoop.fs.Path;
40 import org.apache.hadoop.hbase.HColumnDescriptor;
41 import org.apache.hadoop.hbase.HRegionInfo;
42 import org.apache.hadoop.hbase.HTableDescriptor;
43 import org.apache.hadoop.hbase.TableName;
44 import org.apache.hadoop.hbase.backup.HFileArchiver;
45 import org.apache.hadoop.hbase.catalog.CatalogTracker;
46 import org.apache.hadoop.hbase.catalog.MetaEditor;
47 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
48 import org.apache.hadoop.hbase.io.HFileLink;
49 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
50 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
51 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
52 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
53 import org.apache.hadoop.hbase.regionserver.HRegion;
54 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
55 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
56 import org.apache.hadoop.hbase.util.Bytes;
57 import org.apache.hadoop.hbase.util.FSTableDescriptors;
58 import org.apache.hadoop.hbase.util.FSUtils;
59 import org.apache.hadoop.hbase.util.FSVisitor;
60 import org.apache.hadoop.hbase.util.ModifyRegionUtils;
61 import org.apache.hadoop.hbase.util.Pair;
62 import org.apache.hadoop.io.IOUtils;
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105 @InterfaceAudience.Private
106 public class RestoreSnapshotHelper {
107 private static final Log LOG = LogFactory.getLog(RestoreSnapshotHelper.class);
108
109 private final Map<byte[], byte[]> regionsMap =
110 new TreeMap<byte[], byte[]>(Bytes.BYTES_COMPARATOR);
111
112 private final Map<String, Pair<String, String> > parentsMap =
113 new HashMap<String, Pair<String, String> >();
114
115 private final ForeignExceptionDispatcher monitor;
116 private final MonitoredTask status;
117
118 private final SnapshotDescription snapshotDesc;
119 private final TableName snapshotTable;
120 private final Path snapshotDir;
121
122 private final HTableDescriptor tableDesc;
123 private final Path rootDir;
124 private final Path tableDir;
125
126 private final Configuration conf;
127 private final FileSystem fs;
128
129 public RestoreSnapshotHelper(final Configuration conf,
130 final FileSystem fs,
131 final SnapshotDescription snapshotDescription,
132 final Path snapshotDir,
133 final HTableDescriptor tableDescriptor,
134 final Path rootDir,
135 final ForeignExceptionDispatcher monitor,
136 final MonitoredTask status)
137 {
138 this.fs = fs;
139 this.conf = conf;
140 this.snapshotDesc = snapshotDescription;
141 this.snapshotTable = TableName.valueOf(snapshotDescription.getTable());
142 this.snapshotDir = snapshotDir;
143 this.tableDesc = tableDescriptor;
144 this.rootDir = rootDir;
145 this.tableDir = FSUtils.getTableDir(rootDir, tableDesc.getTableName());
146 this.monitor = monitor;
147 this.status = status;
148 }
149
150
151
152
153
154 public RestoreMetaChanges restoreHdfsRegions() throws IOException {
155 LOG.debug("starting restore");
156 Set<String> snapshotRegionNames = SnapshotReferenceUtil.getSnapshotRegionNames(fs, snapshotDir);
157 if (snapshotRegionNames == null) {
158 LOG.warn("Nothing to restore. Snapshot " + snapshotDesc + " looks empty");
159 return null;
160 }
161
162 RestoreMetaChanges metaChanges = new RestoreMetaChanges(parentsMap);
163
164
165
166 List<HRegionInfo> tableRegions = getTableRegions();
167 if (tableRegions != null) {
168 monitor.rethrowException();
169 for (HRegionInfo regionInfo: tableRegions) {
170 String regionName = regionInfo.getEncodedName();
171 if (snapshotRegionNames.contains(regionName)) {
172 LOG.info("region to restore: " + regionName);
173 snapshotRegionNames.remove(regionName);
174 metaChanges.addRegionToRestore(regionInfo);
175 } else {
176 LOG.info("region to remove: " + regionName);
177 metaChanges.addRegionToRemove(regionInfo);
178 }
179 }
180
181
182 monitor.rethrowException();
183 status.setStatus("Restoring table regions...");
184 restoreHdfsRegions(metaChanges.getRegionsToRestore());
185 status.setStatus("Finished restoring all table regions.");
186
187
188 monitor.rethrowException();
189 status.setStatus("Starting to delete excess regions from table");
190 removeHdfsRegions(metaChanges.getRegionsToRemove());
191 status.setStatus("Finished deleting excess regions from table.");
192 }
193
194
195 if (snapshotRegionNames.size() > 0) {
196 List<HRegionInfo> regionsToAdd = new LinkedList<HRegionInfo>();
197
198 monitor.rethrowException();
199 for (String regionName: snapshotRegionNames) {
200 LOG.info("region to add: " + regionName);
201 Path regionDir = new Path(snapshotDir, regionName);
202 regionsToAdd.add(HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir));
203 }
204
205
206 monitor.rethrowException();
207 status.setStatus("Cloning regions...");
208 HRegionInfo[] clonedRegions = cloneHdfsRegions(regionsToAdd);
209 metaChanges.setNewRegions(clonedRegions);
210 status.setStatus("Finished cloning regions.");
211 }
212
213
214 monitor.rethrowException();
215 status.setStatus("Restoring WALs to table...");
216 restoreWALs();
217 status.setStatus("Finished restoring WALs to table.");
218
219 return metaChanges;
220 }
221
222
223
224
225 public static class RestoreMetaChanges {
226 private final Map<String, Pair<String, String> > parentsMap;
227
228 private List<HRegionInfo> regionsToRestore = null;
229 private List<HRegionInfo> regionsToRemove = null;
230 private List<HRegionInfo> regionsToAdd = null;
231
232 RestoreMetaChanges(final Map<String, Pair<String, String> > parentsMap) {
233 this.parentsMap = parentsMap;
234 }
235
236
237
238
239 public boolean hasRegionsToAdd() {
240 return this.regionsToAdd != null && this.regionsToAdd.size() > 0;
241 }
242
243
244
245
246
247
248
249 public List<HRegionInfo> getRegionsToAdd() {
250 return this.regionsToAdd;
251 }
252
253
254
255
256 public boolean hasRegionsToRestore() {
257 return this.regionsToRestore != null && this.regionsToRestore.size() > 0;
258 }
259
260
261
262
263
264
265 public List<HRegionInfo> getRegionsToRestore() {
266 return this.regionsToRestore;
267 }
268
269
270
271
272 public boolean hasRegionsToRemove() {
273 return this.regionsToRemove != null && this.regionsToRemove.size() > 0;
274 }
275
276
277
278
279
280
281
282 public List<HRegionInfo> getRegionsToRemove() {
283 return this.regionsToRemove;
284 }
285
286 void setNewRegions(final HRegionInfo[] hris) {
287 if (hris != null) {
288 regionsToAdd = Arrays.asList(hris);
289 } else {
290 regionsToAdd = null;
291 }
292 }
293
294 void addRegionToRemove(final HRegionInfo hri) {
295 if (regionsToRemove == null) {
296 regionsToRemove = new LinkedList<HRegionInfo>();
297 }
298 regionsToRemove.add(hri);
299 }
300
301 void addRegionToRestore(final HRegionInfo hri) {
302 if (regionsToRestore == null) {
303 regionsToRestore = new LinkedList<HRegionInfo>();
304 }
305 regionsToRestore.add(hri);
306 }
307
308 public void updateMetaParentRegions(final CatalogTracker catalogTracker,
309 final List<HRegionInfo> regionInfos) throws IOException {
310 if (regionInfos == null || parentsMap.isEmpty()) return;
311
312
313 Map<String, HRegionInfo> regionsByName = new HashMap<String, HRegionInfo>(regionInfos.size());
314 List<HRegionInfo> parentRegions = new LinkedList();
315 for (HRegionInfo regionInfo: regionInfos) {
316 if (regionInfo.isSplitParent()) {
317 parentRegions.add(regionInfo);
318 } else {
319 regionsByName.put(regionInfo.getEncodedName(), regionInfo);
320 }
321 }
322
323
324 for (HRegionInfo regionInfo: parentRegions) {
325 Pair<String, String> daughters = parentsMap.get(regionInfo.getEncodedName());
326 if (daughters == null) {
327
328
329 LOG.warn("Skip update of unreferenced offline parent: " + regionInfo);
330 continue;
331 }
332
333
334 if (daughters.getSecond() == null) {
335 daughters.setSecond(daughters.getFirst());
336 }
337
338 LOG.debug("Update splits parent " + regionInfo.getEncodedName() + " -> " + daughters);
339 MetaEditor.addRegionToMeta(catalogTracker, regionInfo,
340 regionsByName.get(daughters.getFirst()),
341 regionsByName.get(daughters.getSecond()));
342 }
343 }
344 }
345
346
347
348
349 private void removeHdfsRegions(final List<HRegionInfo> regions) throws IOException {
350 if (regions != null && regions.size() > 0) {
351 for (HRegionInfo hri: regions) {
352 HFileArchiver.archiveRegion(conf, fs, hri);
353 }
354 }
355 }
356
357
358
359
360 private void restoreHdfsRegions(final List<HRegionInfo> regions) throws IOException {
361 if (regions == null || regions.size() == 0) return;
362 for (HRegionInfo hri: regions) restoreRegion(hri);
363 }
364
365
366
367
368
369 private void restoreRegion(HRegionInfo regionInfo) throws IOException {
370 Path snapshotRegionDir = new Path(snapshotDir, regionInfo.getEncodedName());
371 Map<String, List<String>> snapshotFiles =
372 SnapshotReferenceUtil.getRegionHFileReferences(fs, snapshotRegionDir);
373 Path regionDir = new Path(tableDir, regionInfo.getEncodedName());
374 String tableName = tableDesc.getTableName().getNameAsString();
375
376
377 for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
378 byte[] family = Bytes.toBytes(familyDir.getName());
379 Set<String> familyFiles = getTableRegionFamilyFiles(familyDir);
380 List<String> snapshotFamilyFiles = snapshotFiles.remove(familyDir.getName());
381 if (snapshotFamilyFiles != null) {
382 List<String> hfilesToAdd = new LinkedList<String>();
383 for (String hfileName: snapshotFamilyFiles) {
384 if (familyFiles.contains(hfileName)) {
385
386 familyFiles.remove(hfileName);
387 } else {
388
389 hfilesToAdd.add(hfileName);
390 }
391 }
392
393
394 for (String hfileName: familyFiles) {
395 Path hfile = new Path(familyDir, hfileName);
396 LOG.trace("Removing hfile=" + hfile +
397 " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
398 HFileArchiver.archiveStoreFile(conf, fs, regionInfo, tableDir, family, hfile);
399 }
400
401
402 for (String hfileName: hfilesToAdd) {
403 LOG.trace("Adding HFileLink " + hfileName +
404 " to region=" + regionInfo.getEncodedName() + " table=" + tableName);
405 restoreStoreFile(familyDir, regionInfo, hfileName);
406 }
407 } else {
408
409 LOG.trace("Removing family=" + Bytes.toString(family) +
410 " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
411 HFileArchiver.archiveFamily(fs, conf, regionInfo, tableDir, family);
412 fs.delete(familyDir, true);
413 }
414 }
415
416
417 for (Map.Entry<String, List<String>> familyEntry: snapshotFiles.entrySet()) {
418 Path familyDir = new Path(regionDir, familyEntry.getKey());
419 if (!fs.mkdirs(familyDir)) {
420 throw new IOException("Unable to create familyDir=" + familyDir);
421 }
422
423 for (String hfileName: familyEntry.getValue()) {
424 LOG.trace("Adding HFileLink " + hfileName + " to table=" + tableName);
425 restoreStoreFile(familyDir, regionInfo, hfileName);
426 }
427 }
428 }
429
430
431
432
433 private Set<String> getTableRegionFamilyFiles(final Path familyDir) throws IOException {
434 Set<String> familyFiles = new HashSet<String>();
435
436 FileStatus[] hfiles = FSUtils.listStatus(fs, familyDir);
437 if (hfiles == null) return familyFiles;
438
439 for (FileStatus hfileRef: hfiles) {
440 String hfileName = hfileRef.getPath().getName();
441 familyFiles.add(hfileName);
442 }
443
444 return familyFiles;
445 }
446
447
448
449
450
451 private HRegionInfo[] cloneHdfsRegions(final List<HRegionInfo> regions) throws IOException {
452 if (regions == null || regions.size() == 0) return null;
453
454 final Map<String, HRegionInfo> snapshotRegions =
455 new HashMap<String, HRegionInfo>(regions.size());
456
457
458 HRegionInfo[] clonedRegionsInfo = new HRegionInfo[regions.size()];
459 for (int i = 0; i < clonedRegionsInfo.length; ++i) {
460
461 HRegionInfo snapshotRegionInfo = regions.get(i);
462 clonedRegionsInfo[i] = cloneRegionInfo(snapshotRegionInfo);
463
464
465 String snapshotRegionName = snapshotRegionInfo.getEncodedName();
466 String clonedRegionName = clonedRegionsInfo[i].getEncodedName();
467 regionsMap.put(Bytes.toBytes(snapshotRegionName), Bytes.toBytes(clonedRegionName));
468 LOG.info("clone region=" + snapshotRegionName + " as " + clonedRegionName);
469
470
471 snapshotRegions.put(clonedRegionName, snapshotRegionInfo);
472 }
473
474
475 ModifyRegionUtils.createRegions(conf, rootDir, tableDir,
476 tableDesc, clonedRegionsInfo, new ModifyRegionUtils.RegionFillTask() {
477 @Override
478 public void fillRegion(final HRegion region) throws IOException {
479 cloneRegion(region, snapshotRegions.get(region.getRegionInfo().getEncodedName()));
480 }
481 });
482
483 return clonedRegionsInfo;
484 }
485
486
487
488
489
490
491
492
493
494
495
496
497 private void cloneRegion(final HRegion region, final HRegionInfo snapshotRegionInfo)
498 throws IOException {
499 final Path snapshotRegionDir = new Path(snapshotDir, snapshotRegionInfo.getEncodedName());
500 final Path regionDir = new Path(tableDir, region.getRegionInfo().getEncodedName());
501 final String tableName = tableDesc.getTableName().getNameAsString();
502 SnapshotReferenceUtil.visitRegionStoreFiles(fs, snapshotRegionDir,
503 new FSVisitor.StoreFileVisitor() {
504 @Override
505 public void storeFile (final String region, final String family, final String hfile)
506 throws IOException {
507 LOG.info("Adding HFileLink " + hfile + " to table=" + tableName);
508 Path familyDir = new Path(regionDir, family);
509 restoreStoreFile(familyDir, snapshotRegionInfo, hfile);
510 }
511 });
512 }
513
514
515
516
517
518
519
520
521
522
523
524
525
526 private void restoreStoreFile(final Path familyDir, final HRegionInfo regionInfo,
527 final String hfileName) throws IOException {
528 if (HFileLink.isHFileLink(hfileName)) {
529 HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName);
530 } else if (StoreFileInfo.isReference(hfileName)) {
531 restoreReferenceFile(familyDir, regionInfo, hfileName);
532 } else {
533 HFileLink.create(conf, fs, familyDir, regionInfo, hfileName);
534 }
535 }
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555 private void restoreReferenceFile(final Path familyDir, final HRegionInfo regionInfo,
556 final String hfileName) throws IOException {
557
558 Path refPath = StoreFileInfo.getReferredToFile(new Path(new Path(new Path(
559 snapshotTable.getNameAsString(), regionInfo.getEncodedName()), familyDir.getName()),
560 hfileName));
561 String snapshotRegionName = refPath.getParent().getParent().getName();
562 String fileName = refPath.getName();
563
564
565 String clonedRegionName = Bytes.toString(regionsMap.get(Bytes.toBytes(snapshotRegionName)));
566 if (clonedRegionName == null) clonedRegionName = snapshotRegionName;
567
568
569 Path linkPath = null;
570 String refLink = fileName;
571 if (!HFileLink.isHFileLink(fileName)) {
572 refLink = HFileLink.createHFileLinkName(snapshotTable, snapshotRegionName, fileName);
573 linkPath = new Path(familyDir,
574 HFileLink.createHFileLinkName(snapshotTable, regionInfo.getEncodedName(), hfileName));
575 }
576
577 Path outPath = new Path(familyDir, refLink + '.' + clonedRegionName);
578
579
580 InputStream in;
581 if (linkPath != null) {
582 in = new HFileLink(conf, linkPath).open(fs);
583 } else {
584 linkPath = new Path(new Path(HRegion.getRegionDir(snapshotDir, regionInfo.getEncodedName()),
585 familyDir.getName()), hfileName);
586 in = fs.open(linkPath);
587 }
588 OutputStream out = fs.create(outPath);
589 IOUtils.copyBytes(in, out, conf);
590
591
592 String regionName = Bytes.toString(regionsMap.get(regionInfo.getEncodedNameAsBytes()));
593 LOG.debug("Restore reference " + regionName + " to " + clonedRegionName);
594 synchronized (parentsMap) {
595 Pair<String, String> daughters = parentsMap.get(clonedRegionName);
596 if (daughters == null) {
597 daughters = new Pair<String, String>(regionName, null);
598 parentsMap.put(clonedRegionName, daughters);
599 } else if (!regionName.equals(daughters.getFirst())) {
600 daughters.setSecond(regionName);
601 }
602 }
603 }
604
605
606
607
608
609
610
611
612
613 public HRegionInfo cloneRegionInfo(final HRegionInfo snapshotRegionInfo) {
614 HRegionInfo regionInfo = new HRegionInfo(tableDesc.getTableName(),
615 snapshotRegionInfo.getStartKey(), snapshotRegionInfo.getEndKey(),
616 snapshotRegionInfo.isSplit(), snapshotRegionInfo.getRegionId());
617 regionInfo.setOffline(snapshotRegionInfo.isOffline());
618 return regionInfo;
619 }
620
621
622
623
624
625
626
627
628
629
630 private void restoreWALs() throws IOException {
631 final SnapshotLogSplitter logSplitter = new SnapshotLogSplitter(conf, fs, tableDir,
632 snapshotTable, regionsMap);
633
634
635 try {
636
637 SnapshotReferenceUtil.visitRecoveredEdits(fs, snapshotDir,
638 new FSVisitor.RecoveredEditsVisitor() {
639 @Override
640 public void recoveredEdits (final String region, final String logfile) throws IOException {
641 Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
642 logSplitter.splitRecoveredEdit(path);
643 }
644 });
645
646
647 SnapshotReferenceUtil.visitLogFiles(fs, snapshotDir, new FSVisitor.LogFileVisitor() {
648 @Override
649 public void logFile (final String server, final String logfile) throws IOException {
650 logSplitter.splitLog(server, logfile);
651 }
652 });
653 } finally {
654 logSplitter.close();
655 }
656 }
657
658
659
660
661 private List<HRegionInfo> getTableRegions() throws IOException {
662 LOG.debug("get table regions: " + tableDir);
663 FileStatus[] regionDirs = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
664 if (regionDirs == null) return null;
665
666 List<HRegionInfo> regions = new LinkedList<HRegionInfo>();
667 for (FileStatus regionDir: regionDirs) {
668 HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir.getPath());
669 regions.add(hri);
670 }
671 LOG.debug("found " + regions.size() + " regions for table=" +
672 tableDesc.getTableName().getNameAsString());
673 return regions;
674 }
675
676
677
678
679
680
681
682
683
684 public static HTableDescriptor cloneTableSchema(final HTableDescriptor snapshotTableDescriptor,
685 final TableName tableName) throws IOException {
686 HTableDescriptor htd = new HTableDescriptor(tableName);
687 for (HColumnDescriptor hcd: snapshotTableDescriptor.getColumnFamilies()) {
688 htd.addFamily(hcd);
689 }
690 for (Map.Entry<ImmutableBytesWritable, ImmutableBytesWritable> e:
691 snapshotTableDescriptor.getValues().entrySet()) {
692 htd.setValue(e.getKey(), e.getValue());
693 }
694 for (Map.Entry<String, String> e: snapshotTableDescriptor.getConfiguration().entrySet()) {
695 htd.setConfiguration(e.getKey(), e.getValue());
696 }
697 return htd;
698 }
699
700
701
702
703
704
705
706
707
708
709 public static void copySnapshotForScanner(Configuration conf, FileSystem fs, Path rootDir,
710 Path restoreDir, String snapshotName) throws IOException {
711
712 if (!restoreDir.getFileSystem(conf).getUri().equals(rootDir.getFileSystem(conf).getUri())) {
713 throw new IllegalArgumentException("Filesystems for restore directory and HBase root directory " +
714 "should be the same");
715 }
716 if (restoreDir.toUri().getPath().startsWith(rootDir.toUri().getPath())) {
717 throw new IllegalArgumentException("Restore directory cannot be a sub directory of HBase " +
718 "root directory. RootDir: " + rootDir + ", restoreDir: " + restoreDir);
719 }
720
721 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
722 SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
723
724
725 HTableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir);
726
727 MonitoredTask status = TaskMonitor.get().createStatus(
728 "Restoring snapshot '" + snapshotName + "' to directory " + restoreDir);
729 ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher();
730
731 RestoreSnapshotHelper helper = new RestoreSnapshotHelper(conf, fs, snapshotDesc,
732 snapshotDir, htd, restoreDir, monitor, status);
733 helper.restoreHdfsRegions();
734
735 if (LOG.isDebugEnabled()) {
736 LOG.debug("Restored table dir:" + restoreDir);
737 FSUtils.logFileSystemState(fs, restoreDir, LOG);
738 }
739 }
740 }