1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master.snapshot;
19
20 import java.io.FileNotFoundException;
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.Collections;
24 import java.util.HashMap;
25 import java.util.HashSet;
26 import java.util.Iterator;
27 import java.util.List;
28 import java.util.Map;
29 import java.util.Set;
30 import java.util.concurrent.ThreadPoolExecutor;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.classification.InterfaceAudience;
35 import org.apache.hadoop.classification.InterfaceStability;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FSDataInputStream;
38 import org.apache.hadoop.fs.FileStatus;
39 import org.apache.hadoop.fs.FileSystem;
40 import org.apache.hadoop.fs.Path;
41 import org.apache.hadoop.hbase.TableName;
42 import org.apache.hadoop.hbase.HConstants;
43 import org.apache.hadoop.hbase.HTableDescriptor;
44 import org.apache.hadoop.hbase.Stoppable;
45 import org.apache.hadoop.hbase.catalog.MetaReader;
46 import org.apache.hadoop.hbase.errorhandling.ForeignException;
47 import org.apache.hadoop.hbase.executor.ExecutorService;
48 import org.apache.hadoop.hbase.master.AssignmentManager;
49 import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
50 import org.apache.hadoop.hbase.master.MasterFileSystem;
51 import org.apache.hadoop.hbase.master.MasterServices;
52 import org.apache.hadoop.hbase.master.MetricsMaster;
53 import org.apache.hadoop.hbase.master.SnapshotSentinel;
54 import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
55 import org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner;
56 import org.apache.hadoop.hbase.procedure.MasterProcedureManager;
57 import org.apache.hadoop.hbase.procedure.Procedure;
58 import org.apache.hadoop.hbase.procedure.ProcedureCoordinator;
59 import org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs;
60 import org.apache.hadoop.hbase.procedure.ZKProcedureCoordinatorRpcs;
61 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair;
62 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ProcedureDescription;
63 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
64 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type;
65 import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
66 import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
67 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
68 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
69 import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
70 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
71 import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
72 import org.apache.hadoop.hbase.snapshot.SnapshotExistsException;
73 import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
74 import org.apache.hadoop.hbase.snapshot.TablePartiallyOpenException;
75 import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
76 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
77 import org.apache.hadoop.hbase.util.FSTableDescriptors;
78 import org.apache.hadoop.hbase.util.FSUtils;
79 import org.apache.zookeeper.KeeperException;
80
81
82
83
84
85
86
87
88
89
90 @InterfaceAudience.Private
91 @InterfaceStability.Unstable
92 public class SnapshotManager extends MasterProcedureManager implements Stoppable {
93 private static final Log LOG = LogFactory.getLog(SnapshotManager.class);
94
95
96 private static final int SNAPSHOT_WAKE_MILLIS_DEFAULT = 500;
97
98
99
100
101
102
103
104
105
106
107
108
109 private static final int SNAPSHOT_SENTINELS_CLEANUP_TIMEOUT = 60 * 1000;
110
111
112 public static final String HBASE_SNAPSHOT_ENABLED = "hbase.snapshot.enabled";
113
114
115
116
117
118 private static final String SNAPSHOT_WAKE_MILLIS_KEY = "hbase.snapshot.master.wakeMillis";
119
120
121 private static final int SNAPSHOT_TIMEOUT_MILLIS_DEFAULT = 60000;
122
123
124
125
126
127 private static final String SNAPSHOT_TIMEOUT_MILLIS_KEY = "hbase.snapshot.master.timeoutMillis";
128
129
130 public static final String ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION = "online-snapshot";
131
132
133 private static final String SNAPSHOT_POOL_THREADS_KEY = "hbase.snapshot.master.threads";
134
135
136 private static final int SNAPSHOT_POOL_THREADS_DEFAULT = 1;
137
138 private boolean stopped;
139 private MasterServices master;
140 private MetricsMaster metricsMaster;
141 private ProcedureCoordinator coordinator;
142
143
144 private boolean isSnapshotSupported = false;
145
146
147
148
149
150 private Map<TableName, SnapshotSentinel> snapshotHandlers =
151 new HashMap<TableName, SnapshotSentinel>();
152
153
154
155
156
157 private Map<TableName, SnapshotSentinel> restoreHandlers =
158 new HashMap<TableName, SnapshotSentinel>();
159
160 private Path rootDir;
161 private ExecutorService executorService;
162
163 public SnapshotManager() {}
164
165
166
167
168
169
170
171 public SnapshotManager(final MasterServices master, final MetricsMaster metricsMaster,
172 ProcedureCoordinator coordinator, ExecutorService pool)
173 throws IOException, UnsupportedOperationException {
174 this.master = master;
175 this.metricsMaster = metricsMaster;
176
177 this.rootDir = master.getMasterFileSystem().getRootDir();
178 checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
179
180 this.coordinator = coordinator;
181 this.executorService = pool;
182 resetTempDir();
183 }
184
185
186
187
188
189
190 public List<SnapshotDescription> getCompletedSnapshots() throws IOException {
191 return getCompletedSnapshots(SnapshotDescriptionUtils.getSnapshotsDir(rootDir));
192 }
193
194
195
196
197
198
199
200 private List<SnapshotDescription> getCompletedSnapshots(Path snapshotDir) throws IOException {
201 List<SnapshotDescription> snapshotDescs = new ArrayList<SnapshotDescription>();
202
203 FileSystem fs = master.getMasterFileSystem().getFileSystem();
204 if (snapshotDir == null) snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
205
206
207 if (!fs.exists(snapshotDir)) {
208 return snapshotDescs;
209 }
210
211
212 FileStatus[] snapshots = fs.listStatus(snapshotDir,
213 new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
214
215 for (FileStatus snapshot : snapshots) {
216 Path info = new Path(snapshot.getPath(), SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
217
218 if (!fs.exists(info)) {
219 LOG.error("Snapshot information for " + snapshot.getPath() + " doesn't exist");
220 continue;
221 }
222 FSDataInputStream in = null;
223 try {
224 in = fs.open(info);
225 SnapshotDescription desc = SnapshotDescription.parseFrom(in);
226 snapshotDescs.add(desc);
227 } catch (IOException e) {
228 LOG.warn("Found a corrupted snapshot " + snapshot.getPath(), e);
229 } finally {
230 if (in != null) {
231 in.close();
232 }
233 }
234 }
235 return snapshotDescs;
236 }
237
238
239
240
241
242
243
244 void resetTempDir() throws IOException {
245
246 Path tmpdir = SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir);
247 if (master.getMasterFileSystem().getFileSystem().exists(tmpdir)) {
248 if (!master.getMasterFileSystem().getFileSystem().delete(tmpdir, true)) {
249 LOG.warn("Couldn't delete working snapshot directory: " + tmpdir);
250 }
251 }
252 }
253
254
255
256
257
258
259
260 public void deleteSnapshot(SnapshotDescription snapshot) throws SnapshotDoesNotExistException, IOException {
261
262
263 MasterCoprocessorHost cpHost = master.getCoprocessorHost();
264 if (cpHost != null) {
265 cpHost.preDeleteSnapshot(snapshot);
266 }
267
268
269 if (!isSnapshotCompleted(snapshot)) {
270 throw new SnapshotDoesNotExistException(snapshot);
271 }
272
273 String snapshotName = snapshot.getName();
274 LOG.debug("Deleting snapshot: " + snapshotName);
275
276 MasterFileSystem fs = master.getMasterFileSystem();
277 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
278
279
280 if (!fs.getFileSystem().delete(snapshotDir, true)) {
281 throw new HBaseSnapshotException("Failed to delete snapshot directory: " + snapshotDir);
282 }
283
284
285 if (cpHost != null) {
286 cpHost.postDeleteSnapshot(snapshot);
287 }
288
289 }
290
291
292
293
294
295
296
297
298
299 public boolean isSnapshotDone(SnapshotDescription expected) throws IOException {
300
301 if (expected == null) {
302 throw new UnknownSnapshotException(
303 "No snapshot name passed in request, can't figure out which snapshot you want to check.");
304 }
305
306 String ssString = ClientSnapshotDescriptionUtils.toString(expected);
307
308
309
310 SnapshotSentinel handler = removeSentinelIfFinished(this.snapshotHandlers, expected);
311
312
313 cleanupSentinels();
314
315 if (handler == null) {
316
317
318
319
320
321
322 if (!isSnapshotCompleted(expected)) {
323 throw new UnknownSnapshotException("Snapshot " + ssString
324 + " is not currently running or one of the known completed snapshots.");
325 }
326
327 return true;
328 }
329
330
331 try {
332 handler.rethrowExceptionIfFailed();
333 } catch (ForeignException e) {
334
335 String status;
336 Procedure p = coordinator.getProcedure(expected.getName());
337 if (p != null) {
338 status = p.getStatus();
339 } else {
340 status = expected.getName() + " not found in proclist " + coordinator.getProcedureNames();
341 }
342 throw new HBaseSnapshotException("Snapshot " + ssString + " had an error. " + status, e,
343 expected);
344 }
345
346
347 if (handler.isFinished()) {
348 LOG.debug("Snapshot '" + ssString + "' has completed, notifying client.");
349 return true;
350 } else if (LOG.isDebugEnabled()) {
351 LOG.debug("Snapshoting '" + ssString + "' is still in progress!");
352 }
353 return false;
354 }
355
356
357
358
359
360
361
362
363
364 synchronized boolean isTakingSnapshot(final SnapshotDescription snapshot) {
365 TableName snapshotTable = TableName.valueOf(snapshot.getTable());
366 if (isTakingSnapshot(snapshotTable)) {
367 return true;
368 }
369 Iterator<Map.Entry<TableName, SnapshotSentinel>> it = this.snapshotHandlers.entrySet().iterator();
370 while (it.hasNext()) {
371 Map.Entry<TableName, SnapshotSentinel> entry = it.next();
372 SnapshotSentinel sentinel = entry.getValue();
373 if (snapshot.getName().equals(sentinel.getSnapshot().getName()) && !sentinel.isFinished()) {
374 return true;
375 }
376 }
377 return false;
378 }
379
380
381
382
383
384
385
386 synchronized boolean isTakingSnapshot(final TableName tableName) {
387 SnapshotSentinel handler = this.snapshotHandlers.get(tableName);
388 return handler != null && !handler.isFinished();
389 }
390
391
392
393
394
395
396
397 private synchronized void prepareToTakeSnapshot(SnapshotDescription snapshot)
398 throws HBaseSnapshotException {
399 FileSystem fs = master.getMasterFileSystem().getFileSystem();
400 Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
401 TableName snapshotTable =
402 TableName.valueOf(snapshot.getTable());
403
404
405 if (isTakingSnapshot(snapshot)) {
406 SnapshotSentinel handler = this.snapshotHandlers.get(snapshotTable);
407 throw new SnapshotCreationException("Rejected taking "
408 + ClientSnapshotDescriptionUtils.toString(snapshot)
409 + " because we are already running another snapshot "
410 + (handler != null ? ("on the same table " +
411 ClientSnapshotDescriptionUtils.toString(handler.getSnapshot()))
412 : "with the same name"), snapshot);
413 }
414
415
416 if (isRestoringTable(snapshotTable)) {
417 SnapshotSentinel handler = restoreHandlers.get(snapshotTable);
418 throw new SnapshotCreationException("Rejected taking "
419 + ClientSnapshotDescriptionUtils.toString(snapshot)
420 + " because we are already have a restore in progress on the same snapshot "
421 + ClientSnapshotDescriptionUtils.toString(handler.getSnapshot()), snapshot);
422 }
423
424 try {
425
426
427 fs.delete(workingDir, true);
428
429
430 if (!fs.mkdirs(workingDir)) {
431 throw new SnapshotCreationException("Couldn't create working directory (" + workingDir
432 + ") for snapshot" , snapshot);
433 }
434 } catch (HBaseSnapshotException e) {
435 throw e;
436 } catch (IOException e) {
437 throw new SnapshotCreationException(
438 "Exception while checking to see if snapshot could be started.", e, snapshot);
439 }
440 }
441
442
443
444
445
446
447 private synchronized void snapshotDisabledTable(SnapshotDescription snapshot)
448 throws HBaseSnapshotException {
449
450 prepareToTakeSnapshot(snapshot);
451
452
453 snapshot = snapshot.toBuilder().setType(Type.DISABLED).build();
454
455
456 DisabledTableSnapshotHandler handler =
457 new DisabledTableSnapshotHandler(snapshot, master);
458 snapshotTable(snapshot, handler);
459 }
460
461
462
463
464
465
466 private synchronized void snapshotEnabledTable(SnapshotDescription snapshot)
467 throws HBaseSnapshotException {
468
469 prepareToTakeSnapshot(snapshot);
470
471
472 EnabledTableSnapshotHandler handler =
473 new EnabledTableSnapshotHandler(snapshot, master, this);
474 snapshotTable(snapshot, handler);
475 }
476
477
478
479
480
481
482
483
484
485 private synchronized void snapshotTable(SnapshotDescription snapshot,
486 final TakeSnapshotHandler handler) throws HBaseSnapshotException {
487 try {
488 handler.prepare();
489 this.executorService.submit(handler);
490 this.snapshotHandlers.put(TableName.valueOf(snapshot.getTable()), handler);
491 } catch (Exception e) {
492
493 Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
494 try {
495 if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true)) {
496 LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
497 ClientSnapshotDescriptionUtils.toString(snapshot));
498 }
499 } catch (IOException e1) {
500 LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
501 ClientSnapshotDescriptionUtils.toString(snapshot));
502 }
503
504 throw new SnapshotCreationException("Could not build snapshot handler", e, snapshot);
505 }
506 }
507
508
509
510
511
512
513
514
515 public void takeSnapshot(SnapshotDescription snapshot) throws IOException {
516
517 if (isSnapshotCompleted(snapshot)) {
518 throw new SnapshotExistsException("Snapshot '" + snapshot.getName()
519 + "' already stored on the filesystem.", snapshot);
520 }
521
522 LOG.debug("No existing snapshot, attempting snapshot...");
523
524
525 cleanupSentinels();
526
527
528 HTableDescriptor desc = null;
529 try {
530 desc = master.getTableDescriptors().get(
531 TableName.valueOf(snapshot.getTable()));
532 } catch (FileNotFoundException e) {
533 String msg = "Table:" + snapshot.getTable() + " info doesn't exist!";
534 LOG.error(msg);
535 throw new SnapshotCreationException(msg, e, snapshot);
536 } catch (IOException e) {
537 throw new SnapshotCreationException("Error while geting table description for table "
538 + snapshot.getTable(), e, snapshot);
539 }
540 if (desc == null) {
541 throw new SnapshotCreationException("Table '" + snapshot.getTable()
542 + "' doesn't exist, can't take snapshot.", snapshot);
543 }
544
545
546 snapshot = snapshot.toBuilder().setVersion(SnapshotDescriptionUtils.SNAPSHOT_LAYOUT_VERSION)
547 .build();
548
549
550 MasterCoprocessorHost cpHost = master.getCoprocessorHost();
551 if (cpHost != null) {
552 cpHost.preSnapshot(snapshot, desc);
553 }
554
555
556 TableName snapshotTable = TableName.valueOf(snapshot.getTable());
557 AssignmentManager assignmentMgr = master.getAssignmentManager();
558 if (assignmentMgr.getZKTable().isEnabledTable(snapshotTable)) {
559 LOG.debug("Table enabled, starting distributed snapshot.");
560 snapshotEnabledTable(snapshot);
561 LOG.debug("Started snapshot: " + ClientSnapshotDescriptionUtils.toString(snapshot));
562 }
563
564 else if (assignmentMgr.getZKTable().isDisabledTable(snapshotTable)) {
565 LOG.debug("Table is disabled, running snapshot entirely on master.");
566 snapshotDisabledTable(snapshot);
567 LOG.debug("Started snapshot: " + ClientSnapshotDescriptionUtils.toString(snapshot));
568 } else {
569 LOG.error("Can't snapshot table '" + snapshot.getTable()
570 + "', isn't open or closed, we don't know what to do!");
571 TablePartiallyOpenException tpoe = new TablePartiallyOpenException(snapshot.getTable()
572 + " isn't fully open.");
573 throw new SnapshotCreationException("Table is not entirely open or closed", tpoe, snapshot);
574 }
575
576
577 if (cpHost != null) {
578 cpHost.postSnapshot(snapshot, desc);
579 }
580 }
581
582
583
584
585
586
587
588
589
590
591 public synchronized void setSnapshotHandlerForTesting(
592 final TableName tableName,
593 final SnapshotSentinel handler) {
594 if (handler != null) {
595 this.snapshotHandlers.put(tableName, handler);
596 } else {
597 this.snapshotHandlers.remove(tableName);
598 }
599 }
600
601
602
603
604 ProcedureCoordinator getCoordinator() {
605 return coordinator;
606 }
607
608
609
610
611
612
613
614
615
616
617
618 private boolean isSnapshotCompleted(SnapshotDescription snapshot) throws IOException {
619 try {
620 final Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
621 FileSystem fs = master.getMasterFileSystem().getFileSystem();
622
623 return fs.exists(snapshotDir);
624 } catch (IllegalArgumentException iae) {
625 throw new UnknownSnapshotException("Unexpected exception thrown", iae);
626 }
627 }
628
629
630
631
632
633
634
635
636 synchronized void cloneSnapshot(final SnapshotDescription snapshot,
637 final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
638 TableName tableName = hTableDescriptor.getTableName();
639
640
641 if (isTakingSnapshot(tableName)) {
642 throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
643 }
644
645
646 if (isRestoringTable(tableName)) {
647 throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
648 }
649
650 try {
651 CloneSnapshotHandler handler =
652 new CloneSnapshotHandler(master, snapshot, hTableDescriptor).prepare();
653 this.executorService.submit(handler);
654 this.restoreHandlers.put(tableName, handler);
655 } catch (Exception e) {
656 String msg = "Couldn't clone the snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) +
657 " on table=" + tableName;
658 LOG.error(msg, e);
659 throw new RestoreSnapshotException(msg, e);
660 }
661 }
662
663
664
665
666
667
668 public void restoreSnapshot(SnapshotDescription reqSnapshot) throws IOException {
669 FileSystem fs = master.getMasterFileSystem().getFileSystem();
670 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(reqSnapshot, rootDir);
671 MasterCoprocessorHost cpHost = master.getCoprocessorHost();
672
673
674 if (!fs.exists(snapshotDir)) {
675 LOG.error("A Snapshot named '" + reqSnapshot.getName() + "' does not exist.");
676 throw new SnapshotDoesNotExistException(reqSnapshot);
677 }
678
679
680 SnapshotDescription fsSnapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
681 HTableDescriptor snapshotTableDesc =
682 FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir);
683 TableName tableName = TableName.valueOf(reqSnapshot.getTable());
684
685
686 cleanupSentinels();
687
688
689 SnapshotReferenceUtil.verifySnapshot(master.getConfiguration(), fs, snapshotDir, fsSnapshot);
690
691
692 if (MetaReader.tableExists(master.getCatalogTracker(), tableName)) {
693 if (master.getAssignmentManager().getZKTable().isEnabledTable(
694 TableName.valueOf(fsSnapshot.getTable()))) {
695 throw new UnsupportedOperationException("Table '" +
696 TableName.valueOf(fsSnapshot.getTable()) + "' must be disabled in order to " +
697 "perform a restore operation" +
698 ".");
699 }
700
701
702 if (cpHost != null) {
703 cpHost.preRestoreSnapshot(reqSnapshot, snapshotTableDesc);
704 }
705 restoreSnapshot(fsSnapshot, snapshotTableDesc);
706 LOG.info("Restore snapshot=" + fsSnapshot.getName() + " as table=" + tableName);
707
708 if (cpHost != null) {
709 cpHost.postRestoreSnapshot(reqSnapshot, snapshotTableDesc);
710 }
711 } else {
712 HTableDescriptor htd = RestoreSnapshotHelper.cloneTableSchema(snapshotTableDesc, tableName);
713 if (cpHost != null) {
714 cpHost.preCloneSnapshot(reqSnapshot, htd);
715 }
716 cloneSnapshot(fsSnapshot, htd);
717 LOG.info("Clone snapshot=" + fsSnapshot.getName() + " as table=" + tableName);
718
719 if (cpHost != null) {
720 cpHost.postCloneSnapshot(reqSnapshot, htd);
721 }
722 }
723 }
724
725
726
727
728
729
730
731
732 private synchronized void restoreSnapshot(final SnapshotDescription snapshot,
733 final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
734 TableName tableName = hTableDescriptor.getTableName();
735
736
737 if (isTakingSnapshot(tableName)) {
738 throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
739 }
740
741
742 if (isRestoringTable(tableName)) {
743 throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
744 }
745
746 try {
747 RestoreSnapshotHandler handler =
748 new RestoreSnapshotHandler(master, snapshot, hTableDescriptor).prepare();
749 this.executorService.submit(handler);
750 restoreHandlers.put(tableName, handler);
751 } catch (Exception e) {
752 String msg = "Couldn't restore the snapshot=" + ClientSnapshotDescriptionUtils.toString(
753 snapshot) +
754 " on table=" + tableName;
755 LOG.error(msg, e);
756 throw new RestoreSnapshotException(msg, e);
757 }
758 }
759
760
761
762
763
764
765
766 private synchronized boolean isRestoringTable(final TableName tableName) {
767 SnapshotSentinel sentinel = this.restoreHandlers.get(tableName);
768 return(sentinel != null && !sentinel.isFinished());
769 }
770
771
772
773
774
775
776
777
778
779 public boolean isRestoreDone(final SnapshotDescription snapshot) throws IOException {
780
781
782 SnapshotSentinel sentinel = removeSentinelIfFinished(this.restoreHandlers, snapshot);
783
784
785 cleanupSentinels();
786
787 if (sentinel == null) {
788
789 return true;
790 }
791
792 LOG.debug("Verify snapshot=" + snapshot.getName() + " against="
793 + sentinel.getSnapshot().getName() + " table=" +
794 TableName.valueOf(snapshot.getTable()));
795
796
797 sentinel.rethrowExceptionIfFailed();
798
799
800 if (sentinel.isFinished()) {
801 LOG.debug("Restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) +
802 " has completed. Notifying the client.");
803 return true;
804 }
805
806 if (LOG.isDebugEnabled()) {
807 LOG.debug("Sentinel is not yet finished with restoring snapshot=" +
808 ClientSnapshotDescriptionUtils.toString(snapshot));
809 }
810 return false;
811 }
812
813
814
815
816
817
818
819
820 private synchronized SnapshotSentinel removeSentinelIfFinished(
821 final Map<TableName, SnapshotSentinel> sentinels,
822 final SnapshotDescription snapshot) {
823 if (!snapshot.hasTable()) {
824 return null;
825 }
826
827 TableName snapshotTable = TableName.valueOf(snapshot.getTable());
828 SnapshotSentinel h = sentinels.get(snapshotTable);
829 if (h == null) {
830 return null;
831 }
832
833 if (!h.getSnapshot().getName().equals(snapshot.getName())) {
834
835 return null;
836 }
837
838
839 if (h.isFinished()) {
840 sentinels.remove(snapshotTable);
841 }
842
843 return h;
844 }
845
846
847
848
849
850
851
852
853 private void cleanupSentinels() {
854 cleanupSentinels(this.snapshotHandlers);
855 cleanupSentinels(this.restoreHandlers);
856 }
857
858
859
860
861
862
863 private synchronized void cleanupSentinels(final Map<TableName, SnapshotSentinel> sentinels) {
864 long currentTime = EnvironmentEdgeManager.currentTimeMillis();
865 Iterator<Map.Entry<TableName, SnapshotSentinel>> it =
866 sentinels.entrySet().iterator();
867 while (it.hasNext()) {
868 Map.Entry<TableName, SnapshotSentinel> entry = it.next();
869 SnapshotSentinel sentinel = entry.getValue();
870 if (sentinel.isFinished() &&
871 (currentTime - sentinel.getCompletionTimestamp()) > SNAPSHOT_SENTINELS_CLEANUP_TIMEOUT)
872 {
873 it.remove();
874 }
875 }
876 }
877
878
879
880
881
882 @Override
883 public void stop(String why) {
884
885 if (this.stopped) return;
886
887 this.stopped = true;
888
889 for (SnapshotSentinel snapshotHandler: this.snapshotHandlers.values()) {
890 snapshotHandler.cancel(why);
891 }
892
893
894 for (SnapshotSentinel restoreHandler: this.restoreHandlers.values()) {
895 restoreHandler.cancel(why);
896 }
897 try {
898 coordinator.close();
899 } catch (IOException e) {
900 LOG.error("stop ProcedureCoordinator error", e);
901 }
902 }
903
904 @Override
905 public boolean isStopped() {
906 return this.stopped;
907 }
908
909
910
911
912
913
914 public void checkSnapshotSupport() throws UnsupportedOperationException {
915 if (!this.isSnapshotSupported) {
916 throw new UnsupportedOperationException(
917 "To use snapshots, You must add to the hbase-site.xml of the HBase Master: '" +
918 HBASE_SNAPSHOT_ENABLED + "' property with value 'true'.");
919 }
920 }
921
922
923
924
925
926
927
928
929
930
931
932 private void checkSnapshotSupport(final Configuration conf, final MasterFileSystem mfs)
933 throws IOException, UnsupportedOperationException {
934
935 String enabled = conf.get(HBASE_SNAPSHOT_ENABLED);
936 boolean snapshotEnabled = conf.getBoolean(HBASE_SNAPSHOT_ENABLED, false);
937 boolean userDisabled = (enabled != null && enabled.trim().length() > 0 && !snapshotEnabled);
938
939
940 Set<String> hfileCleaners = new HashSet<String>();
941 String[] cleaners = conf.getStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS);
942 if (cleaners != null) Collections.addAll(hfileCleaners, cleaners);
943
944 Set<String> logCleaners = new HashSet<String>();
945 cleaners = conf.getStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS);
946 if (cleaners != null) Collections.addAll(logCleaners, cleaners);
947
948
949 Path oldSnapshotDir = new Path(mfs.getRootDir(), HConstants.OLD_SNAPSHOT_DIR_NAME);
950 FileSystem fs = mfs.getFileSystem();
951 List<SnapshotDescription> ss = getCompletedSnapshots(new Path(rootDir, oldSnapshotDir));
952 if (ss != null && !ss.isEmpty()) {
953 LOG.error("Snapshots from an earlier release were found under: " + oldSnapshotDir);
954 LOG.error("Please rename the directory as " + HConstants.SNAPSHOT_DIR_NAME);
955 }
956
957
958
959
960 if (snapshotEnabled) {
961
962 hfileCleaners.add(SnapshotHFileCleaner.class.getName());
963 hfileCleaners.add(HFileLinkCleaner.class.getName());
964 logCleaners.add(SnapshotLogCleaner.class.getName());
965
966
967 conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS,
968 hfileCleaners.toArray(new String[hfileCleaners.size()]));
969 conf.setStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS,
970 logCleaners.toArray(new String[logCleaners.size()]));
971 } else {
972
973 snapshotEnabled = logCleaners.contains(SnapshotLogCleaner.class.getName()) &&
974 hfileCleaners.contains(SnapshotHFileCleaner.class.getName()) &&
975 hfileCleaners.contains(HFileLinkCleaner.class.getName());
976
977
978 if (snapshotEnabled) {
979 LOG.warn("Snapshot log and hfile cleaners are present in the configuration, " +
980 "but the '" + HBASE_SNAPSHOT_ENABLED + "' property " +
981 (userDisabled ? "is set to 'false'." : "is not set."));
982 }
983 }
984
985
986 this.isSnapshotSupported = snapshotEnabled && !userDisabled;
987
988
989
990 if (!snapshotEnabled) {
991 LOG.info("Snapshot feature is not enabled, missing log and hfile cleaners.");
992 Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(mfs.getRootDir());
993 if (fs.exists(snapshotDir)) {
994 FileStatus[] snapshots = FSUtils.listStatus(fs, snapshotDir,
995 new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
996 if (snapshots != null) {
997 LOG.error("Snapshots are present, but cleaners are not enabled.");
998 checkSnapshotSupport();
999 }
1000 }
1001 }
1002 }
1003
1004 @Override
1005 public void initialize(MasterServices master, MetricsMaster metricsMaster) throws KeeperException,
1006 IOException, UnsupportedOperationException {
1007 this.master = master;
1008 this.metricsMaster = metricsMaster;
1009
1010 this.rootDir = master.getMasterFileSystem().getRootDir();
1011 checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
1012
1013
1014 Configuration conf = master.getConfiguration();
1015 long wakeFrequency = conf.getInt(SNAPSHOT_WAKE_MILLIS_KEY, SNAPSHOT_WAKE_MILLIS_DEFAULT);
1016 long timeoutMillis = conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, SNAPSHOT_TIMEOUT_MILLIS_DEFAULT);
1017 int opThreads = conf.getInt(SNAPSHOT_POOL_THREADS_KEY, SNAPSHOT_POOL_THREADS_DEFAULT);
1018
1019
1020 String name = master.getServerName().toString();
1021 ThreadPoolExecutor tpool = ProcedureCoordinator.defaultPool(name, opThreads);
1022 ProcedureCoordinatorRpcs comms = new ZKProcedureCoordinatorRpcs(
1023 master.getZooKeeper(), SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION, name);
1024
1025 this.coordinator = new ProcedureCoordinator(comms, tpool, timeoutMillis, wakeFrequency);
1026 this.executorService = master.getExecutorService();
1027 resetTempDir();
1028 }
1029
1030 @Override
1031 public String getProcedureSignature() {
1032 return ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION;
1033 }
1034
1035 @Override
1036 public void execProcedure(ProcedureDescription desc) throws IOException {
1037 takeSnapshot(toSnapshotDescription(desc));
1038 }
1039
1040 @Override
1041 public boolean isProcedureDone(ProcedureDescription desc) throws IOException {
1042 return isSnapshotDone(toSnapshotDescription(desc));
1043 }
1044
1045 private SnapshotDescription toSnapshotDescription(ProcedureDescription desc)
1046 throws IOException {
1047 SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
1048 if (!desc.hasInstance()) {
1049 throw new IOException("Snapshot name is not defined: " + desc.toString());
1050 }
1051 String snapshotName = desc.getInstance();
1052 List<NameStringPair> props = desc.getConfigurationList();
1053 String table = null;
1054 for (NameStringPair prop : props) {
1055 if ("table".equalsIgnoreCase(prop.getName())) {
1056 table = prop.getValue();
1057 }
1058 }
1059 if (table == null) {
1060 throw new IOException("Snapshot table is not defined: " + desc.toString());
1061 }
1062 TableName tableName = TableName.valueOf(table);
1063 builder.setTable(tableName.getNameAsString());
1064 builder.setName(snapshotName);
1065 builder.setType(SnapshotDescription.Type.FLUSH);
1066 return builder.build();
1067 }
1068 }