1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.snapshot;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertTrue;
22
23 import java.io.IOException;
24 import java.util.ArrayList;
25 import java.util.Collections;
26 import java.util.List;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.fs.FileStatus;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.FileUtil;
34 import org.apache.hadoop.fs.Path;
35 import org.apache.hadoop.hbase.HBaseTestingUtility;
36 import org.apache.hadoop.hbase.HColumnDescriptor;
37 import org.apache.hadoop.hbase.HConstants;
38 import org.apache.hadoop.hbase.HRegionInfo;
39 import org.apache.hadoop.hbase.HTableDescriptor;
40 import org.apache.hadoop.hbase.SmallTests;
41 import org.apache.hadoop.hbase.TableName;
42 import org.apache.hadoop.hbase.catalog.CatalogTracker;
43 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
44 import org.apache.hadoop.hbase.io.HFileLink;
45 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
46 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
47 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
48 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
49 import org.apache.hadoop.hbase.util.FSTableDescriptors;
50 import org.apache.hadoop.hbase.util.FSUtils;
51 import org.junit.After;
52 import org.junit.Before;
53 import org.junit.Test;
54 import org.junit.experimental.categories.Category;
55 import org.mockito.Mockito;
56
57
58
59
60 @Category(SmallTests.class)
61 public class TestRestoreSnapshotHelper {
62 final Log LOG = LogFactory.getLog(getClass());
63
64 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
65 private final static String TEST_FAMILY = "cf";
66 private final static String TEST_HFILE = "abc";
67
68 private Configuration conf;
69 private Path archiveDir;
70 private FileSystem fs;
71 private Path rootDir;
72
73 @Before
74 public void setup() throws Exception {
75 rootDir = TEST_UTIL.getDataTestDir("testRestore");
76 archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
77 fs = TEST_UTIL.getTestFileSystem();
78 conf = TEST_UTIL.getConfiguration();
79 FSUtils.setRootDir(conf, rootDir);
80 }
81
82 @After
83 public void tearDown() throws Exception {
84 fs.delete(TEST_UTIL.getDataTestDir(), true);
85 }
86
87 @Test
88 public void testRestore() throws IOException {
89 HTableDescriptor htd = createTableDescriptor("testtb");
90
91 Path snapshotDir = new Path(rootDir, "snapshot");
92 createSnapshot(rootDir, snapshotDir, htd);
93
94
95 HTableDescriptor htdClone = createTableDescriptor("testtb-clone");
96 testRestore(snapshotDir, htd.getTableName().getNameAsString(), htdClone);
97 verifyRestore(rootDir, htd, htdClone);
98
99
100 Path cloneDir = FSUtils.getTableDir(rootDir, htdClone.getTableName());
101 HTableDescriptor htdClone2 = createTableDescriptor("testtb-clone2");
102 testRestore(cloneDir, htdClone.getTableName().getNameAsString(), htdClone2);
103 verifyRestore(rootDir, htd, htdClone2);
104 }
105
106 private void verifyRestore(final Path rootDir, final HTableDescriptor sourceHtd,
107 final HTableDescriptor htdClone) throws IOException {
108 String[] files = getHFiles(FSUtils.getTableDir(rootDir, htdClone.getTableName()));
109 assertEquals(2, files.length);
110 assertTrue(files[0] + " should be a HFileLink", HFileLink.isHFileLink(files[0]));
111 assertTrue(files[1] + " should be a Referene", StoreFileInfo.isReference(files[1]));
112 assertEquals(sourceHtd.getTableName(), HFileLink.getReferencedTableName(files[0]));
113 assertEquals(TEST_HFILE, HFileLink.getReferencedHFileName(files[0]));
114 Path refPath = getReferredToFile(files[1]);
115 assertTrue(refPath.getName() + " should be a HFileLink", HFileLink.isHFileLink(refPath.getName()));
116 assertEquals(files[0], refPath.getName());
117 }
118
119
120
121
122
123
124
125 public void testRestore(final Path snapshotDir, final String sourceTableName,
126 final HTableDescriptor htdClone) throws IOException {
127 LOG.debug("pre-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir);
128 FSUtils.logFileSystemState(fs, rootDir, LOG);
129
130 new FSTableDescriptors(conf).createTableDescriptor(htdClone);
131 RestoreSnapshotHelper helper = getRestoreHelper(rootDir, snapshotDir, sourceTableName, htdClone);
132 helper.restoreHdfsRegions();
133
134 LOG.debug("post-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir);
135 FSUtils.logFileSystemState(fs, rootDir, LOG);
136 }
137
138
139
140
141 private RestoreSnapshotHelper getRestoreHelper(final Path rootDir, final Path snapshotDir,
142 final String sourceTableName, final HTableDescriptor htdClone) throws IOException {
143 CatalogTracker catalogTracker = Mockito.mock(CatalogTracker.class);
144 HTableDescriptor tableDescriptor = Mockito.mock(HTableDescriptor.class);
145 ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class);
146 MonitoredTask status = Mockito.mock(MonitoredTask.class);
147
148 SnapshotDescription sd = SnapshotDescription.newBuilder()
149 .setName("snapshot")
150 .setTable(sourceTableName)
151 .build();
152
153 return new RestoreSnapshotHelper(conf, fs, sd, snapshotDir,
154 htdClone, rootDir, monitor, status);
155 }
156
157 private void createSnapshot(final Path rootDir, final Path snapshotDir, final HTableDescriptor htd)
158 throws IOException {
159
160 HRegionInfo hri = new HRegionInfo(htd.getTableName());
161 HRegionFileSystem r0fs = HRegionFileSystem.createRegionOnFileSystem(conf,
162 fs, FSUtils.getTableDir(archiveDir, hri.getTable()), hri);
163 Path storeFile = new Path(rootDir, TEST_HFILE);
164 fs.createNewFile(storeFile);
165 r0fs.commitStoreFile(TEST_FAMILY, storeFile);
166
167
168
169 hri = new HRegionInfo(htd.getTableName());
170 HRegionFileSystem r1fs = HRegionFileSystem.createRegionOnFileSystem(conf,
171 fs, FSUtils.getTableDir(archiveDir, hri.getTable()), hri);
172 storeFile = new Path(rootDir, TEST_HFILE + '.' + r0fs.getRegionInfo().getEncodedName());
173 fs.createNewFile(storeFile);
174 r1fs.commitStoreFile(TEST_FAMILY, storeFile);
175
176 Path tableDir = FSUtils.getTableDir(archiveDir, htd.getTableName());
177 FileUtil.copy(fs, tableDir, fs, snapshotDir, false, conf);
178 }
179
180 private HTableDescriptor createTableDescriptor(final String tableName) {
181 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
182 htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
183 return htd;
184 }
185
186 private Path getReferredToFile(final String referenceName) {
187 Path fakeBasePath = new Path(new Path("table", "region"), "cf");
188 return StoreFileInfo.getReferredToFile(new Path(fakeBasePath, referenceName));
189 }
190
191 private String[] getHFiles(final Path tableDir) throws IOException {
192 List<String> files = new ArrayList<String>();
193 for (Path regionDir: FSUtils.getRegionDirs(fs, tableDir)) {
194 for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
195 for (FileStatus file: FSUtils.listStatus(fs, familyDir)) {
196 files.add(file.getPath().getName());
197 }
198 }
199 }
200 Collections.sort(files);
201 return files.toArray(new String[files.size()]);
202 }
203 }