1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.mapreduce;
20
21 import org.apache.hadoop.conf.Configuration;
22 import org.apache.hadoop.fs.FileStatus;
23 import org.apache.hadoop.fs.FileSystem;
24 import org.apache.hadoop.fs.Path;
25 import org.apache.hadoop.hbase.CategoryBasedTimeout;
26 import org.apache.hadoop.hbase.Cell;
27 import org.apache.hadoop.hbase.CellScanner;
28 import org.apache.hadoop.hbase.HBaseTestingUtility;
29 import org.apache.hadoop.hbase.TableName;
30 import org.apache.hadoop.hbase.client.Admin;
31 import org.apache.hadoop.hbase.client.HTable;
32 import org.apache.hadoop.hbase.client.Result;
33 import org.apache.hadoop.hbase.io.HFileLink;
34 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
35 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
36 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
37 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
38 import org.apache.hadoop.hbase.util.Bytes;
39 import org.apache.hadoop.hbase.util.FSUtils;
40 import org.apache.hadoop.hbase.util.HFileArchiveUtil;
41 import org.junit.Assert;
42 import org.junit.Rule;
43 import org.junit.Test;
44 import org.junit.rules.TestRule;
45
46 import static org.junit.Assert.assertFalse;
47
48 import java.io.IOException;
49 import java.util.Arrays;
50
51 public abstract class TableSnapshotInputFormatTestBase {
52 @Rule public final TestRule timeout = CategoryBasedTimeout.builder().
53 withTimeout(this.getClass()).withLookingForStuckThread(true).build();
54 protected final HBaseTestingUtility UTIL = new HBaseTestingUtility();
55 protected static final int NUM_REGION_SERVERS = 2;
56 protected static final byte[][] FAMILIES = {Bytes.toBytes("f1"), Bytes.toBytes("f2")};
57
58 protected FileSystem fs;
59 protected Path rootDir;
60
61 public void setupCluster() throws Exception {
62 setupConf(UTIL.getConfiguration());
63 UTIL.setJobWithoutMRCluster();
64 UTIL.startMiniCluster(NUM_REGION_SERVERS, true);
65 rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
66 fs = rootDir.getFileSystem(UTIL.getConfiguration());
67 }
68
69 public void tearDownCluster() throws Exception {
70 UTIL.shutdownMiniCluster();
71 }
72
73 private static void setupConf(Configuration conf) {
74
75 conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
76 }
77
78 protected abstract void testWithMockedMapReduce(HBaseTestingUtility util, String snapshotName,
79 int numRegions, int expectedNumSplits) throws Exception;
80
81 protected abstract void testWithMapReduceImpl(HBaseTestingUtility util, TableName tableName,
82 String snapshotName, Path tableDir, int numRegions, int expectedNumSplits,
83 boolean shutdownCluster) throws Exception;
84
85 protected abstract byte[] getStartRow();
86
87 protected abstract byte[] getEndRow();
88
89 @Test
90 public void testWithMockedMapReduceSingleRegion() throws Exception {
91 testWithMockedMapReduce(UTIL, "testWithMockedMapReduceSingleRegion", 1, 1);
92 }
93
94 @Test
95 public void testWithMockedMapReduceMultiRegion() throws Exception {
96 testWithMockedMapReduce(UTIL, "testWithMockedMapReduceMultiRegion", 10, 8);
97 }
98
99 @Test
100 public void testWithMapReduceSingleRegion() throws Exception {
101 testWithMapReduce(UTIL, "testWithMapReduceSingleRegion", 1, 1, false);
102 }
103
104 @Test
105 public void testWithMapReduceMultiRegion() throws Exception {
106 testWithMapReduce(UTIL, "testWithMapReduceMultiRegion", 10, 8, false);
107 }
108
109 @Test
110
111 public void testWithMapReduceAndOfflineHBaseMultiRegion() throws Exception {
112 testWithMapReduce(UTIL, "testWithMapReduceAndOfflineHBaseMultiRegion", 10, 8, true);
113 }
114
115
116 @Test
117 public void testRestoreSnapshotDoesNotCreateBackRefLinks() throws Exception {
118 setupCluster();
119 TableName tableName = TableName.valueOf("testRestoreSnapshotDoesNotCreateBackRefLinks");
120 String snapshotName = "foo";
121
122 try {
123 createTableAndSnapshot(UTIL, tableName, snapshotName, getStartRow(), getEndRow(), 1);
124
125 Path tmpTableDir = UTIL.getRandomDir();
126
127 testRestoreSnapshotDoesNotCreateBackRefLinksInit(tableName, snapshotName,tmpTableDir);
128
129 Path rootDir = FSUtils.getRootDir(UTIL.getConfiguration());
130 for (Path regionDir : FSUtils.getRegionDirs(fs, FSUtils.getTableDir(rootDir, tableName))) {
131 for (Path storeDir : FSUtils.getFamilyDirs(fs, regionDir)) {
132 for (FileStatus status : fs.listStatus(storeDir)) {
133 System.out.println(status.getPath());
134 if (StoreFileInfo.isValid(status)) {
135 Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(UTIL.getConfiguration(),
136 tableName, regionDir.getName(), storeDir.getName());
137
138 Path path = HFileLink.getBackReferencesDir(storeDir, status.getPath().getName());
139
140 assertFalse("There is a back reference in " + path, fs.exists(path));
141
142 path = HFileLink.getBackReferencesDir(archiveStoreDir, status.getPath().getName());
143
144 assertFalse("There is a back reference in " + path, fs.exists(path));
145 }
146 }
147 }
148 }
149 } finally {
150 UTIL.getHBaseAdmin().deleteSnapshot(snapshotName);
151 UTIL.deleteTable(tableName);
152 tearDownCluster();
153 }
154 }
155
156 public abstract void testRestoreSnapshotDoesNotCreateBackRefLinksInit(TableName tableName,
157 String snapshotName, Path tmpTableDir) throws Exception;
158
159 protected void testWithMapReduce(HBaseTestingUtility util, String snapshotName,
160 int numRegions, int expectedNumSplits, boolean shutdownCluster) throws Exception {
161 setupCluster();
162 try {
163 Path tableDir = util.getRandomDir();
164 TableName tableName = TableName.valueOf("testWithMapReduce");
165 testWithMapReduceImpl(util, tableName, snapshotName, tableDir, numRegions,
166 expectedNumSplits, shutdownCluster);
167 } finally {
168 tearDownCluster();
169 }
170 }
171
172 protected static void verifyRowFromMap(ImmutableBytesWritable key, Result result)
173 throws IOException {
174 byte[] row = key.get();
175 CellScanner scanner = result.cellScanner();
176 while (scanner.advance()) {
177 Cell cell = scanner.current();
178
179
180 Assert.assertEquals(0, Bytes.compareTo(row, 0, row.length,
181 cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()));
182 }
183
184 for (int j = 0; j < FAMILIES.length; j++) {
185 byte[] actual = result.getValue(FAMILIES[j], null);
186 Assert.assertArrayEquals("Row in snapshot does not match, expected:" + Bytes.toString(row)
187 + " ,actual:" + Bytes.toString(actual), row, actual);
188 }
189 }
190
191 protected static void createTableAndSnapshot(HBaseTestingUtility util, TableName tableName,
192 String snapshotName, byte[] startRow, byte[] endRow, int numRegions)
193 throws Exception {
194 try {
195 util.deleteTable(tableName);
196 } catch(Exception ex) {
197
198 }
199
200 if (numRegions > 1) {
201 util.createTable(tableName, FAMILIES, 1, startRow, endRow, numRegions);
202 } else {
203 util.createTable(tableName, FAMILIES);
204 }
205 Admin admin = util.getHBaseAdmin();
206
207
208 HTable table = new HTable(util.getConfiguration(), tableName);
209 util.loadTable(table, FAMILIES);
210
211 Path rootDir = FSUtils.getRootDir(util.getConfiguration());
212 FileSystem fs = rootDir.getFileSystem(util.getConfiguration());
213
214 SnapshotTestingUtils.createSnapshotAndValidate(admin, tableName,
215 Arrays.asList(FAMILIES), null, snapshotName, rootDir, fs, true);
216
217
218 byte[] value = Bytes.toBytes("after_snapshot_value");
219 util.loadTable(table, FAMILIES, value);
220
221
222 admin.flush(tableName);
223 table.close();
224 }
225
226 }