1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master.cleaner;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertFalse;
22 import static org.junit.Assert.assertTrue;
23
24 import java.io.IOException;
25
26 import org.apache.hadoop.conf.Configuration;
27 import org.apache.hadoop.fs.FileStatus;
28 import org.apache.hadoop.fs.FileSystem;
29 import org.apache.hadoop.fs.Path;
30 import org.apache.hadoop.hbase.ChoreService;
31 import org.apache.hadoop.hbase.CoordinatedStateManager;
32 import org.apache.hadoop.hbase.HBaseTestingUtility;
33 import org.apache.hadoop.hbase.HRegionInfo;
34 import org.apache.hadoop.hbase.Server;
35 import org.apache.hadoop.hbase.ServerName;
36 import org.apache.hadoop.hbase.TableName;
37 import org.apache.hadoop.hbase.client.ClusterConnection;
38 import org.apache.hadoop.hbase.io.HFileLink;
39 import org.apache.hadoop.hbase.testclassification.SmallTests;
40 import org.apache.hadoop.hbase.util.FSUtils;
41 import org.apache.hadoop.hbase.util.HFileArchiveUtil;
42 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
43 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
44 import org.junit.Test;
45 import org.junit.experimental.categories.Category;
46
47
48
49
50
51 @Category(SmallTests.class)
52 public class TestHFileLinkCleaner {
53
54 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
55
56 @Test
57 public void testHFileLinkCleaning() throws Exception {
58 Configuration conf = TEST_UTIL.getConfiguration();
59 FSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir());
60 conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, HFileLinkCleaner.class.getName());
61 Path rootDir = FSUtils.getRootDir(conf);
62 FileSystem fs = FileSystem.get(conf);
63
64 final TableName tableName = TableName.valueOf("test-table");
65 final TableName tableLinkName = TableName.valueOf("test-link");
66 final String hfileName = "1234567890";
67 final String familyName = "cf";
68
69 HRegionInfo hri = new HRegionInfo(tableName);
70 HRegionInfo hriLink = new HRegionInfo(tableLinkName);
71
72 Path archiveDir = HFileArchiveUtil.getArchivePath(conf);
73 Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf,
74 tableName, hri.getEncodedName(), familyName);
75 Path archiveLinkStoreDir = HFileArchiveUtil.getStoreArchivePath(conf,
76 tableLinkName, hriLink.getEncodedName(), familyName);
77
78
79 Path familyPath = getFamilyDirPath(archiveDir, tableName, hri.getEncodedName(), familyName);
80 fs.mkdirs(familyPath);
81 Path hfilePath = new Path(familyPath, hfileName);
82 fs.createNewFile(hfilePath);
83
84
85 Path familyLinkPath = getFamilyDirPath(rootDir, tableLinkName,
86 hriLink.getEncodedName(), familyName);
87 fs.mkdirs(familyLinkPath);
88 HFileLink.create(conf, fs, familyLinkPath, hri, hfileName);
89 Path linkBackRefDir = HFileLink.getBackReferencesDir(archiveStoreDir, hfileName);
90 assertTrue(fs.exists(linkBackRefDir));
91 FileStatus[] backRefs = fs.listStatus(linkBackRefDir);
92 assertEquals(1, backRefs.length);
93 Path linkBackRef = backRefs[0].getPath();
94
95
96 final long ttl = 1000;
97 conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, ttl);
98 Server server = new DummyServer();
99 HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archiveDir);
100
101
102 cleaner.chore();
103 assertTrue(fs.exists(linkBackRef));
104 assertTrue(fs.exists(hfilePath));
105
106
107 fs.rename(FSUtils.getTableDir(rootDir, tableLinkName),
108 FSUtils.getTableDir(archiveDir, tableLinkName));
109 cleaner.chore();
110 assertFalse("Link should be deleted", fs.exists(linkBackRef));
111
112
113 Thread.sleep(ttl * 2);
114 cleaner.chore();
115 assertFalse("HFile should be deleted", fs.exists(hfilePath));
116
117
118 for (int i = 0; i < 4; ++i) {
119 Thread.sleep(ttl * 2);
120 cleaner.chore();
121 }
122 assertFalse("HFile should be deleted", fs.exists(FSUtils.getTableDir(archiveDir, tableName)));
123 assertFalse("Link should be deleted", fs.exists(FSUtils.getTableDir(archiveDir, tableLinkName)));
124 }
125
126 private static Path getFamilyDirPath (final Path rootDir, final TableName table,
127 final String region, final String family) {
128 return new Path(new Path(FSUtils.getTableDir(rootDir, table), region), family);
129 }
130
131 static class DummyServer implements Server {
132
133 @Override
134 public Configuration getConfiguration() {
135 return TEST_UTIL.getConfiguration();
136 }
137
138 @Override
139 public ZooKeeperWatcher getZooKeeper() {
140 try {
141 return new ZooKeeperWatcher(getConfiguration(), "dummy server", this);
142 } catch (IOException e) {
143 e.printStackTrace();
144 }
145 return null;
146 }
147
148 @Override
149 public CoordinatedStateManager getCoordinatedStateManager() {
150 return null;
151 }
152
153 @Override
154 public ClusterConnection getConnection() {
155 return null;
156 }
157
158 @Override
159 public MetaTableLocator getMetaTableLocator() {
160 return null;
161 }
162
163 @Override
164 public ServerName getServerName() {
165 return ServerName.valueOf("regionserver,60020,000000");
166 }
167
168 @Override
169 public void abort(String why, Throwable e) {}
170
171 @Override
172 public boolean isAborted() {
173 return false;
174 }
175
176 @Override
177 public void stop(String why) {}
178
179 @Override
180 public boolean isStopped() {
181 return false;
182 }
183
184 @Override
185 public ChoreService getChoreService() {
186 return null;
187 }
188 }
189 }