1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.util;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertFalse;
22 import static org.junit.Assert.assertNotNull;
23 import static org.junit.Assert.assertNull;
24 import static org.junit.Assert.assertTrue;
25 import static org.junit.Assert.fail;
26
27 import java.io.FileNotFoundException;
28 import java.io.IOException;
29 import java.util.Arrays;
30 import java.util.Comparator;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.fs.FileStatus;
35 import org.apache.hadoop.fs.FileSystem;
36 import org.apache.hadoop.fs.Path;
37 import org.apache.hadoop.hbase.TableName;
38 import org.apache.hadoop.hbase.HBaseTestingUtility;
39 import org.apache.hadoop.hbase.HColumnDescriptor;
40 import org.apache.hadoop.hbase.HConstants;
41 import org.apache.hadoop.hbase.HTableDescriptor;
42 import org.apache.hadoop.hbase.MediumTests;
43 import org.apache.hadoop.hbase.TableDescriptors;
44 import org.apache.hadoop.hbase.TableExistsException;
45 import org.junit.Test;
46 import org.junit.experimental.categories.Category;
47
48
49
50
51
52
53 @Category(MediumTests.class)
54 public class TestFSTableDescriptors {
55 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
56 private static final Log LOG = LogFactory.getLog(TestFSTableDescriptors.class);
57
58 @Test (expected=IllegalArgumentException.class)
59 public void testRegexAgainstOldStyleTableInfo() {
60 Path p = new Path("/tmp", FSTableDescriptors.TABLEINFO_FILE_PREFIX);
61 int i = FSTableDescriptors.getTableInfoSequenceId(p);
62 assertEquals(0, i);
63
64 p = new Path("/tmp", "abc");
65 FSTableDescriptors.getTableInfoSequenceId(p);
66 }
67
68 @Test
69 public void testCreateAndUpdate() throws IOException {
70 Path testdir = UTIL.getDataTestDir("testCreateAndUpdate");
71 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testCreate"));
72 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
73 FSTableDescriptors fstd = new FSTableDescriptors(fs, testdir);
74 assertTrue(fstd.createTableDescriptor(htd));
75 assertFalse(fstd.createTableDescriptor(htd));
76 FileStatus [] statuses = fs.listStatus(testdir);
77 assertTrue("statuses.length="+statuses.length, statuses.length == 1);
78 for (int i = 0; i < 10; i++) {
79 fstd.updateTableDescriptor(htd);
80 }
81 statuses = fs.listStatus(testdir);
82 assertTrue(statuses.length == 1);
83 Path tmpTableDir = new Path(FSUtils.getTableDir(testdir, htd.getTableName()), ".tmp");
84 statuses = fs.listStatus(tmpTableDir);
85 assertTrue(statuses.length == 0);
86 }
87
88 @Test
89 public void testSequenceIdAdvancesOnTableInfo() throws IOException {
90 Path testdir = UTIL.getDataTestDir("testSequenceidAdvancesOnTableInfo");
91 HTableDescriptor htd = new HTableDescriptor(
92 TableName.valueOf("testSequenceidAdvancesOnTableInfo"));
93 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
94 FSTableDescriptors fstd = new FSTableDescriptors(fs, testdir);
95 Path p0 = fstd.updateTableDescriptor(htd);
96 int i0 = FSTableDescriptors.getTableInfoSequenceId(p0);
97 Path p1 = fstd.updateTableDescriptor(htd);
98
99 assertTrue(!fs.exists(p0));
100 int i1 = FSTableDescriptors.getTableInfoSequenceId(p1);
101 assertTrue(i1 == i0 + 1);
102 Path p2 = fstd.updateTableDescriptor(htd);
103
104 assertTrue(!fs.exists(p1));
105 int i2 = FSTableDescriptors.getTableInfoSequenceId(p2);
106 assertTrue(i2 == i1 + 1);
107 }
108
109 @Test
110 public void testFormatTableInfoSequenceId() {
111 Path p0 = assertWriteAndReadSequenceId(0);
112
113 StringBuilder sb = new StringBuilder();
114 for (int i = 0; i < FSTableDescriptors.WIDTH_OF_SEQUENCE_ID; i++) {
115 sb.append("0");
116 }
117 assertEquals(FSTableDescriptors.TABLEINFO_FILE_PREFIX + "." + sb.toString(),
118 p0.getName());
119
120 Path p2 = assertWriteAndReadSequenceId(2);
121 Path p10000 = assertWriteAndReadSequenceId(10000);
122
123 Path p = new Path(p0.getParent(), FSTableDescriptors.TABLEINFO_FILE_PREFIX);
124 FileStatus fs = new FileStatus(0, false, 0, 0, 0, p);
125 FileStatus fs0 = new FileStatus(0, false, 0, 0, 0, p0);
126 FileStatus fs2 = new FileStatus(0, false, 0, 0, 0, p2);
127 FileStatus fs10000 = new FileStatus(0, false, 0, 0, 0, p10000);
128 Comparator<FileStatus> comparator = FSTableDescriptors.TABLEINFO_FILESTATUS_COMPARATOR;
129 assertTrue(comparator.compare(fs, fs0) > 0);
130 assertTrue(comparator.compare(fs0, fs2) > 0);
131 assertTrue(comparator.compare(fs2, fs10000) > 0);
132 }
133
134 private Path assertWriteAndReadSequenceId(final int i) {
135 Path p = new Path("/tmp", FSTableDescriptors.getTableInfoFileName(i));
136 int ii = FSTableDescriptors.getTableInfoSequenceId(p);
137 assertEquals(i, ii);
138 return p;
139 }
140
141 @Test
142 public void testRemoves() throws IOException {
143 final String name = "testRemoves";
144 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
145
146 Path rootdir = new Path(UTIL.getDataTestDir(), name);
147 TableDescriptors htds = new FSTableDescriptors(fs, rootdir);
148 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
149 htds.add(htd);
150 assertNotNull(htds.remove(htd.getTableName()));
151 assertNull(htds.remove(htd.getTableName()));
152 }
153
154 @Test public void testReadingHTDFromFS() throws IOException {
155 final String name = "testReadingHTDFromFS";
156 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
157 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
158 Path rootdir = UTIL.getDataTestDir(name);
159 FSTableDescriptors fstd = new FSTableDescriptors(fs, rootdir);
160 fstd.createTableDescriptor(htd);
161 HTableDescriptor htd2 =
162 FSTableDescriptors.getTableDescriptorFromFs(fs, rootdir, htd.getTableName());
163 assertTrue(htd.equals(htd2));
164 }
165
166 @Test public void testHTableDescriptors()
167 throws IOException, InterruptedException {
168 final String name = "testHTableDescriptors";
169 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
170
171 Path rootdir = new Path(UTIL.getDataTestDir(), name);
172 FSTableDescriptors htds = new FSTableDescriptors(fs, rootdir) {
173 @Override
174 public HTableDescriptor get(TableName tablename)
175 throws TableExistsException, FileNotFoundException, IOException {
176 LOG.info(tablename + ", cachehits=" + this.cachehits);
177 return super.get(tablename);
178 }
179 };
180 final int count = 10;
181
182 for (int i = 0; i < count; i++) {
183 HTableDescriptor htd = new HTableDescriptor(name + i);
184 htds.createTableDescriptor(htd);
185 }
186
187 for (int i = 0; i < count; i++) {
188 assertTrue(htds.get(TableName.valueOf(name + i)) != null);
189 }
190 for (int i = 0; i < count; i++) {
191 assertTrue(htds.get(TableName.valueOf(name + i)) != null);
192 }
193
194 for (int i = 0; i < count; i++) {
195 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name + i));
196 htd.addFamily(new HColumnDescriptor("" + i));
197 htds.updateTableDescriptor(htd);
198 }
199
200 Thread.sleep(100);
201 for (int i = 0; i < count; i++) {
202 assertTrue(htds.get(TableName.valueOf(name + i)) != null);
203 }
204 for (int i = 0; i < count; i++) {
205 assertTrue(htds.get(TableName.valueOf(name + i)) != null);
206 }
207 assertEquals(count * 4, htds.invocations);
208 assertTrue("expected=" + (count * 2) + ", actual=" + htds.cachehits,
209 htds.cachehits >= (count * 2));
210 }
211
212 @Test
213 public void testNoSuchTable() throws IOException {
214 final String name = "testNoSuchTable";
215 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
216
217 Path rootdir = new Path(UTIL.getDataTestDir(), name);
218 TableDescriptors htds = new FSTableDescriptors(fs, rootdir);
219 assertNull("There shouldn't be any HTD for this table",
220 htds.get(TableName.valueOf("NoSuchTable")));
221 }
222
223 @Test
224 public void testUpdates() throws IOException {
225 final String name = "testUpdates";
226 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
227
228 Path rootdir = new Path(UTIL.getDataTestDir(), name);
229 TableDescriptors htds = new FSTableDescriptors(fs, rootdir);
230 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
231 htds.add(htd);
232 htds.add(htd);
233 htds.add(htd);
234 }
235
236 @Test
237 public void testTableInfoFileStatusComparator() {
238 FileStatus bare =
239 new FileStatus(0, false, 0, 0, -1,
240 new Path("/tmp", FSTableDescriptors.TABLEINFO_FILE_PREFIX));
241 FileStatus future =
242 new FileStatus(0, false, 0, 0, -1,
243 new Path("/tmp/tablinfo." + System.currentTimeMillis()));
244 FileStatus farFuture =
245 new FileStatus(0, false, 0, 0, -1,
246 new Path("/tmp/tablinfo." + System.currentTimeMillis() + 1000));
247 FileStatus [] alist = {bare, future, farFuture};
248 FileStatus [] blist = {bare, farFuture, future};
249 FileStatus [] clist = {farFuture, bare, future};
250 Comparator<FileStatus> c = FSTableDescriptors.TABLEINFO_FILESTATUS_COMPARATOR;
251 Arrays.sort(alist, c);
252 Arrays.sort(blist, c);
253 Arrays.sort(clist, c);
254
255 for (int i = 0; i < alist.length; i++) {
256 assertTrue(alist[i].equals(blist[i]));
257 assertTrue(blist[i].equals(clist[i]));
258 assertTrue(clist[i].equals(i == 0? farFuture: i == 1? future: bare));
259 }
260 }
261
262 @Test
263 public void testReadingArchiveDirectoryFromFS() throws IOException {
264 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
265 try {
266 new FSTableDescriptors(fs, FSUtils.getRootDir(UTIL.getConfiguration()))
267 .get(TableName.valueOf(HConstants.HFILE_ARCHIVE_DIRECTORY));
268 fail("Shouldn't be able to read a table descriptor for the archive directory.");
269 } catch (Exception e) {
270 LOG.debug("Correctly got error when reading a table descriptor from the archive directory: "
271 + e.getMessage());
272 }
273 }
274
275 @Test
276 public void testCreateTableDescriptorUpdatesIfExistsAlready() throws IOException {
277 Path testdir = UTIL.getDataTestDir("testCreateTableDescriptorUpdatesIfThereExistsAlready");
278 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(
279 "testCreateTableDescriptorUpdatesIfThereExistsAlready"));
280 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
281 FSTableDescriptors fstd = new FSTableDescriptors(fs, testdir);
282 assertTrue(fstd.createTableDescriptor(htd));
283 assertFalse(fstd.createTableDescriptor(htd));
284 htd.setValue(Bytes.toBytes("mykey"), Bytes.toBytes("myValue"));
285 assertTrue(fstd.createTableDescriptor(htd));
286 Path tableDir = fstd.getTableDir(htd.getTableName());
287 Path tmpTableDir = new Path(tableDir, FSTableDescriptors.TMP_DIR);
288 FileStatus[] statuses = fs.listStatus(tmpTableDir);
289 assertTrue(statuses.length == 0);
290
291 assertEquals(htd, FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir));
292 }
293
294 }
295