View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.util;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertNotNull;
23  import static org.junit.Assert.assertNull;
24  import static org.junit.Assert.assertTrue;
25  import static org.junit.Assert.fail;
26  
27  import java.io.FileNotFoundException;
28  import java.io.IOException;
29  import java.util.Arrays;
30  import java.util.Comparator;
31  
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.fs.FileStatus;
35  import org.apache.hadoop.fs.FileSystem;
36  import org.apache.hadoop.fs.Path;
37  import org.apache.hadoop.hbase.TableName;
38  import org.apache.hadoop.hbase.HBaseTestingUtility;
39  import org.apache.hadoop.hbase.HColumnDescriptor;
40  import org.apache.hadoop.hbase.HConstants;
41  import org.apache.hadoop.hbase.HTableDescriptor;
42  import org.apache.hadoop.hbase.MediumTests;
43  import org.apache.hadoop.hbase.TableDescriptors;
44  import org.apache.hadoop.hbase.TableExistsException;
45  import org.junit.Test;
46  import org.junit.experimental.categories.Category;
47  
48  
49  /**
50   * Tests for {@link FSTableDescriptors}.
51   */
52  // Do not support to be executed in he same JVM as other tests
53  @Category(MediumTests.class)
54  public class TestFSTableDescriptors {
55    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
56    private static final Log LOG = LogFactory.getLog(TestFSTableDescriptors.class);
57  
58    @Test (expected=IllegalArgumentException.class)
59    public void testRegexAgainstOldStyleTableInfo() {
60      Path p = new Path("/tmp", FSTableDescriptors.TABLEINFO_FILE_PREFIX);
61      int i = FSTableDescriptors.getTableInfoSequenceId(p);
62      assertEquals(0, i);
63      // Assert it won't eat garbage -- that it fails
64      p = new Path("/tmp", "abc");
65      FSTableDescriptors.getTableInfoSequenceId(p);
66    }
67  
68    @Test
69    public void testCreateAndUpdate() throws IOException {
70      Path testdir = UTIL.getDataTestDir("testCreateAndUpdate");
71      HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testCreate"));
72      FileSystem fs = FileSystem.get(UTIL.getConfiguration());
73      FSTableDescriptors fstd = new FSTableDescriptors(fs, testdir);
74      assertTrue(fstd.createTableDescriptor(htd));
75      assertFalse(fstd.createTableDescriptor(htd));
76      FileStatus [] statuses = fs.listStatus(testdir);
77      assertTrue("statuses.length="+statuses.length, statuses.length == 1);
78      for (int i = 0; i < 10; i++) {
79        fstd.updateTableDescriptor(htd);
80      }
81      statuses = fs.listStatus(testdir);
82      assertTrue(statuses.length == 1);
83      Path tmpTableDir = new Path(FSUtils.getTableDir(testdir, htd.getTableName()), ".tmp");
84      statuses = fs.listStatus(tmpTableDir);
85      assertTrue(statuses.length == 0);
86    }
87  
88    @Test
89    public void testSequenceIdAdvancesOnTableInfo() throws IOException {
90      Path testdir = UTIL.getDataTestDir("testSequenceidAdvancesOnTableInfo");
91      HTableDescriptor htd = new HTableDescriptor(
92          TableName.valueOf("testSequenceidAdvancesOnTableInfo"));
93      FileSystem fs = FileSystem.get(UTIL.getConfiguration());
94      FSTableDescriptors fstd = new FSTableDescriptors(fs, testdir);
95      Path p0 = fstd.updateTableDescriptor(htd);
96      int i0 = FSTableDescriptors.getTableInfoSequenceId(p0);
97      Path p1 = fstd.updateTableDescriptor(htd);
98      // Assert we cleaned up the old file.
99      assertTrue(!fs.exists(p0));
100     int i1 = FSTableDescriptors.getTableInfoSequenceId(p1);
101     assertTrue(i1 == i0 + 1);
102     Path p2 = fstd.updateTableDescriptor(htd);
103     // Assert we cleaned up the old file.
104     assertTrue(!fs.exists(p1));
105     int i2 = FSTableDescriptors.getTableInfoSequenceId(p2);
106     assertTrue(i2 == i1 + 1);
107   }
108 
109   @Test
110   public void testFormatTableInfoSequenceId() {
111     Path p0 = assertWriteAndReadSequenceId(0);
112     // Assert p0 has format we expect.
113     StringBuilder sb = new StringBuilder();
114     for (int i = 0; i < FSTableDescriptors.WIDTH_OF_SEQUENCE_ID; i++) {
115       sb.append("0");
116     }
117     assertEquals(FSTableDescriptors.TABLEINFO_FILE_PREFIX + "." + sb.toString(),
118       p0.getName());
119     // Check a few more.
120     Path p2 = assertWriteAndReadSequenceId(2);
121     Path p10000 = assertWriteAndReadSequenceId(10000);
122     // Get a .tablinfo that has no sequenceid suffix.
123     Path p = new Path(p0.getParent(), FSTableDescriptors.TABLEINFO_FILE_PREFIX);
124     FileStatus fs = new FileStatus(0, false, 0, 0, 0, p);
125     FileStatus fs0 = new FileStatus(0, false, 0, 0, 0, p0);
126     FileStatus fs2 = new FileStatus(0, false, 0, 0, 0, p2);
127     FileStatus fs10000 = new FileStatus(0, false, 0, 0, 0, p10000);
128     Comparator<FileStatus> comparator = FSTableDescriptors.TABLEINFO_FILESTATUS_COMPARATOR;
129     assertTrue(comparator.compare(fs, fs0) > 0);
130     assertTrue(comparator.compare(fs0, fs2) > 0);
131     assertTrue(comparator.compare(fs2, fs10000) > 0);
132   }
133 
134   private Path assertWriteAndReadSequenceId(final int i) {
135     Path p = new Path("/tmp", FSTableDescriptors.getTableInfoFileName(i));
136     int ii = FSTableDescriptors.getTableInfoSequenceId(p);
137     assertEquals(i, ii);
138     return p;
139   }
140 
141   @Test
142   public void testRemoves() throws IOException {
143     final String name = "testRemoves";
144     FileSystem fs = FileSystem.get(UTIL.getConfiguration());
145     // Cleanup old tests if any detrius laying around.
146     Path rootdir = new Path(UTIL.getDataTestDir(), name);
147     TableDescriptors htds = new FSTableDescriptors(fs, rootdir);
148     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
149     htds.add(htd);
150     assertNotNull(htds.remove(htd.getTableName()));
151     assertNull(htds.remove(htd.getTableName()));
152   }
153 
154   @Test public void testReadingHTDFromFS() throws IOException {
155     final String name = "testReadingHTDFromFS";
156     FileSystem fs = FileSystem.get(UTIL.getConfiguration());
157     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
158     Path rootdir = UTIL.getDataTestDir(name);
159     FSTableDescriptors fstd = new FSTableDescriptors(fs, rootdir);
160     fstd.createTableDescriptor(htd);
161     HTableDescriptor htd2 =
162       FSTableDescriptors.getTableDescriptorFromFs(fs, rootdir, htd.getTableName());
163     assertTrue(htd.equals(htd2));
164   }
165 
166   @Test public void testHTableDescriptors()
167   throws IOException, InterruptedException {
168     final String name = "testHTableDescriptors";
169     FileSystem fs = FileSystem.get(UTIL.getConfiguration());
170     // Cleanup old tests if any debris laying around.
171     Path rootdir = new Path(UTIL.getDataTestDir(), name);
172     FSTableDescriptors htds = new FSTableDescriptors(fs, rootdir) {
173       @Override
174       public HTableDescriptor get(TableName tablename)
175           throws TableExistsException, FileNotFoundException, IOException {
176         LOG.info(tablename + ", cachehits=" + this.cachehits);
177         return super.get(tablename);
178       }
179     };
180     final int count = 10;
181     // Write out table infos.
182     for (int i = 0; i < count; i++) {
183       HTableDescriptor htd = new HTableDescriptor(name + i);
184       htds.createTableDescriptor(htd);
185     }
186 
187     for (int i = 0; i < count; i++) {
188       assertTrue(htds.get(TableName.valueOf(name + i)) !=  null);
189     }
190     for (int i = 0; i < count; i++) {
191       assertTrue(htds.get(TableName.valueOf(name + i)) !=  null);
192     }
193     // Update the table infos
194     for (int i = 0; i < count; i++) {
195       HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name + i));
196       htd.addFamily(new HColumnDescriptor("" + i));
197       htds.updateTableDescriptor(htd);
198     }
199     // Wait a while so mod time we write is for sure different.
200     Thread.sleep(100);
201     for (int i = 0; i < count; i++) {
202       assertTrue(htds.get(TableName.valueOf(name + i)) !=  null);
203     }
204     for (int i = 0; i < count; i++) {
205       assertTrue(htds.get(TableName.valueOf(name + i)) !=  null);
206     }
207     assertEquals(count * 4, htds.invocations);
208     assertTrue("expected=" + (count * 2) + ", actual=" + htds.cachehits,
209       htds.cachehits >= (count * 2));
210   }
211 
212   @Test
213   public void testNoSuchTable() throws IOException {
214     final String name = "testNoSuchTable";
215     FileSystem fs = FileSystem.get(UTIL.getConfiguration());
216     // Cleanup old tests if any detrius laying around.
217     Path rootdir = new Path(UTIL.getDataTestDir(), name);
218     TableDescriptors htds = new FSTableDescriptors(fs, rootdir);
219     assertNull("There shouldn't be any HTD for this table",
220       htds.get(TableName.valueOf("NoSuchTable")));
221   }
222 
223   @Test
224   public void testUpdates() throws IOException {
225     final String name = "testUpdates";
226     FileSystem fs = FileSystem.get(UTIL.getConfiguration());
227     // Cleanup old tests if any detrius laying around.
228     Path rootdir = new Path(UTIL.getDataTestDir(), name);
229     TableDescriptors htds = new FSTableDescriptors(fs, rootdir);
230     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
231     htds.add(htd);
232     htds.add(htd);
233     htds.add(htd);
234   }
235 
236   @Test
237   public void testTableInfoFileStatusComparator() {
238     FileStatus bare =
239       new FileStatus(0, false, 0, 0, -1,
240         new Path("/tmp", FSTableDescriptors.TABLEINFO_FILE_PREFIX));
241     FileStatus future =
242       new FileStatus(0, false, 0, 0, -1,
243         new Path("/tmp/tablinfo." + System.currentTimeMillis()));
244     FileStatus farFuture =
245       new FileStatus(0, false, 0, 0, -1,
246         new Path("/tmp/tablinfo." + System.currentTimeMillis() + 1000));
247     FileStatus [] alist = {bare, future, farFuture};
248     FileStatus [] blist = {bare, farFuture, future};
249     FileStatus [] clist = {farFuture, bare, future};
250     Comparator<FileStatus> c = FSTableDescriptors.TABLEINFO_FILESTATUS_COMPARATOR;
251     Arrays.sort(alist, c);
252     Arrays.sort(blist, c);
253     Arrays.sort(clist, c);
254     // Now assert all sorted same in way we want.
255     for (int i = 0; i < alist.length; i++) {
256       assertTrue(alist[i].equals(blist[i]));
257       assertTrue(blist[i].equals(clist[i]));
258       assertTrue(clist[i].equals(i == 0? farFuture: i == 1? future: bare));
259     }
260   }
261 
262   @Test
263   public void testReadingArchiveDirectoryFromFS() throws IOException {
264     FileSystem fs = FileSystem.get(UTIL.getConfiguration());
265     try {
266       new FSTableDescriptors(fs, FSUtils.getRootDir(UTIL.getConfiguration()))
267           .get(TableName.valueOf(HConstants.HFILE_ARCHIVE_DIRECTORY));
268       fail("Shouldn't be able to read a table descriptor for the archive directory.");
269     } catch (Exception e) {
270       LOG.debug("Correctly got error when reading a table descriptor from the archive directory: "
271           + e.getMessage());
272     }
273   }
274 
275   @Test
276   public void testCreateTableDescriptorUpdatesIfExistsAlready() throws IOException {
277     Path testdir = UTIL.getDataTestDir("testCreateTableDescriptorUpdatesIfThereExistsAlready");
278     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(
279         "testCreateTableDescriptorUpdatesIfThereExistsAlready"));
280     FileSystem fs = FileSystem.get(UTIL.getConfiguration());
281     FSTableDescriptors fstd = new FSTableDescriptors(fs, testdir);
282     assertTrue(fstd.createTableDescriptor(htd));
283     assertFalse(fstd.createTableDescriptor(htd));
284     htd.setValue(Bytes.toBytes("mykey"), Bytes.toBytes("myValue"));
285     assertTrue(fstd.createTableDescriptor(htd)); //this will re-create
286     Path tableDir = fstd.getTableDir(htd.getTableName());
287     Path tmpTableDir = new Path(tableDir, FSTableDescriptors.TMP_DIR);
288     FileStatus[] statuses = fs.listStatus(tmpTableDir);
289     assertTrue(statuses.length == 0);
290 
291     assertEquals(htd, FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir));
292   }
293 
294 }
295