View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase;
20  
21  import java.io.IOException;
22  import java.util.Iterator;
23  import java.util.List;
24  import java.util.NavigableMap;
25  
26  import junit.framework.AssertionFailedError;
27  import junit.framework.TestCase;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.fs.FileSystem;
33  import org.apache.hadoop.fs.Path;
34  import org.apache.hadoop.hbase.client.Delete;
35  import org.apache.hadoop.hbase.client.Durability;
36  import org.apache.hadoop.hbase.client.Get;
37  import org.apache.hadoop.hbase.client.HTable;
38  import org.apache.hadoop.hbase.client.Put;
39  import org.apache.hadoop.hbase.client.Result;
40  import org.apache.hadoop.hbase.client.ResultScanner;
41  import org.apache.hadoop.hbase.client.Scan;
42  import org.apache.hadoop.hbase.regionserver.HRegion;
43  import org.apache.hadoop.hbase.regionserver.InternalScanner;
44  import org.apache.hadoop.hbase.util.Bytes;
45  import org.apache.hadoop.hbase.util.FSUtils;
46  import org.apache.hadoop.hdfs.MiniDFSCluster;
47  
48  /**
49   * Abstract HBase test class.  Initializes a few things that can come in handly
50   * like an HBaseConfiguration and filesystem.
51   * @deprecated Write junit4 unit tests using {@link HBaseTestingUtility}
52   */
53  public abstract class HBaseTestCase extends TestCase {
54    private static final Log LOG = LogFactory.getLog(HBaseTestCase.class);
55  
56    protected final static byte [] fam1 = Bytes.toBytes("colfamily11");
57    protected final static byte [] fam2 = Bytes.toBytes("colfamily21");
58    protected final static byte [] fam3 = Bytes.toBytes("colfamily31");
59  
60    protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
61  
62    private boolean localfs = false;
63    protected static Path testDir = null;
64    protected FileSystem fs = null;
65    protected HRegion meta = null;
66    protected static final char FIRST_CHAR = 'a';
67    protected static final char LAST_CHAR = 'z';
68    protected static final String PUNCTUATION = "~`@#$%^&*()-_+=:;',.<>/?[]{}|";
69    protected static final byte [] START_KEY_BYTES = {FIRST_CHAR, FIRST_CHAR, FIRST_CHAR};
70    protected String START_KEY = new String(START_KEY_BYTES, HConstants.UTF8_CHARSET);
71    protected static final int MAXVERSIONS = 3;
72  
73    protected final HBaseTestingUtility testUtil = new HBaseTestingUtility();
74  
75    public volatile Configuration conf = HBaseConfiguration.create();
76  
77    /** constructor */
78    public HBaseTestCase() {
79      super();
80    }
81  
82    /**
83     * @param name
84     */
85    public HBaseTestCase(String name) {
86      super(name);
87    }
88  
89    /**
90     * Note that this method must be called after the mini hdfs cluster has
91     * started or we end up with a local file system.
92     */
93    @Override
94    protected void setUp() throws Exception {
95      super.setUp();
96      localfs =
97        (conf.get("fs.defaultFS", "file:///").compareTo("file:///") == 0);
98  
99      if (fs == null) {
100       this.fs = FileSystem.get(conf);
101     }
102     try {
103       if (localfs) {
104         this.testDir = getUnitTestdir(getName());
105         if (fs.exists(testDir)) {
106           fs.delete(testDir, true);
107         }
108       } else {
109         this.testDir = FSUtils.getRootDir(conf);
110       }
111     } catch (Exception e) {
112       LOG.fatal("error during setup", e);
113       throw e;
114     }
115   }
116 
117   @Override
118   protected void tearDown() throws Exception {
119     try {
120       if (localfs) {
121         if (this.fs.exists(testDir)) {
122           this.fs.delete(testDir, true);
123         }
124       }
125     } catch (Exception e) {
126       LOG.fatal("error during tear down", e);
127     }
128     super.tearDown();
129   }
130 
131   /**
132    * @see HBaseTestingUtility#getBaseTestDir
133    * @param testName
134    * @return directory to use for this test
135    */
136     protected Path getUnitTestdir(String testName) {
137       return testUtil.getDataTestDir(testName);
138     }
139 
140   /**
141    * You must call close on the returned region and then close on the log file
142    * it created. Do {@link HRegion#close()} followed by {@link HRegion#getLog()}
143    * and on it call close.
144    * @param desc
145    * @param startKey
146    * @param endKey
147    * @return An {@link HRegion}
148    * @throws IOException
149    */
150   public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
151       byte [] endKey)
152   throws IOException {
153     return createNewHRegion(desc, startKey, endKey, this.conf);
154   }
155 
156   public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
157       byte [] endKey, Configuration conf)
158   throws IOException {
159     HRegionInfo hri = new HRegionInfo(desc.getTableName(), startKey, endKey);
160     return HRegion.createHRegion(hri, testDir, conf, desc);
161   }
162 
163   protected HRegion openClosedRegion(final HRegion closedRegion)
164   throws IOException {
165     return HRegion.openHRegion(closedRegion, null);
166   }
167 
168   /**
169    * Create a table of name <code>name</code> with {@link COLUMNS} for
170    * families.
171    * @param name Name to give table.
172    * @return Column descriptor.
173    */
174   protected HTableDescriptor createTableDescriptor(final String name) {
175     return createTableDescriptor(name, MAXVERSIONS);
176   }
177 
178   /**
179    * Create a table of name <code>name</code> with {@link COLUMNS} for
180    * families.
181    * @param name Name to give table.
182    * @param versions How many versions to allow per column.
183    * @return Column descriptor.
184    */
185   protected HTableDescriptor createTableDescriptor(final String name,
186       final int versions) {
187     return createTableDescriptor(name, HColumnDescriptor.DEFAULT_MIN_VERSIONS,
188         versions, HConstants.FOREVER, HColumnDescriptor.DEFAULT_KEEP_DELETED);
189   }
190 
191   /**
192    * Create a table of name <code>name</code> with {@link COLUMNS} for
193    * families.
194    * @param name Name to give table.
195    * @param versions How many versions to allow per column.
196    * @return Column descriptor.
197    */
198   protected HTableDescriptor createTableDescriptor(final String name,
199       final int minVersions, final int versions, final int ttl, boolean keepDeleted) {
200     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
201     for (byte[] cfName : new byte[][]{ fam1, fam2, fam3 }) {
202       htd.addFamily(new HColumnDescriptor(cfName)
203           .setMinVersions(minVersions)
204           .setMaxVersions(versions)
205           .setKeepDeletedCells(keepDeleted)
206           .setBlockCacheEnabled(false)
207           .setTimeToLive(ttl)
208       );
209     }
210     return htd;
211   }
212 
213   /**
214    * Add content to region <code>r</code> on the passed column
215    * <code>column</code>.
216    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
217    * @param r
218    * @param columnFamily
219    * @param column
220    * @throws IOException
221    * @return count of what we added.
222    */
223   public static long addContent(final HRegion r, final byte [] columnFamily, final byte[] column)
224   throws IOException {
225     byte [] startKey = r.getRegionInfo().getStartKey();
226     byte [] endKey = r.getRegionInfo().getEndKey();
227     byte [] startKeyBytes = startKey;
228     if (startKeyBytes == null || startKeyBytes.length == 0) {
229       startKeyBytes = START_KEY_BYTES;
230     }
231     return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily), Bytes.toString(column),
232       startKeyBytes, endKey, -1);
233   }
234 
235   public static long addContent(final HRegion r, final byte [] columnFamily)
236   throws IOException {
237     return addContent(r, columnFamily, null);
238   }
239 
240   /**
241    * Add content to region <code>r</code> on the passed column
242    * <code>column</code>.
243    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
244    * @param updater  An instance of {@link Incommon}.
245    * @param columnFamily
246    * @param writeToWAL
247    * @throws IOException
248    * @return count of what we added.
249    */
250   public static long addContent(final Incommon updater,
251       final String columnFamily) throws IOException {
252     return addContent(updater, columnFamily, START_KEY_BYTES, null);
253   }
254 
255   public static long addContent(final Incommon updater, final String family,
256       final String column) throws IOException {
257     return addContent(updater, family, column, START_KEY_BYTES, null);
258   }
259 
260   /**
261    * Add content to region <code>r</code> on the passed column
262    * <code>column</code>.
263    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
264    * @param updater  An instance of {@link Incommon}.
265    * @param columnFamily
266    * @param startKeyBytes Where to start the rows inserted
267    * @param endKey Where to stop inserting rows.
268    * @param writeToWAL
269    * @return count of what we added.
270    * @throws IOException
271    */
272   public static long addContent(final Incommon updater, final String columnFamily,
273       final byte [] startKeyBytes, final byte [] endKey)
274   throws IOException {
275     return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1);
276   }
277 
278   public static long addContent(final Incommon updater, final String family, String column,
279       final byte [] startKeyBytes, final byte [] endKey) throws IOException {
280     return addContent(updater, family, column, startKeyBytes, endKey, -1);
281   }
282 
283   /**
284    * Add content to region <code>r</code> on the passed column
285    * <code>column</code>.
286    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
287    * @param updater  An instance of {@link Incommon}.
288    * @param column
289    * @param startKeyBytes Where to start the rows inserted
290    * @param endKey Where to stop inserting rows.
291    * @param ts Timestamp to write the content with.
292    * @param writeToWAL
293    * @return count of what we added.
294    * @throws IOException
295    */
296   public static long addContent(final Incommon updater,
297                                    final String columnFamily, 
298                                    final String column,
299       final byte [] startKeyBytes, final byte [] endKey, final long ts)
300   throws IOException {
301     long count = 0;
302     // Add rows of three characters.  The first character starts with the
303     // 'a' character and runs up to 'z'.  Per first character, we run the
304     // second character over same range.  And same for the third so rows
305     // (and values) look like this: 'aaa', 'aab', 'aac', etc.
306     char secondCharStart = (char)startKeyBytes[1];
307     char thirdCharStart = (char)startKeyBytes[2];
308     EXIT: for (char c = (char)startKeyBytes[0]; c <= LAST_CHAR; c++) {
309       for (char d = secondCharStart; d <= LAST_CHAR; d++) {
310         for (char e = thirdCharStart; e <= LAST_CHAR; e++) {
311           byte [] t = new byte [] {(byte)c, (byte)d, (byte)e};
312           if (endKey != null && endKey.length > 0
313               && Bytes.compareTo(endKey, t) <= 0) {
314             break EXIT;
315           }
316           try {
317             Put put;
318             if(ts != -1) {
319               put = new Put(t, ts);
320             } else {
321               put = new Put(t);
322             }
323             try {
324               StringBuilder sb = new StringBuilder();
325               if (column != null && column.contains(":")) {
326                 sb.append(column);
327               } else {
328                 if (columnFamily != null) {
329                   sb.append(columnFamily);
330                   if (!columnFamily.endsWith(":")) {
331                     sb.append(":");
332                   }
333                   if (column != null) {
334                     sb.append(column);
335                   }
336                 }
337               }
338               byte[][] split =
339                 KeyValue.parseColumn(Bytes.toBytes(sb.toString()));
340               if(split.length == 1) {
341                 put.add(split[0], new byte[0], t);
342               } else {
343                 put.add(split[0], split[1], t);
344               }
345               put.setDurability(Durability.SKIP_WAL);
346               updater.put(put);
347               count++;
348             } catch (RuntimeException ex) {
349               ex.printStackTrace();
350               throw ex;
351             } catch (IOException ex) {
352               ex.printStackTrace();
353               throw ex;
354             }
355           } catch (RuntimeException ex) {
356             ex.printStackTrace();
357             throw ex;
358           } catch (IOException ex) {
359             ex.printStackTrace();
360             throw ex;
361           }
362         }
363         // Set start character back to FIRST_CHAR after we've done first loop.
364         thirdCharStart = FIRST_CHAR;
365       }
366       secondCharStart = FIRST_CHAR;
367     }
368     return count;
369   }
370 
371   /**
372    * Implementors can flushcache.
373    */
374   public interface FlushCache {
375     /**
376      * @throws IOException
377      */
378     void flushcache() throws IOException;
379   }
380 
381   /**
382    * Interface used by tests so can do common operations against an HTable
383    * or an HRegion.
384    *
385    * TOOD: Come up w/ a better name for this interface.
386    */
387   public interface Incommon {
388     /**
389      *
390      * @param delete
391      * @param writeToWAL
392      * @throws IOException
393      */
394     void delete(Delete delete, boolean writeToWAL)
395     throws IOException;
396 
397     /**
398      * @param put
399      * @throws IOException
400      */
401     void put(Put put) throws IOException;
402 
403     Result get(Get get) throws IOException;
404 
405     /**
406      * @param family
407      * @param qualifiers
408      * @param firstRow
409      * @param ts
410      * @return scanner for specified columns, first row and timestamp
411      * @throws IOException
412      */
413     ScannerIncommon getScanner(
414       byte[] family, byte[][] qualifiers, byte[] firstRow, long ts
415     )
416     throws IOException;
417   }
418 
419   /**
420    * A class that makes a {@link Incommon} out of a {@link HRegion}
421    */
422   public static class HRegionIncommon implements Incommon, FlushCache {
423     final HRegion region;
424 
425     /**
426      * @param HRegion
427      */
428     public HRegionIncommon(final HRegion HRegion) {
429       this.region = HRegion;
430     }
431 
432     public void put(Put put) throws IOException {
433       region.put(put);
434     }
435 
436     public void delete(Delete delete,  boolean writeToWAL)
437     throws IOException {
438       this.region.delete(delete);
439     }
440 
441     public Result get(Get get) throws IOException {
442       return region.get(get);
443     }
444 
445     public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
446         byte [] firstRow, long ts)
447       throws IOException {
448         Scan scan = new Scan(firstRow);
449         if(qualifiers == null || qualifiers.length == 0) {
450           scan.addFamily(family);
451         } else {
452           for(int i=0; i<qualifiers.length; i++){
453             scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
454           }
455         }
456         scan.setTimeRange(0, ts);
457         return new
458           InternalScannerIncommon(region.getScanner(scan));
459       }
460 
461     public void flushcache() throws IOException {
462       this.region.flushcache();
463     }
464   }
465 
466   /**
467    * A class that makes a {@link Incommon} out of a {@link HTable}
468    */
469   public static class HTableIncommon implements Incommon {
470     final HTable table;
471 
472     /**
473      * @param table
474      */
475     public HTableIncommon(final HTable table) {
476       super();
477       this.table = table;
478     }
479 
480     public void put(Put put) throws IOException {
481       table.put(put);
482     }
483 
484 
485     public void delete(Delete delete, boolean writeToWAL)
486     throws IOException {
487       this.table.delete(delete);
488     }
489 
490     public Result get(Get get) throws IOException {
491       return table.get(get);
492     }
493 
494     public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
495         byte [] firstRow, long ts)
496       throws IOException {
497       Scan scan = new Scan(firstRow);
498       if(qualifiers == null || qualifiers.length == 0) {
499         scan.addFamily(family);
500       } else {
501         for(int i=0; i<qualifiers.length; i++){
502           scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
503         }
504       }
505       scan.setTimeRange(0, ts);
506       return new
507         ClientScannerIncommon(table.getScanner(scan));
508     }
509   }
510 
511   public interface ScannerIncommon
512   extends Iterable<Result> {
513     boolean next(List<Cell> values)
514     throws IOException;
515 
516     void close() throws IOException;
517   }
518 
519   public static class ClientScannerIncommon implements ScannerIncommon {
520     ResultScanner scanner;
521     public ClientScannerIncommon(ResultScanner scanner) {
522       this.scanner = scanner;
523     }
524 
525     @Override
526     public boolean next(List<Cell> values)
527     throws IOException {
528       Result results = scanner.next();
529       if (results == null) {
530         return false;
531       }
532       values.clear();
533       values.addAll(results.listCells());
534       return true;
535     }
536 
537     public void close() throws IOException {
538       scanner.close();
539     }
540 
541     public Iterator<Result> iterator() {
542       return scanner.iterator();
543     }
544   }
545 
546   public static class InternalScannerIncommon implements ScannerIncommon {
547     InternalScanner scanner;
548 
549     public InternalScannerIncommon(InternalScanner scanner) {
550       this.scanner = scanner;
551     }
552 
553     @Override
554     public boolean next(List<Cell> results)
555     throws IOException {
556       return scanner.next(results);
557     }
558 
559     @Override
560     public void close() throws IOException {
561       scanner.close();
562     }
563 
564     @Override
565     public Iterator<Result> iterator() {
566       throw new UnsupportedOperationException();
567     }
568   }
569 
570   protected void assertResultEquals(final HRegion region, final byte [] row,
571       final byte [] family, final byte [] qualifier, final long timestamp,
572       final byte [] value)
573     throws IOException {
574       Get get = new Get(row);
575       get.setTimeStamp(timestamp);
576       Result res = region.get(get);
577       NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map =
578         res.getMap();
579       byte [] res_value = map.get(family).get(qualifier).get(timestamp);
580 
581       if (value == null) {
582         assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
583             " at timestamp " + timestamp, null, res_value);
584       } else {
585         if (res_value == null) {
586           fail(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
587               " at timestamp " + timestamp + "\" was expected to be \"" +
588               Bytes.toStringBinary(value) + " but was null");
589         }
590         if (res_value != null) {
591           assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
592               " at timestamp " +
593               timestamp, value, new String(res_value));
594         }
595       }
596     }
597 
598   /**
599    * Common method to close down a MiniDFSCluster and the associated file system
600    *
601    * @param cluster
602    */
603   public static void shutdownDfs(MiniDFSCluster cluster) {
604     if (cluster != null) {
605       LOG.info("Shutting down Mini DFS ");
606       try {
607         cluster.shutdown();
608       } catch (Exception e) {
609         /// Can get a java.lang.reflect.UndeclaredThrowableException thrown
610         // here because of an InterruptedException. Don't let exceptions in
611         // here be cause of test failure.
612       }
613       try {
614         FileSystem fs = cluster.getFileSystem();
615         if (fs != null) {
616           LOG.info("Shutting down FileSystem");
617           fs.close();
618         }
619         FileSystem.closeAll();
620       } catch (IOException e) {
621         LOG.error("error closing file system", e);
622       }
623     }
624   }
625 
626   /**
627    * You must call {@link #closeRootAndMeta()} when done after calling this
628    * method. It does cleanup.
629    * @throws IOException
630    */
631   protected void createMetaRegion() throws IOException {
632     meta = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, testDir,
633         conf, HTableDescriptor.META_TABLEDESC);
634   }
635 
636   protected void closeRootAndMeta() throws IOException {
637     HRegion.closeHRegion(meta);
638   }
639 
640   public static void assertByteEquals(byte[] expected,
641                                byte[] actual) {
642     if (Bytes.compareTo(expected, actual) != 0) {
643       throw new AssertionFailedError("expected:<" +
644       Bytes.toString(expected) + "> but was:<" +
645       Bytes.toString(actual) + ">");
646     }
647   }
648 
649   public static void assertEquals(byte[] expected,
650                                byte[] actual) {
651     if (Bytes.compareTo(expected, actual) != 0) {
652       throw new AssertionFailedError("expected:<" +
653       Bytes.toStringBinary(expected) + "> but was:<" +
654       Bytes.toStringBinary(actual) + ">");
655     }
656   }
657 
658 }