View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.mapreduce;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertTrue;
23  import static org.junit.Assert.fail;
24  
25  import java.io.ByteArrayOutputStream;
26  import java.io.IOException;
27  import java.io.PrintStream;
28  import java.util.ArrayList;
29  
30  import org.apache.commons.logging.Log;
31  import org.apache.commons.logging.LogFactory;
32  import org.apache.hadoop.conf.Configuration;
33  import org.apache.hadoop.hbase.HBaseTestingUtility;
34  import org.apache.hadoop.hbase.MediumTests;
35  import org.apache.hadoop.hbase.client.HTable;
36  import org.apache.hadoop.hbase.client.Put;
37  import org.apache.hadoop.hbase.mapreduce.RowCounter.RowCounterMapper;
38  import org.apache.hadoop.hbase.util.Bytes;
39  import org.apache.hadoop.hbase.util.LauncherSecurityManager;
40  import org.apache.hadoop.mapreduce.Counter;
41  import org.apache.hadoop.mapreduce.Job;
42  import org.apache.hadoop.util.GenericOptionsParser;
43  import org.junit.AfterClass;
44  import org.junit.BeforeClass;
45  import org.junit.Test;
46  import org.junit.experimental.categories.Category;
47  
48  /**
49   * Test the rowcounter map reduce job.
50   */
51  @Category(MediumTests.class)
52  public class TestRowCounter {
53    final Log LOG = LogFactory.getLog(getClass());
54    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
55    private final static String TABLE_NAME = "testRowCounter";
56    private final static String COL_FAM = "col_fam";
57    private final static String COL1 = "c1";
58    private final static String COL2 = "c2";
59    private final static String COMPOSITE_COLUMN = "C:A:A";
60    private final static int TOTAL_ROWS = 10;
61    private final static int ROWS_WITH_ONE_COL = 2;
62  
63    /**
64     * @throws java.lang.Exception
65     */
66    @BeforeClass
67    public static void setUpBeforeClass() throws Exception {
68      TEST_UTIL.startMiniCluster();
69      TEST_UTIL.startMiniMapReduceCluster();
70      HTable table = TEST_UTIL.createTable(Bytes.toBytes(TABLE_NAME),
71          Bytes.toBytes(COL_FAM));
72      writeRows(table);
73      table.close();
74    }
75  
76    /**
77     * @throws java.lang.Exception
78     */
79    @AfterClass
80    public static void tearDownAfterClass() throws Exception {
81      TEST_UTIL.shutdownMiniCluster();
82      TEST_UTIL.shutdownMiniMapReduceCluster();
83    }
84  
85    /**
86     * Test a case when no column was specified in command line arguments.
87     * 
88     * @throws Exception
89     */
90    @Test
91    public void testRowCounterNoColumn() throws Exception {
92      String[] args = new String[] {
93          TABLE_NAME
94      };
95      runRowCount(args, 10);
96    }
97  
98    /**
99     * Test a case when the column specified in command line arguments is
100    * exclusive for few rows.
101    * 
102    * @throws Exception
103    */
104   @Test
105   public void testRowCounterExclusiveColumn() throws Exception {
106     String[] args = new String[] {
107         TABLE_NAME, COL_FAM + ":" + COL1
108     };
109     runRowCount(args, 8);
110   }
111 
112   /**
113    * Test a case when the column specified in command line arguments is
114    * one for which the qualifier contains colons.
115    *
116    * @throws Exception
117    */
118   @Test
119   public void testRowCounterColumnWithColonInQualifier() throws Exception {
120     String[] args = new String[] {
121         TABLE_NAME, COL_FAM + ":" + COMPOSITE_COLUMN
122     };
123     runRowCount(args, 8);
124   }
125 
126   /**
127    * Test a case when the column specified in command line arguments is not part
128    * of first KV for a row.
129    * 
130    * @throws Exception
131    */
132   @Test
133   public void testRowCounterHiddenColumn() throws Exception {
134     String[] args = new String[] {
135         TABLE_NAME, COL_FAM + ":" + COL2
136     };
137     runRowCount(args, 10);
138   }
139 
140   /**
141    * Run the RowCounter map reduce job and verify the row count.
142    * 
143    * @param args the command line arguments to be used for rowcounter job.
144    * @param expectedCount the expected row count (result of map reduce job).
145    * @throws Exception
146    */
147   private void runRowCount(String[] args, int expectedCount) throws Exception {
148     GenericOptionsParser opts = new GenericOptionsParser(
149         TEST_UTIL.getConfiguration(), args);
150     Configuration conf = opts.getConfiguration();
151     args = opts.getRemainingArgs();
152     Job job = RowCounter.createSubmittableJob(conf, args);
153     job.waitForCompletion(true);
154     assertTrue(job.isSuccessful());
155     Counter counter = job.getCounters().findCounter(
156         RowCounterMapper.Counters.ROWS);
157     assertEquals(expectedCount, counter.getValue());
158   }
159 
160   /**
161    * Writes TOTAL_ROWS number of distinct rows in to the table. Few rows have
162    * two columns, Few have one.
163    * 
164    * @param table
165    * @throws IOException
166    */
167   private static void writeRows(HTable table) throws IOException {
168     final byte[] family = Bytes.toBytes(COL_FAM);
169     final byte[] value = Bytes.toBytes("abcd");
170     final byte[] col1 = Bytes.toBytes(COL1);
171     final byte[] col2 = Bytes.toBytes(COL2);
172     final byte[] col3 = Bytes.toBytes(COMPOSITE_COLUMN);
173     ArrayList<Put> rowsUpdate = new ArrayList<Put>();
174     // write few rows with two columns
175     int i = 0;
176     for (; i < TOTAL_ROWS - ROWS_WITH_ONE_COL; i++) {
177       byte[] row = Bytes.toBytes("row" + i);
178       Put put = new Put(row);
179       put.add(family, col1, value);
180       put.add(family, col2, value);
181       put.add(family, col3, value);
182       rowsUpdate.add(put);
183     }
184 
185     // write few rows with only one column
186     for (; i < TOTAL_ROWS; i++) {
187       byte[] row = Bytes.toBytes("row" + i);
188       Put put = new Put(row);
189       put.add(family, col2, value);
190       rowsUpdate.add(put);
191     }
192     table.put(rowsUpdate);
193   }
194 
195   /**
196    * test main method. Import should print help and call System.exit
197    */
198   @Test
199   public void testImportMain() throws Exception {
200     PrintStream oldPrintStream = System.err;
201     SecurityManager SECURITY_MANAGER = System.getSecurityManager();
202     LauncherSecurityManager newSecurityManager= new LauncherSecurityManager();
203     System.setSecurityManager(newSecurityManager);
204     ByteArrayOutputStream data = new ByteArrayOutputStream();
205     String[] args = {};
206     System.setErr(new PrintStream(data));
207     try {
208       System.setErr(new PrintStream(data));
209 
210       try {
211         RowCounter.main(args);
212         fail("should be SecurityException");
213       } catch (SecurityException e) {
214         assertEquals(-1, newSecurityManager.getExitCode());
215         assertTrue(data.toString().contains("Wrong number of parameters:"));
216         assertTrue(data.toString().contains(
217             "Usage: RowCounter [options] <tablename> [--range=[startKey],[endKey]] " +
218             "[<column1> <column2>...]"));
219         assertTrue(data.toString().contains("-Dhbase.client.scanner.caching=100"));
220         assertTrue(data.toString().contains("-Dmapred.map.tasks.speculative.execution=false"));
221       }
222       data.reset();
223       try {
224         args = new String[2];
225         args[0] = "table";
226         args[1] = "--range=1";
227         RowCounter.main(args);
228         fail("should be SecurityException");
229       } catch (SecurityException e) {
230         assertEquals(-1, newSecurityManager.getExitCode());
231         assertTrue(data.toString().contains(
232             "Please specify range in such format as \"--range=a,b\" or, with only one boundary," +
233             " \"--range=,b\" or \"--range=a,\""));
234         assertTrue(data.toString().contains(
235             "Usage: RowCounter [options] <tablename> [--range=[startKey],[endKey]] " +
236             "[<column1> <column2>...]"));
237       }
238 
239     } finally {
240       System.setErr(oldPrintStream);
241       System.setSecurityManager(SECURITY_MANAGER);
242     }
243 
244   }
245 
246 }