1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.mapreduce;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertTrue;
23 import static org.junit.Assert.fail;
24
25 import java.io.ByteArrayOutputStream;
26 import java.io.IOException;
27 import java.io.PrintStream;
28 import java.util.ArrayList;
29
30 import org.apache.commons.logging.Log;
31 import org.apache.commons.logging.LogFactory;
32 import org.apache.hadoop.conf.Configuration;
33 import org.apache.hadoop.hbase.HBaseTestingUtility;
34 import org.apache.hadoop.hbase.MediumTests;
35 import org.apache.hadoop.hbase.client.HTable;
36 import org.apache.hadoop.hbase.client.Put;
37 import org.apache.hadoop.hbase.mapreduce.RowCounter.RowCounterMapper;
38 import org.apache.hadoop.hbase.util.Bytes;
39 import org.apache.hadoop.hbase.util.LauncherSecurityManager;
40 import org.apache.hadoop.mapreduce.Counter;
41 import org.apache.hadoop.mapreduce.Job;
42 import org.apache.hadoop.util.GenericOptionsParser;
43 import org.junit.AfterClass;
44 import org.junit.BeforeClass;
45 import org.junit.Test;
46 import org.junit.experimental.categories.Category;
47
48
49
50
51 @Category(MediumTests.class)
52 public class TestRowCounter {
53 final Log LOG = LogFactory.getLog(getClass());
54 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
55 private final static String TABLE_NAME = "testRowCounter";
56 private final static String COL_FAM = "col_fam";
57 private final static String COL1 = "c1";
58 private final static String COL2 = "c2";
59 private final static String COMPOSITE_COLUMN = "C:A:A";
60 private final static int TOTAL_ROWS = 10;
61 private final static int ROWS_WITH_ONE_COL = 2;
62
63
64
65
66 @BeforeClass
67 public static void setUpBeforeClass() throws Exception {
68 TEST_UTIL.startMiniCluster();
69 TEST_UTIL.startMiniMapReduceCluster();
70 HTable table = TEST_UTIL.createTable(Bytes.toBytes(TABLE_NAME),
71 Bytes.toBytes(COL_FAM));
72 writeRows(table);
73 table.close();
74 }
75
76
77
78
79 @AfterClass
80 public static void tearDownAfterClass() throws Exception {
81 TEST_UTIL.shutdownMiniCluster();
82 TEST_UTIL.shutdownMiniMapReduceCluster();
83 }
84
85
86
87
88
89
90 @Test
91 public void testRowCounterNoColumn() throws Exception {
92 String[] args = new String[] {
93 TABLE_NAME
94 };
95 runRowCount(args, 10);
96 }
97
98
99
100
101
102
103
104 @Test
105 public void testRowCounterExclusiveColumn() throws Exception {
106 String[] args = new String[] {
107 TABLE_NAME, COL_FAM + ":" + COL1
108 };
109 runRowCount(args, 8);
110 }
111
112
113
114
115
116
117
118 @Test
119 public void testRowCounterColumnWithColonInQualifier() throws Exception {
120 String[] args = new String[] {
121 TABLE_NAME, COL_FAM + ":" + COMPOSITE_COLUMN
122 };
123 runRowCount(args, 8);
124 }
125
126
127
128
129
130
131
132 @Test
133 public void testRowCounterHiddenColumn() throws Exception {
134 String[] args = new String[] {
135 TABLE_NAME, COL_FAM + ":" + COL2
136 };
137 runRowCount(args, 10);
138 }
139
140
141
142
143
144
145
146
147 private void runRowCount(String[] args, int expectedCount) throws Exception {
148 GenericOptionsParser opts = new GenericOptionsParser(
149 TEST_UTIL.getConfiguration(), args);
150 Configuration conf = opts.getConfiguration();
151 args = opts.getRemainingArgs();
152 Job job = RowCounter.createSubmittableJob(conf, args);
153 job.waitForCompletion(true);
154 assertTrue(job.isSuccessful());
155 Counter counter = job.getCounters().findCounter(
156 RowCounterMapper.Counters.ROWS);
157 assertEquals(expectedCount, counter.getValue());
158 }
159
160
161
162
163
164
165
166
167 private static void writeRows(HTable table) throws IOException {
168 final byte[] family = Bytes.toBytes(COL_FAM);
169 final byte[] value = Bytes.toBytes("abcd");
170 final byte[] col1 = Bytes.toBytes(COL1);
171 final byte[] col2 = Bytes.toBytes(COL2);
172 final byte[] col3 = Bytes.toBytes(COMPOSITE_COLUMN);
173 ArrayList<Put> rowsUpdate = new ArrayList<Put>();
174
175 int i = 0;
176 for (; i < TOTAL_ROWS - ROWS_WITH_ONE_COL; i++) {
177 byte[] row = Bytes.toBytes("row" + i);
178 Put put = new Put(row);
179 put.add(family, col1, value);
180 put.add(family, col2, value);
181 put.add(family, col3, value);
182 rowsUpdate.add(put);
183 }
184
185
186 for (; i < TOTAL_ROWS; i++) {
187 byte[] row = Bytes.toBytes("row" + i);
188 Put put = new Put(row);
189 put.add(family, col2, value);
190 rowsUpdate.add(put);
191 }
192 table.put(rowsUpdate);
193 }
194
195
196
197
198 @Test
199 public void testImportMain() throws Exception {
200 PrintStream oldPrintStream = System.err;
201 SecurityManager SECURITY_MANAGER = System.getSecurityManager();
202 LauncherSecurityManager newSecurityManager= new LauncherSecurityManager();
203 System.setSecurityManager(newSecurityManager);
204 ByteArrayOutputStream data = new ByteArrayOutputStream();
205 String[] args = {};
206 System.setErr(new PrintStream(data));
207 try {
208 System.setErr(new PrintStream(data));
209
210 try {
211 RowCounter.main(args);
212 fail("should be SecurityException");
213 } catch (SecurityException e) {
214 assertEquals(-1, newSecurityManager.getExitCode());
215 assertTrue(data.toString().contains("Wrong number of parameters:"));
216 assertTrue(data.toString().contains(
217 "Usage: RowCounter [options] <tablename> [--range=[startKey],[endKey]] " +
218 "[<column1> <column2>...]"));
219 assertTrue(data.toString().contains("-Dhbase.client.scanner.caching=100"));
220 assertTrue(data.toString().contains("-Dmapred.map.tasks.speculative.execution=false"));
221 }
222 data.reset();
223 try {
224 args = new String[2];
225 args[0] = "table";
226 args[1] = "--range=1";
227 RowCounter.main(args);
228 fail("should be SecurityException");
229 } catch (SecurityException e) {
230 assertEquals(-1, newSecurityManager.getExitCode());
231 assertTrue(data.toString().contains(
232 "Please specify range in such format as \"--range=a,b\" or, with only one boundary," +
233 " \"--range=,b\" or \"--range=a,\""));
234 assertTrue(data.toString().contains(
235 "Usage: RowCounter [options] <tablename> [--range=[startKey],[endKey]] " +
236 "[<column1> <column2>...]"));
237 }
238
239 } finally {
240 System.setErr(oldPrintStream);
241 System.setSecurityManager(SECURITY_MANAGER);
242 }
243
244 }
245
246 }