View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.mapreduce;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertNotNull;
22  import static org.junit.Assert.assertNull;
23  import static org.junit.Assert.assertTrue;
24  import static org.junit.Assert.fail;
25  
26  import java.io.ByteArrayOutputStream;
27  import java.io.IOException;
28  import java.io.PrintStream;
29  
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.hbase.CellUtil;
32  import org.apache.hadoop.hbase.HBaseTestingUtility;
33  import org.apache.hadoop.hbase.testclassification.LargeTests;
34  import org.apache.hadoop.hbase.TableName;
35  import org.apache.hadoop.hbase.client.Get;
36  import org.apache.hadoop.hbase.client.Put;
37  import org.apache.hadoop.hbase.client.Result;
38  import org.apache.hadoop.hbase.client.Table;
39  import org.apache.hadoop.hbase.util.Bytes;
40  import org.apache.hadoop.hbase.util.LauncherSecurityManager;
41  import org.apache.hadoop.mapreduce.Job;
42  import org.apache.hadoop.util.GenericOptionsParser;
43  import org.junit.AfterClass;
44  import org.junit.BeforeClass;
45  import org.junit.Test;
46  import org.junit.experimental.categories.Category;
47  
48  /**
49   * Basic test for the CopyTable M/R tool
50   */
51  @Category(LargeTests.class)
52  public class TestCopyTable {
53    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
54    private static final byte[] ROW1 = Bytes.toBytes("row1");
55    private static final byte[] ROW2 = Bytes.toBytes("row2");
56    private static final String FAMILY_A_STRING = "a";
57    private static final String FAMILY_B_STRING = "b";
58    private static final byte[] FAMILY_A = Bytes.toBytes(FAMILY_A_STRING);
59    private static final byte[] FAMILY_B = Bytes.toBytes(FAMILY_B_STRING);
60    private static final byte[] QUALIFIER = Bytes.toBytes("q");
61  
62  
63    @BeforeClass
64    public static void beforeClass() throws Exception {
65      TEST_UTIL.setJobWithoutMRCluster();
66      TEST_UTIL.startMiniCluster(3);
67    }
68  
69    @AfterClass
70    public static void afterClass() throws Exception {
71      TEST_UTIL.shutdownMiniCluster();
72    }
73  
74    private void doCopyTableTest(boolean bulkload) throws Exception {
75      final TableName TABLENAME1 = TableName.valueOf("testCopyTable1");
76      final TableName TABLENAME2 = TableName.valueOf("testCopyTable2");
77      final byte[] FAMILY = Bytes.toBytes("family");
78      final byte[] COLUMN1 = Bytes.toBytes("c1");
79  
80      Table t1 = TEST_UTIL.createTable(TABLENAME1, FAMILY);
81      Table t2 = TEST_UTIL.createTable(TABLENAME2, FAMILY);
82  
83      // put rows into the first table
84      for (int i = 0; i < 10; i++) {
85        Put p = new Put(Bytes.toBytes("row" + i));
86        p.add(FAMILY, COLUMN1, COLUMN1);
87        t1.put(p);
88      }
89  
90      CopyTable copy = new CopyTable(TEST_UTIL.getConfiguration());
91  
92      int code;
93      if (bulkload) {
94        code = copy.run(new String[] { "--new.name=" + TABLENAME2.getNameAsString(),
95            "--bulkload", TABLENAME1.getNameAsString() });
96      } else {
97        code = copy.run(new String[] { "--new.name=" + TABLENAME2.getNameAsString(),
98            TABLENAME1.getNameAsString() });
99      }
100     assertEquals("copy job failed", 0, code);
101 
102     // verify the data was copied into table 2
103     for (int i = 0; i < 10; i++) {
104       Get g = new Get(Bytes.toBytes("row" + i));
105       Result r = t2.get(g);
106       assertEquals(1, r.size());
107       assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1));
108     }
109     
110     t1.close();
111     t2.close();
112     TEST_UTIL.deleteTable(TABLENAME1);
113     TEST_UTIL.deleteTable(TABLENAME2);
114   }
115 
116   /**
117    * Simple end-to-end test
118    * @throws Exception
119    */
120   @Test
121   public void testCopyTable() throws Exception {
122     doCopyTableTest(false);
123   }
124   
125   /**
126    * Simple end-to-end test with bulkload.
127    */
128   @Test
129   public void testCopyTableWithBulkload() throws Exception {
130     doCopyTableTest(true);
131   }
132   
133   @Test
134   public void testStartStopRow() throws Exception {
135     final TableName TABLENAME1 = TableName.valueOf("testStartStopRow1");
136     final TableName TABLENAME2 = TableName.valueOf("testStartStopRow2");
137     final byte[] FAMILY = Bytes.toBytes("family");
138     final byte[] COLUMN1 = Bytes.toBytes("c1");
139     final byte[] ROW0 = Bytes.toBytes("row0");
140     final byte[] ROW1 = Bytes.toBytes("row1");
141     final byte[] ROW2 = Bytes.toBytes("row2");
142 
143     Table t1 = TEST_UTIL.createTable(TABLENAME1, FAMILY);
144     Table t2 = TEST_UTIL.createTable(TABLENAME2, FAMILY);
145 
146     // put rows into the first table
147     Put p = new Put(ROW0);
148     p.add(FAMILY, COLUMN1, COLUMN1);
149     t1.put(p);
150     p = new Put(ROW1);
151     p.add(FAMILY, COLUMN1, COLUMN1);
152     t1.put(p);
153     p = new Put(ROW2);
154     p.add(FAMILY, COLUMN1, COLUMN1);
155     t1.put(p);
156 
157     CopyTable copy = new CopyTable(TEST_UTIL.getConfiguration());
158     assertEquals(
159       0,
160       copy.run(new String[] { "--new.name=" + TABLENAME2, "--startrow=row1",
161           "--stoprow=row2", TABLENAME1.getNameAsString() }));
162 
163     // verify the data was copied into table 2
164     // row1 exist, row0, row2 do not exist
165     Get g = new Get(ROW1);
166     Result r = t2.get(g);
167     assertEquals(1, r.size());
168     assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1));
169 
170     g = new Get(ROW0);
171     r = t2.get(g);
172     assertEquals(0, r.size());
173     
174     g = new Get(ROW2);
175     r = t2.get(g);
176     assertEquals(0, r.size());
177     
178     t1.close();
179     t2.close();
180     TEST_UTIL.deleteTable(TABLENAME1);
181     TEST_UTIL.deleteTable(TABLENAME2);
182   }
183 
184   /**
185    * Test copy of table from sourceTable to targetTable all rows from family a
186    */
187   @Test
188   public void testRenameFamily() throws Exception {
189     String sourceTable = "sourceTable";
190     String targetTable = "targetTable";
191 
192     byte[][] families = { FAMILY_A, FAMILY_B };
193 
194     Table t = TEST_UTIL.createTable(Bytes.toBytes(sourceTable), families);
195     Table t2 = TEST_UTIL.createTable(Bytes.toBytes(targetTable), families);
196     Put p = new Put(ROW1);
197     p.add(FAMILY_A, QUALIFIER,  Bytes.toBytes("Data11"));
198     p.add(FAMILY_B, QUALIFIER,  Bytes.toBytes("Data12"));
199     p.add(FAMILY_A, QUALIFIER,  Bytes.toBytes("Data13"));
200     t.put(p);
201     p = new Put(ROW2);
202     p.add(FAMILY_B, QUALIFIER, Bytes.toBytes("Dat21"));
203     p.add(FAMILY_A, QUALIFIER, Bytes.toBytes("Data22"));
204     p.add(FAMILY_B, QUALIFIER, Bytes.toBytes("Data23"));
205     t.put(p);
206 
207     long currentTime = System.currentTimeMillis();
208     String[] args = new String[] { "--new.name=" + targetTable, "--families=a:b", "--all.cells",
209         "--starttime=" + (currentTime - 100000), "--endtime=" + (currentTime + 100000),
210         "--versions=1", sourceTable };
211     assertNull(t2.get(new Get(ROW1)).getRow());
212 
213     assertTrue(runCopy(args));
214 
215     assertNotNull(t2.get(new Get(ROW1)).getRow());
216     Result res = t2.get(new Get(ROW1));
217     byte[] b1 = res.getValue(FAMILY_B, QUALIFIER);
218     assertEquals("Data13", new String(b1));
219     assertNotNull(t2.get(new Get(ROW2)).getRow());
220     res = t2.get(new Get(ROW2));
221     b1 = res.getValue(FAMILY_A, QUALIFIER);
222     // Data from the family of B is not copied
223     assertNull(b1);
224 
225   }
226 
227   /**
228    * Test main method of CopyTable.
229    */
230   @Test
231   public void testMainMethod() throws Exception {
232     String[] emptyArgs = { "-h" };
233     PrintStream oldWriter = System.err;
234     ByteArrayOutputStream data = new ByteArrayOutputStream();
235     PrintStream writer = new PrintStream(data);
236     System.setErr(writer);
237     SecurityManager SECURITY_MANAGER = System.getSecurityManager();
238     LauncherSecurityManager newSecurityManager= new LauncherSecurityManager();
239     System.setSecurityManager(newSecurityManager);
240     try {
241       CopyTable.main(emptyArgs);
242       fail("should be exit");
243     } catch (SecurityException e) {
244       assertEquals(1, newSecurityManager.getExitCode());
245     } finally {
246       System.setErr(oldWriter);
247       System.setSecurityManager(SECURITY_MANAGER);
248     }
249     assertTrue(data.toString().contains("rs.class"));
250     // should print usage information
251     assertTrue(data.toString().contains("Usage:"));
252   }
253 
254   private boolean runCopy(String[] args) throws IOException, InterruptedException,
255       ClassNotFoundException {
256     GenericOptionsParser opts = new GenericOptionsParser(
257         new Configuration(TEST_UTIL.getConfiguration()), args);
258     Configuration configuration = opts.getConfiguration();
259     args = opts.getRemainingArgs();
260     Job job = new CopyTable(configuration).createSubmittableJob(args);
261     job.waitForCompletion(false);
262     return job.isSuccessful();
263   }
264 }