View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.mapreduce;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertNotNull;
22  import static org.junit.Assert.assertNull;
23  import static org.junit.Assert.assertTrue;
24  import static org.junit.Assert.fail;
25  
26  import java.io.ByteArrayOutputStream;
27  import java.io.IOException;
28  import java.io.PrintStream;
29  
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.hbase.CellUtil;
32  import org.apache.hadoop.hbase.HBaseTestingUtility;
33  import org.apache.hadoop.hbase.LargeTests;
34  import org.apache.hadoop.hbase.MiniHBaseCluster;
35  import org.apache.hadoop.hbase.client.Get;
36  import org.apache.hadoop.hbase.client.HTable;
37  import org.apache.hadoop.hbase.client.Put;
38  import org.apache.hadoop.hbase.client.Result;
39  import org.apache.hadoop.hbase.util.Bytes;
40  import org.apache.hadoop.hbase.util.LauncherSecurityManager;
41  import org.apache.hadoop.mapreduce.Job;
42  import org.apache.hadoop.util.GenericOptionsParser;
43  import org.junit.AfterClass;
44  import org.junit.BeforeClass;
45  import org.junit.Test;
46  import org.junit.experimental.categories.Category;
47  
48  /**
49   * Basic test for the CopyTable M/R tool
50   */
51  @Category(LargeTests.class)
52  public class TestCopyTable {
53    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
54    private static MiniHBaseCluster cluster;
55    private static final byte[] ROW1 = Bytes.toBytes("row1");
56    private static final byte[] ROW2 = Bytes.toBytes("row2");
57    private static final String FAMILY_A_STRING = "a";
58    private static final String FAMILY_B_STRING = "b";
59    private static final byte[] FAMILY_A = Bytes.toBytes(FAMILY_A_STRING);
60    private static final byte[] FAMILY_B = Bytes.toBytes(FAMILY_B_STRING);
61    private static final byte[] QUALIFIER = Bytes.toBytes("q");
62  
63  
64    @BeforeClass
65    public static void beforeClass() throws Exception {
66      cluster = TEST_UTIL.startMiniCluster(3);
67      TEST_UTIL.startMiniMapReduceCluster();
68    }
69  
70    @AfterClass
71    public static void afterClass() throws Exception {
72      TEST_UTIL.shutdownMiniMapReduceCluster();
73      TEST_UTIL.shutdownMiniCluster();
74    }
75  
76    /**
77     * Simple end-to-end test
78     * @throws Exception
79     */
80    @Test
81    public void testCopyTable() throws Exception {
82      final byte[] TABLENAME1 = Bytes.toBytes("testCopyTable1");
83      final byte[] TABLENAME2 = Bytes.toBytes("testCopyTable2");
84      final byte[] FAMILY = Bytes.toBytes("family");
85      final byte[] COLUMN1 = Bytes.toBytes("c1");
86  
87      HTable t1 = TEST_UTIL.createTable(TABLENAME1, FAMILY);
88      HTable t2 = TEST_UTIL.createTable(TABLENAME2, FAMILY);
89  
90      // put rows into the first table
91      for (int i = 0; i < 10; i++) {
92        Put p = new Put(Bytes.toBytes("row" + i));
93        p.add(FAMILY, COLUMN1, COLUMN1);
94        t1.put(p);
95      }
96  
97      CopyTable copy = new CopyTable(TEST_UTIL.getConfiguration());
98  
99      assertEquals(
100       0,
101       copy.run(new String[] { "--new.name=" + Bytes.toString(TABLENAME2),
102           Bytes.toString(TABLENAME1) }));
103 
104     // verify the data was copied into table 2
105     for (int i = 0; i < 10; i++) {
106       Get g = new Get(Bytes.toBytes("row" + i));
107       Result r = t2.get(g);
108       assertEquals(1, r.size());
109       assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1));
110     }
111     
112     t1.close();
113     t2.close();
114     TEST_UTIL.deleteTable(TABLENAME1);
115     TEST_UTIL.deleteTable(TABLENAME2);
116   }
117 
118   @Test
119   public void testStartStopRow() throws Exception {
120     final byte[] TABLENAME1 = Bytes.toBytes("testStartStopRow1");
121     final byte[] TABLENAME2 = Bytes.toBytes("testStartStopRow2");
122     final byte[] FAMILY = Bytes.toBytes("family");
123     final byte[] COLUMN1 = Bytes.toBytes("c1");
124     final byte[] ROW0 = Bytes.toBytes("row0");
125     final byte[] ROW1 = Bytes.toBytes("row1");
126     final byte[] ROW2 = Bytes.toBytes("row2");
127 
128     HTable t1 = TEST_UTIL.createTable(TABLENAME1, FAMILY);
129     HTable t2 = TEST_UTIL.createTable(TABLENAME2, FAMILY);
130 
131     // put rows into the first table
132     Put p = new Put(ROW0);
133     p.add(FAMILY, COLUMN1, COLUMN1);
134     t1.put(p);
135     p = new Put(ROW1);
136     p.add(FAMILY, COLUMN1, COLUMN1);
137     t1.put(p);
138     p = new Put(ROW2);
139     p.add(FAMILY, COLUMN1, COLUMN1);
140     t1.put(p);
141 
142     CopyTable copy = new CopyTable(TEST_UTIL.getConfiguration());
143     assertEquals(
144       0,
145       copy.run(new String[] { "--new.name=" + Bytes.toString(TABLENAME2), "--startrow=row1",
146           "--stoprow=row2", Bytes.toString(TABLENAME1) }));
147 
148     // verify the data was copied into table 2
149     // row1 exist, row0, row2 do not exist
150     Get g = new Get(ROW1);
151     Result r = t2.get(g);
152     assertEquals(1, r.size());
153     assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1));
154 
155     g = new Get(ROW0);
156     r = t2.get(g);
157     assertEquals(0, r.size());
158     
159     g = new Get(ROW2);
160     r = t2.get(g);
161     assertEquals(0, r.size());
162     
163     t1.close();
164     t2.close();
165     TEST_UTIL.deleteTable(TABLENAME1);
166     TEST_UTIL.deleteTable(TABLENAME2);
167   }
168 
169   /**
170    * Test copy of table from sourceTable to targetTable all rows from family a
171    */
172   @Test
173   public void testRenameFamily() throws Exception {
174     String sourceTable = "sourceTable";
175     String targetTable = "targetTable";
176 
177     byte[][] families = { FAMILY_A, FAMILY_B };
178 
179     HTable t = TEST_UTIL.createTable(Bytes.toBytes(sourceTable), families);
180     HTable t2 = TEST_UTIL.createTable(Bytes.toBytes(targetTable), families);
181     Put p = new Put(ROW1);
182     p.add(FAMILY_A, QUALIFIER,  Bytes.toBytes("Data11"));
183     p.add(FAMILY_B, QUALIFIER,  Bytes.toBytes("Data12"));
184     p.add(FAMILY_A, QUALIFIER,  Bytes.toBytes("Data13"));
185     t.put(p);
186     p = new Put(ROW2);
187     p.add(FAMILY_B, QUALIFIER, Bytes.toBytes("Dat21"));
188     p.add(FAMILY_A, QUALIFIER, Bytes.toBytes("Data22"));
189     p.add(FAMILY_B, QUALIFIER, Bytes.toBytes("Data23"));
190     t.put(p);
191 
192     long currentTime = System.currentTimeMillis();
193     String[] args = new String[] { "--new.name=" + targetTable, "--families=a:b", "--all.cells",
194         "--starttime=" + (currentTime - 100000), "--endtime=" + (currentTime + 100000),
195         "--versions=1", sourceTable };
196     assertNull(t2.get(new Get(ROW1)).getRow());
197     clean();
198 
199     assertTrue(runCopy(args));
200 
201     assertNotNull(t2.get(new Get(ROW1)).getRow());
202     Result res = t2.get(new Get(ROW1));
203     byte[] b1 = res.getValue(FAMILY_B, QUALIFIER);
204     assertEquals("Data13", new String(b1));
205     assertNotNull(t2.get(new Get(ROW2)).getRow());
206     res = t2.get(new Get(ROW2));
207     b1 = res.getValue(FAMILY_A, QUALIFIER);
208     // Data from the family of B is not copied
209     assertNull(b1);
210 
211   }
212 
213   /**
214    * Test main method of CopyTable.
215    */
216   @Test
217   public void testMainMethod() throws Exception {
218     String[] emptyArgs = { "-h" };
219     PrintStream oldWriter = System.err;
220     ByteArrayOutputStream data = new ByteArrayOutputStream();
221     PrintStream writer = new PrintStream(data);
222     System.setErr(writer);
223     SecurityManager SECURITY_MANAGER = System.getSecurityManager();
224     LauncherSecurityManager newSecurityManager= new LauncherSecurityManager();
225     System.setSecurityManager(newSecurityManager);
226     try {
227       CopyTable.main(emptyArgs);
228       fail("should be exit");
229     } catch (SecurityException e) {
230       assertEquals(1, newSecurityManager.getExitCode());
231     } finally {
232       System.setErr(oldWriter);
233       System.setSecurityManager(SECURITY_MANAGER);
234     }
235     assertTrue(data.toString().contains("rs.class"));
236     // should print usage information
237     assertTrue(data.toString().contains("Usage:"));
238   }
239 
240   private boolean runCopy(String[] args) throws IOException, InterruptedException,
241       ClassNotFoundException {
242     GenericOptionsParser opts = new GenericOptionsParser(
243         new Configuration(TEST_UTIL.getConfiguration()), args);
244     Configuration configuration = opts.getConfiguration();
245     args = opts.getRemainingArgs();
246     clean();
247     Job job = CopyTable.createSubmittableJob(configuration, args);
248     job.waitForCompletion(false);
249     return job.isSuccessful();
250   }
251 
252 
253   private void clean() {
254 
255       CopyTable.startTime = 0;
256       CopyTable.endTime = 0;
257       CopyTable.versions = -1;
258       CopyTable.tableName = null;
259       CopyTable.startRow = null;
260       CopyTable.stopRow = null;
261       CopyTable.newTableName = null;
262       CopyTable.peerAddress = null;
263       CopyTable.families = null;
264       CopyTable.allCells = false;
265   }
266 }