1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.mapreduce;
20
21 import java.io.IOException;
22
23 import org.apache.commons.logging.Log;
24 import org.apache.commons.logging.LogFactory;
25 import org.apache.hadoop.classification.InterfaceAudience;
26 import org.apache.hadoop.classification.InterfaceStability;
27 import org.apache.hadoop.conf.Configurable;
28 import org.apache.hadoop.conf.Configuration;
29 import org.apache.hadoop.hbase.HBaseConfiguration;
30 import org.apache.hadoop.hbase.HConstants;
31 import org.apache.hadoop.hbase.client.Delete;
32 import org.apache.hadoop.hbase.client.HTable;
33 import org.apache.hadoop.hbase.client.Mutation;
34 import org.apache.hadoop.hbase.client.Put;
35 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
36 import org.apache.hadoop.mapreduce.JobContext;
37 import org.apache.hadoop.mapreduce.OutputCommitter;
38 import org.apache.hadoop.mapreduce.OutputFormat;
39 import org.apache.hadoop.mapreduce.RecordWriter;
40 import org.apache.hadoop.mapreduce.TaskAttemptContext;
41
42
43
44
45
46
47
48
49 @InterfaceAudience.Public
50 @InterfaceStability.Stable
51 public class TableOutputFormat<KEY> extends OutputFormat<KEY, Mutation>
52 implements Configurable {
53
54 private final Log LOG = LogFactory.getLog(TableOutputFormat.class);
55
56
57 public static final String OUTPUT_TABLE = "hbase.mapred.outputtable";
58
59
60
61
62
63
64
65 public static final String QUORUM_ADDRESS = "hbase.mapred.output.quorum";
66
67
68 public static final String QUORUM_PORT = "hbase.mapred.output.quorum.port";
69
70
71 public static final String
72 REGION_SERVER_CLASS = "hbase.mapred.output.rs.class";
73
74 public static final String
75 REGION_SERVER_IMPL = "hbase.mapred.output.rs.impl";
76
77
78 private Configuration conf = null;
79
80 private HTable table;
81
82
83
84
85
86
87 protected static class TableRecordWriter<KEY>
88 extends RecordWriter<KEY, Mutation> {
89
90
91 private HTable table;
92
93
94
95
96
97
98 public TableRecordWriter(HTable table) {
99 this.table = table;
100 }
101
102
103
104
105
106
107
108
109 @Override
110 public void close(TaskAttemptContext context)
111 throws IOException {
112 table.close();
113 }
114
115
116
117
118
119
120
121
122
123 @Override
124 public void write(KEY key, Mutation value)
125 throws IOException {
126 if (value instanceof Put) this.table.put(new Put((Put)value));
127 else if (value instanceof Delete) this.table.delete(new Delete((Delete)value));
128 else throw new IOException("Pass a Delete or a Put");
129 }
130 }
131
132
133
134
135
136
137
138
139
140
141 @Override
142 public RecordWriter<KEY, Mutation> getRecordWriter(
143 TaskAttemptContext context)
144 throws IOException, InterruptedException {
145 return new TableRecordWriter<KEY>(this.table);
146 }
147
148
149
150
151
152
153
154
155
156 @Override
157 public void checkOutputSpecs(JobContext context) throws IOException,
158 InterruptedException {
159
160
161 }
162
163
164
165
166
167
168
169
170
171
172 @Override
173 public OutputCommitter getOutputCommitter(TaskAttemptContext context)
174 throws IOException, InterruptedException {
175 return new TableOutputCommitter();
176 }
177
178 public Configuration getConf() {
179 return conf;
180 }
181
182 @Override
183 public void setConf(Configuration otherConf) {
184 this.conf = HBaseConfiguration.create(otherConf);
185
186 String tableName = this.conf.get(OUTPUT_TABLE);
187 if(tableName == null || tableName.length() <= 0) {
188 throw new IllegalArgumentException("Must specify table name");
189 }
190
191 String address = this.conf.get(QUORUM_ADDRESS);
192 int zkClientPort = this.conf.getInt(QUORUM_PORT, 0);
193 String serverClass = this.conf.get(REGION_SERVER_CLASS);
194 String serverImpl = this.conf.get(REGION_SERVER_IMPL);
195
196 try {
197 if (address != null) {
198 ZKUtil.applyClusterKeyToConf(this.conf, address);
199 }
200 if (serverClass != null) {
201 this.conf.set(HConstants.REGION_SERVER_IMPL, serverImpl);
202 }
203 if (zkClientPort != 0) {
204 this.conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkClientPort);
205 }
206 this.table = new HTable(this.conf, tableName);
207 this.table.setAutoFlush(false, true);
208 LOG.info("Created table instance for " + tableName);
209 } catch(IOException e) {
210 LOG.error(e);
211 throw new RuntimeException(e);
212 }
213 }
214 }