1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver.wal;
20 import java.io.DataInput;
21 import java.io.DataOutput;
22 import java.io.IOException;
23
24 import org.apache.hadoop.classification.InterfaceAudience;
25 import org.apache.hadoop.conf.Configuration;
26 import org.apache.hadoop.fs.FileSystem;
27 import org.apache.hadoop.fs.Path;
28 import org.apache.hadoop.hbase.HBaseConfiguration;
29 import org.apache.hadoop.hbase.HConstants;
30 import org.apache.hadoop.hbase.io.util.Dictionary;
31 import org.apache.hadoop.hbase.util.Bytes;
32 import org.apache.hadoop.io.WritableUtils;
33
34 import com.google.common.base.Preconditions;
35
36
37
38
39
40 @InterfaceAudience.Private
41 public class Compressor {
42
43
44
45 public static void main(String[] args) throws IOException {
46 if (args.length != 2 || args[0].equals("--help") || args[0].equals("-h")) {
47 printHelp();
48 System.exit(-1);
49 }
50
51 Path inputPath = new Path(args[0]);
52 Path outputPath = new Path(args[1]);
53
54 transformFile(inputPath, outputPath);
55 }
56
57 private static void printHelp() {
58 System.err.println("usage: Compressor <input> <output>");
59 System.err.println("If <input> HLog is compressed, <output> will be decompressed.");
60 System.err.println("If <input> HLog is uncompressed, <output> will be compressed.");
61 return;
62 }
63
64 private static void transformFile(Path input, Path output)
65 throws IOException {
66 Configuration conf = HBaseConfiguration.create();
67
68 FileSystem inFS = input.getFileSystem(conf);
69 FileSystem outFS = output.getFileSystem(conf);
70
71 HLog.Reader in = HLogFactory.createReader(inFS, input, conf, null, false);
72 HLog.Writer out = null;
73
74 try {
75 if (!(in instanceof ReaderBase)) {
76 System.err.println("Cannot proceed, invalid reader type: " + in.getClass().getName());
77 return;
78 }
79 boolean compress = ((ReaderBase)in).hasCompression();
80 conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, !compress);
81 out = HLogFactory.createWALWriter(outFS, output, conf);
82
83 HLog.Entry e = null;
84 while ((e = in.next()) != null) out.append(e);
85 } finally {
86 in.close();
87 if (out != null) {
88 out.close();
89 out = null;
90 }
91 }
92 }
93
94
95
96
97
98
99
100
101 @Deprecated
102 static byte[] readCompressed(DataInput in, Dictionary dict)
103 throws IOException {
104 byte status = in.readByte();
105
106 if (status == Dictionary.NOT_IN_DICTIONARY) {
107 int length = WritableUtils.readVInt(in);
108
109 byte[] arr = new byte[length];
110 in.readFully(arr);
111 if (dict != null) dict.addEntry(arr, 0, length);
112 return arr;
113 } else {
114
115
116
117 short dictIdx = toShort(status, in.readByte());
118 byte[] entry = dict.getEntry(dictIdx);
119 if (entry == null) {
120 throw new IOException("Missing dictionary entry for index "
121 + dictIdx);
122 }
123 return entry;
124 }
125 }
126
127
128
129
130
131
132
133
134
135
136
137
138 @Deprecated
139 static int uncompressIntoArray(byte[] to, int offset, DataInput in,
140 Dictionary dict) throws IOException {
141 byte status = in.readByte();
142
143 if (status == Dictionary.NOT_IN_DICTIONARY) {
144
145
146 int length = WritableUtils.readVInt(in);
147 in.readFully(to, offset, length);
148 dict.addEntry(to, offset, length);
149 return length;
150 } else {
151
152
153 short dictIdx = toShort(status, in.readByte());
154 byte[] entry;
155 try {
156 entry = dict.getEntry(dictIdx);
157 } catch (Exception ex) {
158 throw new IOException("Unable to uncompress the log entry", ex);
159 }
160 if (entry == null) {
161 throw new IOException("Missing dictionary entry for index "
162 + dictIdx);
163 }
164
165 Bytes.putBytes(to, offset, entry, 0, entry.length);
166 return entry.length;
167 }
168 }
169
170
171
172
173
174
175
176
177 @Deprecated
178 static void writeCompressed(byte[] data, int offset, int length,
179 DataOutput out, Dictionary dict)
180 throws IOException {
181 short dictIdx = Dictionary.NOT_IN_DICTIONARY;
182 if (dict != null) {
183 dictIdx = dict.findEntry(data, offset, length);
184 }
185 if (dictIdx == Dictionary.NOT_IN_DICTIONARY) {
186
187 out.writeByte(Dictionary.NOT_IN_DICTIONARY);
188 WritableUtils.writeVInt(out, length);
189 out.write(data, offset, length);
190 } else {
191 out.writeShort(dictIdx);
192 }
193 }
194
195 static short toShort(byte hi, byte lo) {
196 short s = (short) (((hi & 0xFF) << 8) | (lo & 0xFF));
197 Preconditions.checkArgument(s >= 0);
198 return s;
199 }
200 }