View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements. See the NOTICE file distributed with this
4    * work for additional information regarding copyright ownership. The ASF
5    * licenses this file to you under the Apache License, Version 2.0 (the
6    * "License"); you may not use this file except in compliance with the License.
7    * You may obtain a copy of the License at
8    *
9    * http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14   * License for the specific language governing permissions and limitations under
15   * the License.
16   */
17  package org.apache.hadoop.hbase.io.crypto;
18  
19  import java.io.IOException;
20  import java.io.InputStream;
21  import java.io.OutputStream;
22  import java.security.DigestException;
23  import java.security.Key;
24  import java.security.MessageDigest;
25  import java.security.NoSuchAlgorithmException;
26  import java.security.spec.InvalidKeySpecException;
27  import java.util.Map;
28  import java.util.concurrent.ConcurrentHashMap;
29  
30  import javax.crypto.SecretKeyFactory;
31  import javax.crypto.spec.PBEKeySpec;
32  import javax.crypto.spec.SecretKeySpec;
33  
34  import org.apache.commons.io.IOUtils;
35  import org.apache.commons.logging.Log;
36  import org.apache.commons.logging.LogFactory;
37  import org.apache.hadoop.classification.InterfaceAudience;
38  import org.apache.hadoop.classification.InterfaceStability;
39  import org.apache.hadoop.conf.Configuration;
40  import org.apache.hadoop.hbase.HBaseConfiguration;
41  import org.apache.hadoop.hbase.HConstants;
42  import org.apache.hadoop.hbase.util.Bytes;
43  import org.apache.hadoop.hbase.util.Pair;
44  import org.apache.hadoop.util.ReflectionUtils;
45  
46  /**
47   * A facade for encryption algorithms and related support.
48   */
49  @InterfaceAudience.Public
50  @InterfaceStability.Evolving
51  public final class Encryption {
52  
53    private static final Log LOG = LogFactory.getLog(Encryption.class);
54  
55    /**
56     * Crypto context
57     */
58    public static class Context extends org.apache.hadoop.hbase.io.crypto.Context {
59  
60      /** The null crypto context */
61      public static final Context NONE = new Context();
62  
63      private Context() {
64        super();
65      }
66  
67      private Context(Configuration conf) {
68        super(conf);
69      }
70  
71      @Override
72      public Context setCipher(Cipher cipher) {
73        super.setCipher(cipher);
74        return this;
75      }
76  
77      @Override
78      public Context setKey(Key key) {
79        super.setKey(key);
80        return this;
81      }
82  
83      public Context setKey(byte[] key) {
84        super.setKey(new SecretKeySpec(key, getCipher().getName()));
85        return this;
86      }
87    }
88  
89    public static Context newContext() {
90      return new Context();
91    }
92  
93    public static Context newContext(Configuration conf) {
94      return new Context(conf);
95    }
96  
97    // Prevent instantiation
98    private Encryption() {
99      super();
100   }
101 
102   /**
103    * Get an cipher given a name
104    * @param name the cipher name
105    * @return the cipher, or null if a suitable one could not be found
106    */
107   public static Cipher getCipher(Configuration conf, String name) {
108     return getCipherProvider(conf).getCipher(name);
109   }
110 
111   /**
112    * Get names of supported encryption algorithms
113    *
114    * @return Array of strings, each represents a supported encryption algorithm
115    */
116   public static String[] getSupportedCiphers() {
117     return getSupportedCiphers(HBaseConfiguration.create());
118   }
119 
120   /**
121    * Get names of supported encryption algorithms
122    *
123    * @return Array of strings, each represents a supported encryption algorithm
124    */
125   public static String[] getSupportedCiphers(Configuration conf) {
126     return getCipherProvider(conf).getSupportedCiphers();
127   }
128 
129   /**
130    * Return the MD5 digest of the concatenation of the supplied arguments.
131    */
132   public static byte[] hash128(String... args) {
133     byte[] result = new byte[16];
134     try {
135       MessageDigest md = MessageDigest.getInstance("MD5");
136       for (String arg: args) {
137         md.update(Bytes.toBytes(arg));
138       }
139       md.digest(result, 0, result.length);
140       return result;
141     } catch (NoSuchAlgorithmException e) {
142       throw new RuntimeException(e);
143     } catch (DigestException e) {
144       throw new RuntimeException(e);
145     }
146   }
147 
148   /**
149    * Return the MD5 digest of the concatenation of the supplied arguments.
150    */
151   public static byte[] hash128(byte[]... args) {
152     byte[] result = new byte[16];
153     try {
154       MessageDigest md = MessageDigest.getInstance("MD5");
155       for (byte[] arg: args) {
156         md.update(arg);
157       }
158       md.digest(result, 0, result.length);
159       return result;
160     } catch (NoSuchAlgorithmException e) {
161       throw new RuntimeException(e);
162     } catch (DigestException e) {
163       throw new RuntimeException(e);
164     }
165   }
166 
167   /**
168    * Return the SHA-256 digest of the concatenation of the supplied arguments.
169    */
170   public static byte[] hash256(String... args) {
171     byte[] result = new byte[32];
172     try {
173       MessageDigest md = MessageDigest.getInstance("SHA-256");
174       for (String arg: args) {
175         md.update(Bytes.toBytes(arg));
176       }
177       md.digest(result, 0, result.length);
178       return result;
179     } catch (NoSuchAlgorithmException e) {
180       throw new RuntimeException(e);
181     } catch (DigestException e) {
182       throw new RuntimeException(e);
183     }
184   }
185 
186   /**
187    * Return the SHA-256 digest of the concatenation of the supplied arguments.
188    */
189   public static byte[] hash256(byte[]... args) {
190     byte[] result = new byte[32];
191     try {
192       MessageDigest md = MessageDigest.getInstance("SHA-256");
193       for (byte[] arg: args) {
194         md.update(arg);
195       }
196       md.digest(result, 0, result.length);
197       return result;
198     } catch (NoSuchAlgorithmException e) {
199       throw new RuntimeException(e);
200     } catch (DigestException e) {
201       throw new RuntimeException(e);
202     }
203   }
204 
205   /**
206    * Return a 128 bit key derived from the concatenation of the supplied
207    * arguments using PBKDF2WithHmacSHA1 at 10,000 iterations.
208    * 
209    */
210   public static byte[] pbkdf128(String... args) {
211     byte[] salt = new byte[128];
212     Bytes.random(salt);
213     StringBuilder sb = new StringBuilder();
214     for (String s: args) {
215       sb.append(s);
216     }
217     PBEKeySpec spec = new PBEKeySpec(sb.toString().toCharArray(), salt, 10000, 128);
218     try {
219       return SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1")
220         .generateSecret(spec).getEncoded();
221     } catch (NoSuchAlgorithmException e) {
222       throw new RuntimeException(e);
223     } catch (InvalidKeySpecException e) {
224       throw new RuntimeException(e);
225     }
226   }
227 
228   /**
229    * Return a 128 bit key derived from the concatenation of the supplied
230    * arguments using PBKDF2WithHmacSHA1 at 10,000 iterations.
231    * 
232    */
233   public static byte[] pbkdf128(byte[]... args) {
234     byte[] salt = new byte[128];
235     Bytes.random(salt);
236     StringBuilder sb = new StringBuilder();
237     for (byte[] b: args) {
238       sb.append(b);
239     }
240     PBEKeySpec spec = new PBEKeySpec(sb.toString().toCharArray(), salt, 10000, 128);
241     try {
242       return SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1")
243         .generateSecret(spec).getEncoded();
244     } catch (NoSuchAlgorithmException e) {
245       throw new RuntimeException(e);
246     } catch (InvalidKeySpecException e) {
247       throw new RuntimeException(e);
248     }
249   }
250 
251   /**
252    * Encrypt a block of plaintext
253    * <p>
254    * The encryptor's state will be finalized. It should be reinitialized or
255    * returned to the pool.
256    * @param out ciphertext
257    * @param src plaintext
258    * @param offset
259    * @param length
260    * @param e
261    * @throws IOException
262     */
263   public static void encrypt(OutputStream out, byte[] src, int offset,
264       int length, Encryptor e) throws IOException {
265     OutputStream cout = e.createEncryptionStream(out);
266     try {
267       cout.write(src, offset, length);
268     } finally {
269       cout.close();
270     }
271   }
272 
273   /**
274    * Encrypt a block of plaintext
275    * @param out ciphertext
276    * @param src plaintext
277    * @param offset
278    * @param length
279    * @param context
280    * @param iv
281    * @throws IOException
282     */
283   public static void encrypt(OutputStream out, byte[] src, int offset,
284       int length, Context context, byte[] iv) throws IOException {
285     Encryptor e = context.getCipher().getEncryptor();
286     e.setKey(context.getKey());
287     e.setIv(iv); // can be null
288     e.reset();
289     encrypt(out, src, offset, length, e);
290   }
291 
292   /**
293    * Encrypt a stream of plaintext given an encryptor
294    * <p>
295    * The encryptor's state will be finalized. It should be reinitialized or
296    * returned to the pool.
297    * @param out ciphertext
298    * @param in plaintext
299    * @param e
300    * @throws IOException
301    */
302   public static void encrypt(OutputStream out, InputStream in, Encryptor e)
303       throws IOException {
304     OutputStream cout = e.createEncryptionStream(out);
305     try {
306       IOUtils.copy(in, cout);
307     } finally {
308       cout.close();
309     }
310   }
311 
312   /**
313    * Encrypt a stream of plaintext given a context and IV
314    * @param out ciphertext
315    * @param in plaintet
316    * @param context
317    * @param iv
318    * @throws IOException
319    */
320   public static void encrypt(OutputStream out, InputStream in, Context context,
321       byte[] iv) throws IOException {
322     Encryptor e = context.getCipher().getEncryptor();
323     e.setKey(context.getKey());
324     e.setIv(iv); // can be null
325     e.reset();
326     encrypt(out, in, e);
327   }
328 
329   /**
330    * Decrypt a block of ciphertext read in from a stream with the given
331    * cipher and context
332    * <p>
333    * The decryptor's state will be finalized. It should be reinitialized or
334    * returned to the pool.
335    * @param dest
336    * @param destOffset
337    * @param in
338    * @param destSize
339    * @param d
340    * @throws IOException
341    */
342   public static void decrypt(byte[] dest, int destOffset, InputStream in,
343       int destSize, Decryptor d) throws IOException {
344     InputStream cin = d.createDecryptionStream(in);
345     try {
346       IOUtils.readFully(cin, dest, destOffset, destSize);
347     } finally {
348       cin.close();
349     }
350   }
351 
352   /**
353    * Decrypt a block of ciphertext from a stream given a context and IV
354    * @param dest
355    * @param destOffset
356    * @param in
357    * @param destSize
358    * @param context
359    * @param iv
360    * @throws IOException
361    */
362   public static void decrypt(byte[] dest, int destOffset, InputStream in,
363       int destSize, Context context, byte[] iv) throws IOException {
364     Decryptor d = context.getCipher().getDecryptor();
365     d.setKey(context.getKey());
366     d.setIv(iv); // can be null
367     decrypt(dest, destOffset, in, destSize, d);
368   }
369 
370   /**
371    * Decrypt a stream of ciphertext given a decryptor
372    * @param out
373    * @param in
374    * @param outLen
375    * @param d
376    * @throws IOException
377    */
378   public static void decrypt(OutputStream out, InputStream in, int outLen,
379       Decryptor d) throws IOException {
380     InputStream cin = d.createDecryptionStream(in);
381     byte buf[] = new byte[8*1024];
382     long remaining = outLen;
383     try {
384       while (remaining > 0) {
385         int toRead = (int)(remaining < buf.length ? remaining : buf.length);
386         int read = cin.read(buf, 0, toRead);
387         if (read < 0) {
388           break;
389         }
390         out.write(buf, 0, read);
391         remaining -= read;
392       }
393     } finally {
394       cin.close();
395     }
396   }
397 
398   /**
399    * Decrypt a stream of ciphertext given a context and IV
400    * @param out
401    * @param in
402    * @param outLen
403    * @param context
404    * @param iv
405    * @throws IOException
406    */
407   public static void decrypt(OutputStream out, InputStream in, int outLen,
408       Context context, byte[] iv) throws IOException {
409     Decryptor d = context.getCipher().getDecryptor();
410     d.setKey(context.getKey());
411     d.setIv(iv); // can be null
412     decrypt(out, in, outLen, d);
413   }
414 
415   /**
416    * Resolves a key for the given subject
417    * @param subject
418    * @param conf
419    * @return a key for the given subject
420    * @throws IOException if the key is not found
421    */
422   public static Key getSecretKeyForSubject(String subject, Configuration conf)
423       throws IOException {
424     KeyProvider provider = (KeyProvider)getKeyProvider(conf);
425     if (provider != null) try {
426       Key[] keys = provider.getKeys(new String[] { subject });
427       if (keys != null && keys.length > 0) {
428         return keys[0];
429       }
430     } catch (Exception e) {
431       throw new IOException(e);
432     }
433     throw new IOException("No key found for subject '" + subject + "'");
434   }
435 
436   /**
437    * Encrypts a block of plaintext with the symmetric key resolved for the given subject
438    * @param out ciphertext
439    * @param in plaintext
440    * @param conf configuration
441    * @param cipher the encryption algorithm
442    * @param iv the initialization vector, can be null
443    * @throws IOException
444    */
445   public static void encryptWithSubjectKey(OutputStream out, InputStream in,
446       String subject, Configuration conf, Cipher cipher, byte[] iv)
447       throws IOException {
448     Key key = getSecretKeyForSubject(subject, conf);
449     if (key == null) {
450       throw new IOException("No key found for subject '" + subject + "'");
451     }
452     Encryptor e = cipher.getEncryptor();
453     e.setKey(key);
454     e.setIv(iv); // can be null
455     encrypt(out, in, e);
456   }
457 
458   /**
459    * Decrypts a block of ciphertext with the symmetric key resolved for the given subject
460    * @param out plaintext
461    * @param in ciphertext
462    * @param outLen the expected plaintext length
463    * @param subject the subject's key alias
464    * @param conf configuration
465    * @param cipher the encryption algorithm
466    * @param iv the initialization vector, can be null
467    * @throws IOException
468    */
469   public static void decryptWithSubjectKey(OutputStream out, InputStream in,
470       int outLen, String subject, Configuration conf, Cipher cipher,
471       byte[] iv) throws IOException {
472     Key key = getSecretKeyForSubject(subject, conf);
473     if (key == null) {
474       throw new IOException("No key found for subject '" + subject + "'");
475     }
476     Decryptor d = cipher.getDecryptor();
477     d.setKey(key);
478     d.setIv(iv); // can be null
479     decrypt(out, in, outLen, d);
480   }
481 
482   private static ClassLoader getClassLoaderForClass(Class<?> c) {
483     ClassLoader cl = Thread.currentThread().getContextClassLoader();
484     if (cl == null) {
485       cl = c.getClassLoader();
486     }
487     if (cl == null) {
488       cl = ClassLoader.getSystemClassLoader();
489     }
490     if (cl == null) {
491       throw new RuntimeException("A ClassLoader to load the Cipher could not be determined");
492     }
493     return cl;
494   }
495 
496   public static CipherProvider getCipherProvider(Configuration conf) {
497     String providerClassName = conf.get(HConstants.CRYPTO_CIPHERPROVIDER_CONF_KEY,
498       DefaultCipherProvider.class.getName());
499     try {
500       CipherProvider provider = (CipherProvider)
501         ReflectionUtils.newInstance(getClassLoaderForClass(CipherProvider.class)
502           .loadClass(providerClassName),
503         conf);
504       return provider;
505     } catch (Exception e) {
506       throw new RuntimeException(e);
507     }
508   }
509 
510   static final Map<Pair<String,String>,KeyProvider> keyProviderCache =
511       new ConcurrentHashMap<Pair<String,String>,KeyProvider>();
512 
513   public static KeyProvider getKeyProvider(Configuration conf) {
514     String providerClassName = conf.get(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY,
515       KeyStoreKeyProvider.class.getName());
516     String providerParameters = conf.get(HConstants.CRYPTO_KEYPROVIDER_PARAMETERS_KEY, "");
517     try {
518       Pair<String,String> providerCacheKey = new Pair<String,String>(providerClassName,
519         providerParameters);
520       KeyProvider provider = keyProviderCache.get(providerCacheKey);
521       if (provider != null) {
522         return provider;
523       }
524       provider = (KeyProvider) ReflectionUtils.newInstance(
525         getClassLoaderForClass(KeyProvider.class).loadClass(providerClassName),
526         conf);
527       provider.init(providerParameters);
528       if (LOG.isDebugEnabled()) {
529         LOG.debug("Installed " + providerClassName + " into key provider cache");
530       }
531       keyProviderCache.put(providerCacheKey, provider);
532       return provider;
533     } catch (Exception e) {
534       throw new RuntimeException(e);
535     }
536   }
537 
538   public static void incrementIv(byte[] iv) {
539     incrementIv(iv, 1);
540   }
541 
542   public static void incrementIv(byte[] iv, int v) {
543     int length = iv.length;
544     boolean carry = true;
545     // TODO: Optimize for v > 1, e.g. 16, 32
546     do {
547       for (int i = 0; i < length; i++) {
548         if (carry) {
549           iv[i] = (byte) ((iv[i] + 1) & 0xFF);
550           carry = 0 == iv[i];
551         } else {
552           break;
553         }
554       }
555       v--;
556     } while (v > 0);
557   }
558 
559 }