001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.crypto.key;
020
021import java.io.ByteArrayInputStream;
022import java.io.ByteArrayOutputStream;
023import java.io.IOException;
024import java.io.InputStreamReader;
025import java.io.OutputStreamWriter;
026import java.security.NoSuchAlgorithmException;
027import java.util.Collections;
028import java.util.Date;
029import java.util.HashMap;
030import java.util.List;
031import java.util.Map;
032
033import com.google.gson.stream.JsonReader;
034import com.google.gson.stream.JsonWriter;
035import org.apache.commons.io.Charsets;
036import org.apache.hadoop.classification.InterfaceAudience;
037import org.apache.hadoop.classification.InterfaceStability;
038import org.apache.hadoop.conf.Configuration;
039
040import javax.crypto.KeyGenerator;
041
042/**
043 * A provider of secret key material for Hadoop applications. Provides an
044 * abstraction to separate key storage from users of encryption. It
045 * is intended to support getting or storing keys in a variety of ways,
046 * including third party bindings.
047 * <P/>
048 * <code>KeyProvider</code> implementations must be thread safe.
049 */
050@InterfaceAudience.Public
051@InterfaceStability.Unstable
052public abstract class KeyProvider {
053  public static final String DEFAULT_CIPHER_NAME =
054      "hadoop.security.key.default.cipher";
055  public static final String DEFAULT_CIPHER = "AES/CTR/NoPadding";
056  public static final String DEFAULT_BITLENGTH_NAME =
057      "hadoop.security.key.default.bitlength";
058  public static final int DEFAULT_BITLENGTH = 128;
059
060  private final Configuration conf;
061
062  /**
063   * The combination of both the key version name and the key material.
064   */
065  public static class KeyVersion {
066    private final String name;
067    private final String versionName;
068    private final byte[] material;
069
070    protected KeyVersion(String name, String versionName,
071                         byte[] material) {
072      this.name = name;
073      this.versionName = versionName;
074      this.material = material;
075    }
076
077    public String getName() {
078      return name;
079    }
080
081    public String getVersionName() {
082      return versionName;
083    }
084
085    public byte[] getMaterial() {
086      return material;
087    }
088
089    public String toString() {
090      StringBuilder buf = new StringBuilder();
091      buf.append("key(");
092      buf.append(versionName);
093      buf.append(")=");
094      if (material == null) {
095        buf.append("null");
096      } else {
097        for(byte b: material) {
098          buf.append(' ');
099          int right = b & 0xff;
100          if (right < 0x10) {
101            buf.append('0');
102          }
103          buf.append(Integer.toHexString(right));
104        }
105      }
106      return buf.toString();
107    }
108  }
109
110  /**
111   * Key metadata that is associated with the key.
112   */
113  public static class Metadata {
114    private final static String CIPHER_FIELD = "cipher";
115    private final static String BIT_LENGTH_FIELD = "bitLength";
116    private final static String CREATED_FIELD = "created";
117    private final static String DESCRIPTION_FIELD = "description";
118    private final static String VERSIONS_FIELD = "versions";
119    private final static String ATTRIBUTES_FIELD = "attributes";
120
121    private final String cipher;
122    private final int bitLength;
123    private final String description;
124    private final Date created;
125    private int versions;
126    private Map<String, String> attributes;
127
128    protected Metadata(String cipher, int bitLength, String description,
129        Map<String, String> attributes, Date created, int versions) {
130      this.cipher = cipher;
131      this.bitLength = bitLength;
132      this.description = description;
133      this.attributes = (attributes == null || attributes.isEmpty())
134                        ? null : attributes;
135      this.created = created;
136      this.versions = versions;
137    }
138
139    public String toString() {
140      final StringBuilder metaSB = new StringBuilder();
141      metaSB.append("cipher: ").append(cipher).append(", ");
142      metaSB.append("length: ").append(bitLength).append(", ");
143      metaSB.append("description: ").append(description).append(", ");
144      metaSB.append("created: ").append(created).append(", ");
145      metaSB.append("version: ").append(versions).append(", ");
146      metaSB.append("attributes: ");
147      if ((attributes != null) && !attributes.isEmpty()) {
148        for (Map.Entry<String, String> attribute : attributes.entrySet()) {
149          metaSB.append("[");
150          metaSB.append(attribute.getKey());
151          metaSB.append("=");
152          metaSB.append(attribute.getValue());
153          metaSB.append("], ");
154        }
155        metaSB.deleteCharAt(metaSB.length() - 2);  // remove last ', '
156      } else {
157        metaSB.append("null");
158      }
159      return metaSB.toString();
160    }
161
162    public String getDescription() {
163      return description;
164    }
165
166    public Date getCreated() {
167      return created;
168    }
169
170    public String getCipher() {
171      return cipher;
172    }
173
174    @SuppressWarnings("unchecked")
175    public Map<String, String> getAttributes() {
176      return (attributes == null) ? Collections.EMPTY_MAP : attributes;
177    }
178
179    /**
180     * Get the algorithm from the cipher.
181     * @return the algorithm name
182     */
183    public String getAlgorithm() {
184      int slash = cipher.indexOf('/');
185      if (slash == - 1) {
186        return cipher;
187      } else {
188        return cipher.substring(0, slash);
189      }
190    }
191
192    public int getBitLength() {
193      return bitLength;
194    }
195
196    public int getVersions() {
197      return versions;
198    }
199
200    protected int addVersion() {
201      return versions++;
202    }
203
204    /**
205     * Serialize the metadata to a set of bytes.
206     * @return the serialized bytes
207     * @throws IOException
208     */
209    protected byte[] serialize() throws IOException {
210      ByteArrayOutputStream buffer = new ByteArrayOutputStream();
211      JsonWriter writer = new JsonWriter(
212          new OutputStreamWriter(buffer, Charsets.UTF_8));
213      try {
214        writer.beginObject();
215        if (cipher != null) {
216          writer.name(CIPHER_FIELD).value(cipher);
217        }
218        if (bitLength != 0) {
219          writer.name(BIT_LENGTH_FIELD).value(bitLength);
220        }
221        if (created != null) {
222          writer.name(CREATED_FIELD).value(created.getTime());
223        }
224        if (description != null) {
225          writer.name(DESCRIPTION_FIELD).value(description);
226        }
227        if (attributes != null && attributes.size() > 0) {
228          writer.name(ATTRIBUTES_FIELD).beginObject();
229          for (Map.Entry<String, String> attribute : attributes.entrySet()) {
230            writer.name(attribute.getKey()).value(attribute.getValue());
231          }
232          writer.endObject();
233        }
234        writer.name(VERSIONS_FIELD).value(versions);
235        writer.endObject();
236        writer.flush();
237      } finally {
238        writer.close();
239      }
240      return buffer.toByteArray();
241    }
242
243    /**
244     * Deserialize a new metadata object from a set of bytes.
245     * @param bytes the serialized metadata
246     * @throws IOException
247     */
248    protected Metadata(byte[] bytes) throws IOException {
249      String cipher = null;
250      int bitLength = 0;
251      Date created = null;
252      int versions = 0;
253      String description = null;
254      Map<String, String> attributes = null;
255      JsonReader reader = new JsonReader(new InputStreamReader
256        (new ByteArrayInputStream(bytes), Charsets.UTF_8));
257      try {
258        reader.beginObject();
259        while (reader.hasNext()) {
260          String field = reader.nextName();
261          if (CIPHER_FIELD.equals(field)) {
262            cipher = reader.nextString();
263          } else if (BIT_LENGTH_FIELD.equals(field)) {
264            bitLength = reader.nextInt();
265          } else if (CREATED_FIELD.equals(field)) {
266            created = new Date(reader.nextLong());
267          } else if (VERSIONS_FIELD.equals(field)) {
268            versions = reader.nextInt();
269          } else if (DESCRIPTION_FIELD.equals(field)) {
270            description = reader.nextString();
271          } else if (ATTRIBUTES_FIELD.equalsIgnoreCase(field)) {
272            reader.beginObject();
273            attributes = new HashMap<String, String>();
274            while (reader.hasNext()) {
275              attributes.put(reader.nextName(), reader.nextString());
276            }
277            reader.endObject();
278          }
279        }
280        reader.endObject();
281      } finally {
282        reader.close();
283      }
284      this.cipher = cipher;
285      this.bitLength = bitLength;
286      this.created = created;
287      this.description = description;
288      this.attributes = attributes;
289      this.versions = versions;
290    }
291  }
292
293  /**
294   * Options when creating key objects.
295   */
296  public static class Options {
297    private String cipher;
298    private int bitLength;
299    private String description;
300    private Map<String, String> attributes;
301
302    public Options(Configuration conf) {
303      cipher = conf.get(DEFAULT_CIPHER_NAME, DEFAULT_CIPHER);
304      bitLength = conf.getInt(DEFAULT_BITLENGTH_NAME, DEFAULT_BITLENGTH);
305    }
306
307    public Options setCipher(String cipher) {
308      this.cipher = cipher;
309      return this;
310    }
311
312    public Options setBitLength(int bitLength) {
313      this.bitLength = bitLength;
314      return this;
315    }
316
317    public Options setDescription(String description) {
318      this.description = description;
319      return this;
320    }
321
322    public Options setAttributes(Map<String, String> attributes) {
323      if (attributes != null) {
324        if (attributes.containsKey(null)) {
325          throw new IllegalArgumentException("attributes cannot have a NULL key");
326        }
327        this.attributes = new HashMap<String, String>(attributes);
328      }
329      return this;
330    }
331
332    public String getCipher() {
333      return cipher;
334    }
335
336    public int getBitLength() {
337      return bitLength;
338    }
339
340    public String getDescription() {
341      return description;
342    }
343
344    @SuppressWarnings("unchecked")
345    public Map<String, String> getAttributes() {
346      return (attributes == null) ? Collections.EMPTY_MAP : attributes;
347    }
348
349    @Override
350    public String toString() {
351      return "Options{" +
352          "cipher='" + cipher + '\'' +
353          ", bitLength=" + bitLength +
354          ", description='" + description + '\'' +
355          ", attributes=" + attributes +
356          '}';
357    }
358  }
359
360  /**
361   * Constructor.
362   * 
363   * @param conf configuration for the provider
364   */
365  public KeyProvider(Configuration conf) {
366    this.conf = new Configuration(conf);
367  }
368
369  /**
370   * Return the provider configuration.
371   * 
372   * @return the provider configuration
373   */
374  public Configuration getConf() {
375    return conf;
376  }
377  
378  /**
379   * A helper function to create an options object.
380   * @param conf the configuration to use
381   * @return a new options object
382   */
383  public static Options options(Configuration conf) {
384    return new Options(conf);
385  }
386
387  /**
388   * Indicates whether this provider represents a store
389   * that is intended for transient use - such as the UserProvider
390   * is. These providers are generally used to provide access to
391   * keying material rather than for long term storage.
392   * @return true if transient, false otherwise
393   */
394  public boolean isTransient() {
395    return false;
396  }
397
398  /**
399   * Get the key material for a specific version of the key. This method is used
400   * when decrypting data.
401   * @param versionName the name of a specific version of the key
402   * @return the key material
403   * @throws IOException
404   */
405  public abstract KeyVersion getKeyVersion(String versionName
406                                            ) throws IOException;
407
408  /**
409   * Get the key names for all keys.
410   * @return the list of key names
411   * @throws IOException
412   */
413  public abstract List<String> getKeys() throws IOException;
414
415  /**
416   * Get key metadata in bulk.
417   * @param names the names of the keys to get
418   * @throws IOException
419   */
420  public Metadata[] getKeysMetadata(String... names) throws IOException {
421    Metadata[] result = new Metadata[names.length];
422    for (int i=0; i < names.length; ++i) {
423      result[i] = getMetadata(names[i]);
424    }
425    return result;
426  }
427
428  /**
429   * Get the key material for all versions of a specific key name.
430   * @return the list of key material
431   * @throws IOException
432   */
433  public abstract List<KeyVersion> getKeyVersions(String name) throws IOException;
434
435  /**
436   * Get the current version of the key, which should be used for encrypting new
437   * data.
438   * @param name the base name of the key
439   * @return the version name of the current version of the key or null if the
440   *    key version doesn't exist
441   * @throws IOException
442   */
443  public KeyVersion getCurrentKey(String name) throws IOException {
444    Metadata meta = getMetadata(name);
445    if (meta == null) {
446      return null;
447    }
448    return getKeyVersion(buildVersionName(name, meta.getVersions() - 1));
449  }
450
451  /**
452   * Get metadata about the key.
453   * @param name the basename of the key
454   * @return the key's metadata or null if the key doesn't exist
455   * @throws IOException
456   */
457  public abstract Metadata getMetadata(String name) throws IOException;
458
459  /**
460   * Create a new key. The given key must not already exist.
461   * @param name the base name of the key
462   * @param material the key material for the first version of the key.
463   * @param options the options for the new key.
464   * @return the version name of the first version of the key.
465   * @throws IOException
466   */
467  public abstract KeyVersion createKey(String name, byte[] material,
468                                       Options options) throws IOException;
469
470  /**
471   * Get the algorithm from the cipher.
472   *
473   * @return the algorithm name
474   */
475  private String getAlgorithm(String cipher) {
476    int slash = cipher.indexOf('/');
477    if (slash == -1) {
478      return cipher;
479    } else {
480      return cipher.substring(0, slash);
481    }
482  }
483
484  /**
485   * Generates a key material.
486   *
487   * @param size length of the key.
488   * @param algorithm algorithm to use for generating the key.
489   * @return the generated key.
490   * @throws NoSuchAlgorithmException
491   */
492  protected byte[] generateKey(int size, String algorithm)
493      throws NoSuchAlgorithmException {
494    algorithm = getAlgorithm(algorithm);
495    KeyGenerator keyGenerator = KeyGenerator.getInstance(algorithm);
496    keyGenerator.init(size);
497    byte[] key = keyGenerator.generateKey().getEncoded();
498    return key;
499  }
500
501  /**
502   * Create a new key generating the material for it.
503   * The given key must not already exist.
504   * <p/>
505   * This implementation generates the key material and calls the
506   * {@link #createKey(String, byte[], Options)} method.
507   *
508   * @param name the base name of the key
509   * @param options the options for the new key.
510   * @return the version name of the first version of the key.
511   * @throws IOException
512   * @throws NoSuchAlgorithmException
513   */
514  public KeyVersion createKey(String name, Options options)
515      throws NoSuchAlgorithmException, IOException {
516    byte[] material = generateKey(options.getBitLength(), options.getCipher());
517    return createKey(name, material, options);
518  }
519
520  /**
521   * Delete the given key.
522   * @param name the name of the key to delete
523   * @throws IOException
524   */
525  public abstract void deleteKey(String name) throws IOException;
526
527  /**
528   * Roll a new version of the given key.
529   * @param name the basename of the key
530   * @param material the new key material
531   * @return the name of the new version of the key
532   * @throws IOException
533   */
534  public abstract KeyVersion rollNewVersion(String name,
535                                             byte[] material
536                                            ) throws IOException;
537
538  /**
539   * Can be used by implementing classes to close any resources
540   * that require closing
541   */
542  public void close() throws IOException {
543    // NOP
544  }
545
546  /**
547   * Roll a new version of the given key generating the material for it.
548   * <p/>
549   * This implementation generates the key material and calls the
550   * {@link #rollNewVersion(String, byte[])} method.
551   *
552   * @param name the basename of the key
553   * @return the name of the new version of the key
554   * @throws IOException
555   */
556  public KeyVersion rollNewVersion(String name) throws NoSuchAlgorithmException,
557                                                       IOException {
558    Metadata meta = getMetadata(name);
559    byte[] material = generateKey(meta.getBitLength(), meta.getCipher());
560    return rollNewVersion(name, material);
561  }
562
563  /**
564   * Ensures that any changes to the keys are written to persistent store.
565   * @throws IOException
566   */
567  public abstract void flush() throws IOException;
568
569  /**
570   * Split the versionName in to a base name. Converts "/aaa/bbb/3" to
571   * "/aaa/bbb".
572   * @param versionName the version name to split
573   * @return the base name of the key
574   * @throws IOException
575   */
576  public static String getBaseName(String versionName) throws IOException {
577    int div = versionName.lastIndexOf('@');
578    if (div == -1) {
579      throw new IOException("No version in key path " + versionName);
580    }
581    return versionName.substring(0, div);
582  }
583
584  /**
585   * Build a version string from a basename and version number. Converts
586   * "/aaa/bbb" and 3 to "/aaa/bbb@3".
587   * @param name the basename of the key
588   * @param version the version of the key
589   * @return the versionName of the key.
590   */
591  protected static String buildVersionName(String name, int version) {
592    return name + "@" + version;
593  }
594
595  /**
596   * Find the provider with the given key.
597   * @param providerList the list of providers
598   * @param keyName the key name we are looking for
599   * @return the KeyProvider that has the key
600   */
601  public static KeyProvider findProvider(List<KeyProvider> providerList,
602                                         String keyName) throws IOException {
603    for(KeyProvider provider: providerList) {
604      if (provider.getMetadata(keyName) != null) {
605        return provider;
606      }
607    }
608    throw new IOException("Can't find KeyProvider for key " + keyName);
609  }
610}