001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    
019    package org.apache.hadoop.crypto.key;
020    
021    import java.io.ByteArrayInputStream;
022    import java.io.ByteArrayOutputStream;
023    import java.io.IOException;
024    import java.io.InputStreamReader;
025    import java.io.OutputStreamWriter;
026    import java.security.NoSuchAlgorithmException;
027    import java.util.Collections;
028    import java.util.Date;
029    import java.util.HashMap;
030    import java.util.List;
031    import java.util.Map;
032    
033    import com.google.gson.stream.JsonReader;
034    import com.google.gson.stream.JsonWriter;
035    import org.apache.hadoop.classification.InterfaceAudience;
036    import org.apache.hadoop.classification.InterfaceStability;
037    import org.apache.hadoop.conf.Configuration;
038    
039    import javax.crypto.KeyGenerator;
040    
041    /**
042     * A provider of secret key material for Hadoop applications. Provides an
043     * abstraction to separate key storage from users of encryption. It
044     * is intended to support getting or storing keys in a variety of ways,
045     * including third party bindings.
046     * <P/>
047     * <code>KeyProvider</code> implementations must be thread safe.
048     */
049    @InterfaceAudience.Public
050    @InterfaceStability.Unstable
051    public abstract class KeyProvider {
052      public static final String DEFAULT_CIPHER_NAME =
053          "hadoop.security.key.default.cipher";
054      public static final String DEFAULT_CIPHER = "AES/CTR/NoPadding";
055      public static final String DEFAULT_BITLENGTH_NAME =
056          "hadoop.security.key.default.bitlength";
057      public static final int DEFAULT_BITLENGTH = 128;
058    
059      private final Configuration conf;
060    
061      /**
062       * The combination of both the key version name and the key material.
063       */
064      public static class KeyVersion {
065        private final String name;
066        private final String versionName;
067        private final byte[] material;
068    
069        protected KeyVersion(String name, String versionName,
070                             byte[] material) {
071          this.name = name;
072          this.versionName = versionName;
073          this.material = material;
074        }
075    
076        public String getName() {
077          return name;
078        }
079    
080        public String getVersionName() {
081          return versionName;
082        }
083    
084        public byte[] getMaterial() {
085          return material;
086        }
087    
088        public String toString() {
089          StringBuilder buf = new StringBuilder();
090          buf.append("key(");
091          buf.append(versionName);
092          buf.append(")=");
093          if (material == null) {
094            buf.append("null");
095          } else {
096            for(byte b: material) {
097              buf.append(' ');
098              int right = b & 0xff;
099              if (right < 0x10) {
100                buf.append('0');
101              }
102              buf.append(Integer.toHexString(right));
103            }
104          }
105          return buf.toString();
106        }
107      }
108    
109      /**
110       * Key metadata that is associated with the key.
111       */
112      public static class Metadata {
113        private final static String CIPHER_FIELD = "cipher";
114        private final static String BIT_LENGTH_FIELD = "bitLength";
115        private final static String CREATED_FIELD = "created";
116        private final static String DESCRIPTION_FIELD = "description";
117        private final static String VERSIONS_FIELD = "versions";
118        private final static String ATTRIBUTES_FIELD = "attributes";
119    
120        private final String cipher;
121        private final int bitLength;
122        private final String description;
123        private final Date created;
124        private int versions;
125        private Map<String, String> attributes;
126    
127        protected Metadata(String cipher, int bitLength, String description,
128            Map<String, String> attributes, Date created, int versions) {
129          this.cipher = cipher;
130          this.bitLength = bitLength;
131          this.description = description;
132          this.attributes = (attributes == null || attributes.isEmpty())
133                            ? null : attributes;
134          this.created = created;
135          this.versions = versions;
136        }
137    
138        public String toString() {
139          final StringBuilder metaSB = new StringBuilder();
140          metaSB.append("cipher: ").append(cipher).append(", ");
141          metaSB.append("length: ").append(bitLength).append(", ");
142          metaSB.append("description: ").append(description).append(", ");
143          metaSB.append("created: ").append(created).append(", ");
144          metaSB.append("version: ").append(versions).append(", ");
145          metaSB.append("attributes: ");
146          if ((attributes != null) && !attributes.isEmpty()) {
147            for (Map.Entry<String, String> attribute : attributes.entrySet()) {
148              metaSB.append("[");
149              metaSB.append(attribute.getKey());
150              metaSB.append("=");
151              metaSB.append(attribute.getValue());
152              metaSB.append("], ");
153            }
154            metaSB.deleteCharAt(metaSB.length() - 2);  // remove last ', '
155          } else {
156            metaSB.append("null");
157          }
158          return metaSB.toString();
159        }
160    
161        public String getDescription() {
162          return description;
163        }
164    
165        public Date getCreated() {
166          return created;
167        }
168    
169        public String getCipher() {
170          return cipher;
171        }
172    
173        @SuppressWarnings("unchecked")
174        public Map<String, String> getAttributes() {
175          return (attributes == null) ? Collections.EMPTY_MAP : attributes;
176        }
177    
178        /**
179         * Get the algorithm from the cipher.
180         * @return the algorithm name
181         */
182        public String getAlgorithm() {
183          int slash = cipher.indexOf('/');
184          if (slash == - 1) {
185            return cipher;
186          } else {
187            return cipher.substring(0, slash);
188          }
189        }
190    
191        public int getBitLength() {
192          return bitLength;
193        }
194    
195        public int getVersions() {
196          return versions;
197        }
198    
199        protected int addVersion() {
200          return versions++;
201        }
202    
203        /**
204         * Serialize the metadata to a set of bytes.
205         * @return the serialized bytes
206         * @throws IOException
207         */
208        protected byte[] serialize() throws IOException {
209          ByteArrayOutputStream buffer = new ByteArrayOutputStream();
210          JsonWriter writer = new JsonWriter(new OutputStreamWriter(buffer));
211          try {
212            writer.beginObject();
213            if (cipher != null) {
214              writer.name(CIPHER_FIELD).value(cipher);
215            }
216            if (bitLength != 0) {
217              writer.name(BIT_LENGTH_FIELD).value(bitLength);
218            }
219            if (created != null) {
220              writer.name(CREATED_FIELD).value(created.getTime());
221            }
222            if (description != null) {
223              writer.name(DESCRIPTION_FIELD).value(description);
224            }
225            if (attributes != null && attributes.size() > 0) {
226              writer.name(ATTRIBUTES_FIELD).beginObject();
227              for (Map.Entry<String, String> attribute : attributes.entrySet()) {
228                writer.name(attribute.getKey()).value(attribute.getValue());
229              }
230              writer.endObject();
231            }
232            writer.name(VERSIONS_FIELD).value(versions);
233            writer.endObject();
234            writer.flush();
235          } finally {
236            writer.close();
237          }
238          return buffer.toByteArray();
239        }
240    
241        /**
242         * Deserialize a new metadata object from a set of bytes.
243         * @param bytes the serialized metadata
244         * @throws IOException
245         */
246        protected Metadata(byte[] bytes) throws IOException {
247          String cipher = null;
248          int bitLength = 0;
249          Date created = null;
250          int versions = 0;
251          String description = null;
252          Map<String, String> attributes = null;
253          JsonReader reader = new JsonReader(new InputStreamReader
254            (new ByteArrayInputStream(bytes)));
255          try {
256            reader.beginObject();
257            while (reader.hasNext()) {
258              String field = reader.nextName();
259              if (CIPHER_FIELD.equals(field)) {
260                cipher = reader.nextString();
261              } else if (BIT_LENGTH_FIELD.equals(field)) {
262                bitLength = reader.nextInt();
263              } else if (CREATED_FIELD.equals(field)) {
264                created = new Date(reader.nextLong());
265              } else if (VERSIONS_FIELD.equals(field)) {
266                versions = reader.nextInt();
267              } else if (DESCRIPTION_FIELD.equals(field)) {
268                description = reader.nextString();
269              } else if (ATTRIBUTES_FIELD.equalsIgnoreCase(field)) {
270                reader.beginObject();
271                attributes = new HashMap<String, String>();
272                while (reader.hasNext()) {
273                  attributes.put(reader.nextName(), reader.nextString());
274                }
275                reader.endObject();
276              }
277            }
278            reader.endObject();
279          } finally {
280            reader.close();
281          }
282          this.cipher = cipher;
283          this.bitLength = bitLength;
284          this.created = created;
285          this.description = description;
286          this.attributes = attributes;
287          this.versions = versions;
288        }
289      }
290    
291      /**
292       * Options when creating key objects.
293       */
294      public static class Options {
295        private String cipher;
296        private int bitLength;
297        private String description;
298        private Map<String, String> attributes;
299    
300        public Options(Configuration conf) {
301          cipher = conf.get(DEFAULT_CIPHER_NAME, DEFAULT_CIPHER);
302          bitLength = conf.getInt(DEFAULT_BITLENGTH_NAME, DEFAULT_BITLENGTH);
303        }
304    
305        public Options setCipher(String cipher) {
306          this.cipher = cipher;
307          return this;
308        }
309    
310        public Options setBitLength(int bitLength) {
311          this.bitLength = bitLength;
312          return this;
313        }
314    
315        public Options setDescription(String description) {
316          this.description = description;
317          return this;
318        }
319    
320        public Options setAttributes(Map<String, String> attributes) {
321          if (attributes != null) {
322            if (attributes.containsKey(null)) {
323              throw new IllegalArgumentException("attributes cannot have a NULL key");
324            }
325            this.attributes = new HashMap<String, String>(attributes);
326          }
327          return this;
328        }
329    
330        public String getCipher() {
331          return cipher;
332        }
333    
334        public int getBitLength() {
335          return bitLength;
336        }
337    
338        public String getDescription() {
339          return description;
340        }
341    
342        @SuppressWarnings("unchecked")
343        public Map<String, String> getAttributes() {
344          return (attributes == null) ? Collections.EMPTY_MAP : attributes;
345        }
346    
347        @Override
348        public String toString() {
349          return "Options{" +
350              "cipher='" + cipher + '\'' +
351              ", bitLength=" + bitLength +
352              ", description='" + description + '\'' +
353              ", attributes=" + attributes +
354              '}';
355        }
356      }
357    
358      /**
359       * Constructor.
360       * 
361       * @param conf configuration for the provider
362       */
363      public KeyProvider(Configuration conf) {
364        this.conf = new Configuration(conf);
365      }
366    
367      /**
368       * Return the provider configuration.
369       * 
370       * @return the provider configuration
371       */
372      public Configuration getConf() {
373        return conf;
374      }
375      
376      /**
377       * A helper function to create an options object.
378       * @param conf the configuration to use
379       * @return a new options object
380       */
381      public static Options options(Configuration conf) {
382        return new Options(conf);
383      }
384    
385      /**
386       * Indicates whether this provider represents a store
387       * that is intended for transient use - such as the UserProvider
388       * is. These providers are generally used to provide access to
389       * keying material rather than for long term storage.
390       * @return true if transient, false otherwise
391       */
392      public boolean isTransient() {
393        return false;
394      }
395    
396      /**
397       * Get the key material for a specific version of the key. This method is used
398       * when decrypting data.
399       * @param versionName the name of a specific version of the key
400       * @return the key material
401       * @throws IOException
402       */
403      public abstract KeyVersion getKeyVersion(String versionName
404                                                ) throws IOException;
405    
406      /**
407       * Get the key names for all keys.
408       * @return the list of key names
409       * @throws IOException
410       */
411      public abstract List<String> getKeys() throws IOException;
412    
413      /**
414       * Get key metadata in bulk.
415       * @param names the names of the keys to get
416       * @throws IOException
417       */
418      public Metadata[] getKeysMetadata(String... names) throws IOException {
419        Metadata[] result = new Metadata[names.length];
420        for (int i=0; i < names.length; ++i) {
421          result[i] = getMetadata(names[i]);
422        }
423        return result;
424      }
425    
426      /**
427       * Get the key material for all versions of a specific key name.
428       * @return the list of key material
429       * @throws IOException
430       */
431      public abstract List<KeyVersion> getKeyVersions(String name) throws IOException;
432    
433      /**
434       * Get the current version of the key, which should be used for encrypting new
435       * data.
436       * @param name the base name of the key
437       * @return the version name of the current version of the key or null if the
438       *    key version doesn't exist
439       * @throws IOException
440       */
441      public KeyVersion getCurrentKey(String name) throws IOException {
442        Metadata meta = getMetadata(name);
443        if (meta == null) {
444          return null;
445        }
446        return getKeyVersion(buildVersionName(name, meta.getVersions() - 1));
447      }
448    
449      /**
450       * Get metadata about the key.
451       * @param name the basename of the key
452       * @return the key's metadata or null if the key doesn't exist
453       * @throws IOException
454       */
455      public abstract Metadata getMetadata(String name) throws IOException;
456    
457      /**
458       * Create a new key. The given key must not already exist.
459       * @param name the base name of the key
460       * @param material the key material for the first version of the key.
461       * @param options the options for the new key.
462       * @return the version name of the first version of the key.
463       * @throws IOException
464       */
465      public abstract KeyVersion createKey(String name, byte[] material,
466                                           Options options) throws IOException;
467    
468      /**
469       * Get the algorithm from the cipher.
470       *
471       * @return the algorithm name
472       */
473      private String getAlgorithm(String cipher) {
474        int slash = cipher.indexOf('/');
475        if (slash == -1) {
476          return cipher;
477        } else {
478          return cipher.substring(0, slash);
479        }
480      }
481    
482      /**
483       * Generates a key material.
484       *
485       * @param size length of the key.
486       * @param algorithm algorithm to use for generating the key.
487       * @return the generated key.
488       * @throws NoSuchAlgorithmException
489       */
490      protected byte[] generateKey(int size, String algorithm)
491          throws NoSuchAlgorithmException {
492        algorithm = getAlgorithm(algorithm);
493        KeyGenerator keyGenerator = KeyGenerator.getInstance(algorithm);
494        keyGenerator.init(size);
495        byte[] key = keyGenerator.generateKey().getEncoded();
496        return key;
497      }
498    
499      /**
500       * Create a new key generating the material for it.
501       * The given key must not already exist.
502       * <p/>
503       * This implementation generates the key material and calls the
504       * {@link #createKey(String, byte[], Options)} method.
505       *
506       * @param name the base name of the key
507       * @param options the options for the new key.
508       * @return the version name of the first version of the key.
509       * @throws IOException
510       * @throws NoSuchAlgorithmException
511       */
512      public KeyVersion createKey(String name, Options options)
513          throws NoSuchAlgorithmException, IOException {
514        byte[] material = generateKey(options.getBitLength(), options.getCipher());
515        return createKey(name, material, options);
516      }
517    
518      /**
519       * Delete the given key.
520       * @param name the name of the key to delete
521       * @throws IOException
522       */
523      public abstract void deleteKey(String name) throws IOException;
524    
525      /**
526       * Roll a new version of the given key.
527       * @param name the basename of the key
528       * @param material the new key material
529       * @return the name of the new version of the key
530       * @throws IOException
531       */
532      public abstract KeyVersion rollNewVersion(String name,
533                                                 byte[] material
534                                                ) throws IOException;
535    
536      /**
537       * Can be used by implementing classes to close any resources
538       * that require closing
539       */
540      public void close() throws IOException {
541        // NOP
542      }
543    
544      /**
545       * Roll a new version of the given key generating the material for it.
546       * <p/>
547       * This implementation generates the key material and calls the
548       * {@link #rollNewVersion(String, byte[])} method.
549       *
550       * @param name the basename of the key
551       * @return the name of the new version of the key
552       * @throws IOException
553       */
554      public KeyVersion rollNewVersion(String name) throws NoSuchAlgorithmException,
555                                                           IOException {
556        Metadata meta = getMetadata(name);
557        byte[] material = generateKey(meta.getBitLength(), meta.getCipher());
558        return rollNewVersion(name, material);
559      }
560    
561      /**
562       * Ensures that any changes to the keys are written to persistent store.
563       * @throws IOException
564       */
565      public abstract void flush() throws IOException;
566    
567      /**
568       * Split the versionName in to a base name. Converts "/aaa/bbb/3" to
569       * "/aaa/bbb".
570       * @param versionName the version name to split
571       * @return the base name of the key
572       * @throws IOException
573       */
574      public static String getBaseName(String versionName) throws IOException {
575        int div = versionName.lastIndexOf('@');
576        if (div == -1) {
577          throw new IOException("No version in key path " + versionName);
578        }
579        return versionName.substring(0, div);
580      }
581    
582      /**
583       * Build a version string from a basename and version number. Converts
584       * "/aaa/bbb" and 3 to "/aaa/bbb@3".
585       * @param name the basename of the key
586       * @param version the version of the key
587       * @return the versionName of the key.
588       */
589      protected static String buildVersionName(String name, int version) {
590        return name + "@" + version;
591      }
592    
593      /**
594       * Find the provider with the given key.
595       * @param providerList the list of providers
596       * @param keyName the key name we are looking for
597       * @return the KeyProvider that has the key
598       */
599      public static KeyProvider findProvider(List<KeyProvider> providerList,
600                                             String keyName) throws IOException {
601        for(KeyProvider provider: providerList) {
602          if (provider.getMetadata(keyName) != null) {
603            return provider;
604          }
605        }
606        throw new IOException("Can't find KeyProvider for key " + keyName);
607      }
608    }