001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.hadoop.crypto.key; 020 021import java.io.ByteArrayInputStream; 022import java.io.ByteArrayOutputStream; 023import java.io.IOException; 024import java.io.InputStreamReader; 025import java.io.OutputStreamWriter; 026import java.security.NoSuchAlgorithmException; 027import java.util.Collections; 028import java.util.Date; 029import java.util.HashMap; 030import java.util.List; 031import java.util.Map; 032 033import com.google.gson.stream.JsonReader; 034import com.google.gson.stream.JsonWriter; 035import org.apache.commons.io.Charsets; 036import org.apache.hadoop.classification.InterfaceAudience; 037import org.apache.hadoop.classification.InterfaceStability; 038import org.apache.hadoop.conf.Configuration; 039 040import javax.crypto.KeyGenerator; 041 042/** 043 * A provider of secret key material for Hadoop applications. Provides an 044 * abstraction to separate key storage from users of encryption. It 045 * is intended to support getting or storing keys in a variety of ways, 046 * including third party bindings. 047 * <P/> 048 * <code>KeyProvider</code> implementations must be thread safe. 049 */ 050@InterfaceAudience.Public 051@InterfaceStability.Unstable 052public abstract class KeyProvider { 053 public static final String DEFAULT_CIPHER_NAME = 054 "hadoop.security.key.default.cipher"; 055 public static final String DEFAULT_CIPHER = "AES/CTR/NoPadding"; 056 public static final String DEFAULT_BITLENGTH_NAME = 057 "hadoop.security.key.default.bitlength"; 058 public static final int DEFAULT_BITLENGTH = 128; 059 060 private final Configuration conf; 061 062 /** 063 * The combination of both the key version name and the key material. 064 */ 065 public static class KeyVersion { 066 private final String name; 067 private final String versionName; 068 private final byte[] material; 069 070 protected KeyVersion(String name, String versionName, 071 byte[] material) { 072 this.name = name; 073 this.versionName = versionName; 074 this.material = material; 075 } 076 077 public String getName() { 078 return name; 079 } 080 081 public String getVersionName() { 082 return versionName; 083 } 084 085 public byte[] getMaterial() { 086 return material; 087 } 088 089 public String toString() { 090 StringBuilder buf = new StringBuilder(); 091 buf.append("key("); 092 buf.append(versionName); 093 buf.append(")="); 094 if (material == null) { 095 buf.append("null"); 096 } else { 097 for(byte b: material) { 098 buf.append(' '); 099 int right = b & 0xff; 100 if (right < 0x10) { 101 buf.append('0'); 102 } 103 buf.append(Integer.toHexString(right)); 104 } 105 } 106 return buf.toString(); 107 } 108 } 109 110 /** 111 * Key metadata that is associated with the key. 112 */ 113 public static class Metadata { 114 private final static String CIPHER_FIELD = "cipher"; 115 private final static String BIT_LENGTH_FIELD = "bitLength"; 116 private final static String CREATED_FIELD = "created"; 117 private final static String DESCRIPTION_FIELD = "description"; 118 private final static String VERSIONS_FIELD = "versions"; 119 private final static String ATTRIBUTES_FIELD = "attributes"; 120 121 private final String cipher; 122 private final int bitLength; 123 private final String description; 124 private final Date created; 125 private int versions; 126 private Map<String, String> attributes; 127 128 protected Metadata(String cipher, int bitLength, String description, 129 Map<String, String> attributes, Date created, int versions) { 130 this.cipher = cipher; 131 this.bitLength = bitLength; 132 this.description = description; 133 this.attributes = (attributes == null || attributes.isEmpty()) 134 ? null : attributes; 135 this.created = created; 136 this.versions = versions; 137 } 138 139 public String toString() { 140 final StringBuilder metaSB = new StringBuilder(); 141 metaSB.append("cipher: ").append(cipher).append(", "); 142 metaSB.append("length: ").append(bitLength).append(", "); 143 metaSB.append("description: ").append(description).append(", "); 144 metaSB.append("created: ").append(created).append(", "); 145 metaSB.append("version: ").append(versions).append(", "); 146 metaSB.append("attributes: "); 147 if ((attributes != null) && !attributes.isEmpty()) { 148 for (Map.Entry<String, String> attribute : attributes.entrySet()) { 149 metaSB.append("["); 150 metaSB.append(attribute.getKey()); 151 metaSB.append("="); 152 metaSB.append(attribute.getValue()); 153 metaSB.append("], "); 154 } 155 metaSB.deleteCharAt(metaSB.length() - 2); // remove last ', ' 156 } else { 157 metaSB.append("null"); 158 } 159 return metaSB.toString(); 160 } 161 162 public String getDescription() { 163 return description; 164 } 165 166 public Date getCreated() { 167 return created; 168 } 169 170 public String getCipher() { 171 return cipher; 172 } 173 174 @SuppressWarnings("unchecked") 175 public Map<String, String> getAttributes() { 176 return (attributes == null) ? Collections.EMPTY_MAP : attributes; 177 } 178 179 /** 180 * Get the algorithm from the cipher. 181 * @return the algorithm name 182 */ 183 public String getAlgorithm() { 184 int slash = cipher.indexOf('/'); 185 if (slash == - 1) { 186 return cipher; 187 } else { 188 return cipher.substring(0, slash); 189 } 190 } 191 192 public int getBitLength() { 193 return bitLength; 194 } 195 196 public int getVersions() { 197 return versions; 198 } 199 200 protected int addVersion() { 201 return versions++; 202 } 203 204 /** 205 * Serialize the metadata to a set of bytes. 206 * @return the serialized bytes 207 * @throws IOException 208 */ 209 protected byte[] serialize() throws IOException { 210 ByteArrayOutputStream buffer = new ByteArrayOutputStream(); 211 JsonWriter writer = new JsonWriter( 212 new OutputStreamWriter(buffer, Charsets.UTF_8)); 213 try { 214 writer.beginObject(); 215 if (cipher != null) { 216 writer.name(CIPHER_FIELD).value(cipher); 217 } 218 if (bitLength != 0) { 219 writer.name(BIT_LENGTH_FIELD).value(bitLength); 220 } 221 if (created != null) { 222 writer.name(CREATED_FIELD).value(created.getTime()); 223 } 224 if (description != null) { 225 writer.name(DESCRIPTION_FIELD).value(description); 226 } 227 if (attributes != null && attributes.size() > 0) { 228 writer.name(ATTRIBUTES_FIELD).beginObject(); 229 for (Map.Entry<String, String> attribute : attributes.entrySet()) { 230 writer.name(attribute.getKey()).value(attribute.getValue()); 231 } 232 writer.endObject(); 233 } 234 writer.name(VERSIONS_FIELD).value(versions); 235 writer.endObject(); 236 writer.flush(); 237 } finally { 238 writer.close(); 239 } 240 return buffer.toByteArray(); 241 } 242 243 /** 244 * Deserialize a new metadata object from a set of bytes. 245 * @param bytes the serialized metadata 246 * @throws IOException 247 */ 248 protected Metadata(byte[] bytes) throws IOException { 249 String cipher = null; 250 int bitLength = 0; 251 Date created = null; 252 int versions = 0; 253 String description = null; 254 Map<String, String> attributes = null; 255 JsonReader reader = new JsonReader(new InputStreamReader 256 (new ByteArrayInputStream(bytes), Charsets.UTF_8)); 257 try { 258 reader.beginObject(); 259 while (reader.hasNext()) { 260 String field = reader.nextName(); 261 if (CIPHER_FIELD.equals(field)) { 262 cipher = reader.nextString(); 263 } else if (BIT_LENGTH_FIELD.equals(field)) { 264 bitLength = reader.nextInt(); 265 } else if (CREATED_FIELD.equals(field)) { 266 created = new Date(reader.nextLong()); 267 } else if (VERSIONS_FIELD.equals(field)) { 268 versions = reader.nextInt(); 269 } else if (DESCRIPTION_FIELD.equals(field)) { 270 description = reader.nextString(); 271 } else if (ATTRIBUTES_FIELD.equalsIgnoreCase(field)) { 272 reader.beginObject(); 273 attributes = new HashMap<String, String>(); 274 while (reader.hasNext()) { 275 attributes.put(reader.nextName(), reader.nextString()); 276 } 277 reader.endObject(); 278 } 279 } 280 reader.endObject(); 281 } finally { 282 reader.close(); 283 } 284 this.cipher = cipher; 285 this.bitLength = bitLength; 286 this.created = created; 287 this.description = description; 288 this.attributes = attributes; 289 this.versions = versions; 290 } 291 } 292 293 /** 294 * Options when creating key objects. 295 */ 296 public static class Options { 297 private String cipher; 298 private int bitLength; 299 private String description; 300 private Map<String, String> attributes; 301 302 public Options(Configuration conf) { 303 cipher = conf.get(DEFAULT_CIPHER_NAME, DEFAULT_CIPHER); 304 bitLength = conf.getInt(DEFAULT_BITLENGTH_NAME, DEFAULT_BITLENGTH); 305 } 306 307 public Options setCipher(String cipher) { 308 this.cipher = cipher; 309 return this; 310 } 311 312 public Options setBitLength(int bitLength) { 313 this.bitLength = bitLength; 314 return this; 315 } 316 317 public Options setDescription(String description) { 318 this.description = description; 319 return this; 320 } 321 322 public Options setAttributes(Map<String, String> attributes) { 323 if (attributes != null) { 324 if (attributes.containsKey(null)) { 325 throw new IllegalArgumentException("attributes cannot have a NULL key"); 326 } 327 this.attributes = new HashMap<String, String>(attributes); 328 } 329 return this; 330 } 331 332 public String getCipher() { 333 return cipher; 334 } 335 336 public int getBitLength() { 337 return bitLength; 338 } 339 340 public String getDescription() { 341 return description; 342 } 343 344 @SuppressWarnings("unchecked") 345 public Map<String, String> getAttributes() { 346 return (attributes == null) ? Collections.EMPTY_MAP : attributes; 347 } 348 349 @Override 350 public String toString() { 351 return "Options{" + 352 "cipher='" + cipher + '\'' + 353 ", bitLength=" + bitLength + 354 ", description='" + description + '\'' + 355 ", attributes=" + attributes + 356 '}'; 357 } 358 } 359 360 /** 361 * Constructor. 362 * 363 * @param conf configuration for the provider 364 */ 365 public KeyProvider(Configuration conf) { 366 this.conf = new Configuration(conf); 367 } 368 369 /** 370 * Return the provider configuration. 371 * 372 * @return the provider configuration 373 */ 374 public Configuration getConf() { 375 return conf; 376 } 377 378 /** 379 * A helper function to create an options object. 380 * @param conf the configuration to use 381 * @return a new options object 382 */ 383 public static Options options(Configuration conf) { 384 return new Options(conf); 385 } 386 387 /** 388 * Indicates whether this provider represents a store 389 * that is intended for transient use - such as the UserProvider 390 * is. These providers are generally used to provide access to 391 * keying material rather than for long term storage. 392 * @return true if transient, false otherwise 393 */ 394 public boolean isTransient() { 395 return false; 396 } 397 398 /** 399 * Get the key material for a specific version of the key. This method is used 400 * when decrypting data. 401 * @param versionName the name of a specific version of the key 402 * @return the key material 403 * @throws IOException 404 */ 405 public abstract KeyVersion getKeyVersion(String versionName 406 ) throws IOException; 407 408 /** 409 * Get the key names for all keys. 410 * @return the list of key names 411 * @throws IOException 412 */ 413 public abstract List<String> getKeys() throws IOException; 414 415 /** 416 * Get key metadata in bulk. 417 * @param names the names of the keys to get 418 * @throws IOException 419 */ 420 public Metadata[] getKeysMetadata(String... names) throws IOException { 421 Metadata[] result = new Metadata[names.length]; 422 for (int i=0; i < names.length; ++i) { 423 result[i] = getMetadata(names[i]); 424 } 425 return result; 426 } 427 428 /** 429 * Get the key material for all versions of a specific key name. 430 * @return the list of key material 431 * @throws IOException 432 */ 433 public abstract List<KeyVersion> getKeyVersions(String name) throws IOException; 434 435 /** 436 * Get the current version of the key, which should be used for encrypting new 437 * data. 438 * @param name the base name of the key 439 * @return the version name of the current version of the key or null if the 440 * key version doesn't exist 441 * @throws IOException 442 */ 443 public KeyVersion getCurrentKey(String name) throws IOException { 444 Metadata meta = getMetadata(name); 445 if (meta == null) { 446 return null; 447 } 448 return getKeyVersion(buildVersionName(name, meta.getVersions() - 1)); 449 } 450 451 /** 452 * Get metadata about the key. 453 * @param name the basename of the key 454 * @return the key's metadata or null if the key doesn't exist 455 * @throws IOException 456 */ 457 public abstract Metadata getMetadata(String name) throws IOException; 458 459 /** 460 * Create a new key. The given key must not already exist. 461 * @param name the base name of the key 462 * @param material the key material for the first version of the key. 463 * @param options the options for the new key. 464 * @return the version name of the first version of the key. 465 * @throws IOException 466 */ 467 public abstract KeyVersion createKey(String name, byte[] material, 468 Options options) throws IOException; 469 470 /** 471 * Get the algorithm from the cipher. 472 * 473 * @return the algorithm name 474 */ 475 private String getAlgorithm(String cipher) { 476 int slash = cipher.indexOf('/'); 477 if (slash == -1) { 478 return cipher; 479 } else { 480 return cipher.substring(0, slash); 481 } 482 } 483 484 /** 485 * Generates a key material. 486 * 487 * @param size length of the key. 488 * @param algorithm algorithm to use for generating the key. 489 * @return the generated key. 490 * @throws NoSuchAlgorithmException 491 */ 492 protected byte[] generateKey(int size, String algorithm) 493 throws NoSuchAlgorithmException { 494 algorithm = getAlgorithm(algorithm); 495 KeyGenerator keyGenerator = KeyGenerator.getInstance(algorithm); 496 keyGenerator.init(size); 497 byte[] key = keyGenerator.generateKey().getEncoded(); 498 return key; 499 } 500 501 /** 502 * Create a new key generating the material for it. 503 * The given key must not already exist. 504 * <p/> 505 * This implementation generates the key material and calls the 506 * {@link #createKey(String, byte[], Options)} method. 507 * 508 * @param name the base name of the key 509 * @param options the options for the new key. 510 * @return the version name of the first version of the key. 511 * @throws IOException 512 * @throws NoSuchAlgorithmException 513 */ 514 public KeyVersion createKey(String name, Options options) 515 throws NoSuchAlgorithmException, IOException { 516 byte[] material = generateKey(options.getBitLength(), options.getCipher()); 517 return createKey(name, material, options); 518 } 519 520 /** 521 * Delete the given key. 522 * @param name the name of the key to delete 523 * @throws IOException 524 */ 525 public abstract void deleteKey(String name) throws IOException; 526 527 /** 528 * Roll a new version of the given key. 529 * @param name the basename of the key 530 * @param material the new key material 531 * @return the name of the new version of the key 532 * @throws IOException 533 */ 534 public abstract KeyVersion rollNewVersion(String name, 535 byte[] material 536 ) throws IOException; 537 538 /** 539 * Can be used by implementing classes to close any resources 540 * that require closing 541 */ 542 public void close() throws IOException { 543 // NOP 544 } 545 546 /** 547 * Roll a new version of the given key generating the material for it. 548 * <p/> 549 * This implementation generates the key material and calls the 550 * {@link #rollNewVersion(String, byte[])} method. 551 * 552 * @param name the basename of the key 553 * @return the name of the new version of the key 554 * @throws IOException 555 */ 556 public KeyVersion rollNewVersion(String name) throws NoSuchAlgorithmException, 557 IOException { 558 Metadata meta = getMetadata(name); 559 byte[] material = generateKey(meta.getBitLength(), meta.getCipher()); 560 return rollNewVersion(name, material); 561 } 562 563 /** 564 * Ensures that any changes to the keys are written to persistent store. 565 * @throws IOException 566 */ 567 public abstract void flush() throws IOException; 568 569 /** 570 * Split the versionName in to a base name. Converts "/aaa/bbb/3" to 571 * "/aaa/bbb". 572 * @param versionName the version name to split 573 * @return the base name of the key 574 * @throws IOException 575 */ 576 public static String getBaseName(String versionName) throws IOException { 577 int div = versionName.lastIndexOf('@'); 578 if (div == -1) { 579 throw new IOException("No version in key path " + versionName); 580 } 581 return versionName.substring(0, div); 582 } 583 584 /** 585 * Build a version string from a basename and version number. Converts 586 * "/aaa/bbb" and 3 to "/aaa/bbb@3". 587 * @param name the basename of the key 588 * @param version the version of the key 589 * @return the versionName of the key. 590 */ 591 protected static String buildVersionName(String name, int version) { 592 return name + "@" + version; 593 } 594 595 /** 596 * Find the provider with the given key. 597 * @param providerList the list of providers 598 * @param keyName the key name we are looking for 599 * @return the KeyProvider that has the key 600 */ 601 public static KeyProvider findProvider(List<KeyProvider> providerList, 602 String keyName) throws IOException { 603 for(KeyProvider provider: providerList) { 604 if (provider.getMetadata(keyName) != null) { 605 return provider; 606 } 607 } 608 throw new IOException("Can't find KeyProvider for key " + keyName); 609 } 610}