001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.hadoop.crypto.key; 020 021import java.io.ByteArrayInputStream; 022import java.io.ByteArrayOutputStream; 023import java.io.IOException; 024import java.io.InputStreamReader; 025import java.io.OutputStreamWriter; 026import java.security.NoSuchAlgorithmException; 027import java.util.Collections; 028import java.util.Date; 029import java.util.HashMap; 030import java.util.List; 031import java.util.Map; 032 033import com.google.gson.stream.JsonReader; 034import com.google.gson.stream.JsonWriter; 035import org.apache.commons.io.Charsets; 036import org.apache.hadoop.classification.InterfaceAudience; 037import org.apache.hadoop.classification.InterfaceStability; 038import org.apache.hadoop.conf.Configuration; 039 040import javax.crypto.KeyGenerator; 041 042import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCEKS_KEY_SERIALFILTER; 043 044/** 045 * A provider of secret key material for Hadoop applications. Provides an 046 * abstraction to separate key storage from users of encryption. It 047 * is intended to support getting or storing keys in a variety of ways, 048 * including third party bindings. 049 * <P/> 050 * <code>KeyProvider</code> implementations must be thread safe. 051 */ 052@InterfaceAudience.Public 053@InterfaceStability.Unstable 054public abstract class KeyProvider { 055 public static final String DEFAULT_CIPHER_NAME = 056 "hadoop.security.key.default.cipher"; 057 public static final String DEFAULT_CIPHER = "AES/CTR/NoPadding"; 058 public static final String DEFAULT_BITLENGTH_NAME = 059 "hadoop.security.key.default.bitlength"; 060 public static final int DEFAULT_BITLENGTH = 128; 061 public static final String JCEKS_KEY_SERIALFILTER_DEFAULT = 062 "java.lang.Enum;" 063 + "java.security.KeyRep;" 064 + "java.security.KeyRep$Type;" 065 + "javax.crypto.spec.SecretKeySpec;" 066 + "org.apache.hadoop.crypto.key.JavaKeyStoreProvider$KeyMetadata;" 067 + "!*"; 068 public static final String JCEKS_KEY_SERIAL_FILTER = "jceks.key.serialFilter"; 069 070 private final Configuration conf; 071 072 /** 073 * The combination of both the key version name and the key material. 074 */ 075 public static class KeyVersion { 076 private final String name; 077 private final String versionName; 078 private final byte[] material; 079 080 protected KeyVersion(String name, String versionName, 081 byte[] material) { 082 this.name = name; 083 this.versionName = versionName; 084 this.material = material; 085 } 086 087 public String getName() { 088 return name; 089 } 090 091 public String getVersionName() { 092 return versionName; 093 } 094 095 public byte[] getMaterial() { 096 return material; 097 } 098 099 public String toString() { 100 StringBuilder buf = new StringBuilder(); 101 buf.append("key("); 102 buf.append(versionName); 103 buf.append(")="); 104 if (material == null) { 105 buf.append("null"); 106 } else { 107 for(byte b: material) { 108 buf.append(' '); 109 int right = b & 0xff; 110 if (right < 0x10) { 111 buf.append('0'); 112 } 113 buf.append(Integer.toHexString(right)); 114 } 115 } 116 return buf.toString(); 117 } 118 } 119 120 /** 121 * Key metadata that is associated with the key. 122 */ 123 public static class Metadata { 124 private final static String CIPHER_FIELD = "cipher"; 125 private final static String BIT_LENGTH_FIELD = "bitLength"; 126 private final static String CREATED_FIELD = "created"; 127 private final static String DESCRIPTION_FIELD = "description"; 128 private final static String VERSIONS_FIELD = "versions"; 129 private final static String ATTRIBUTES_FIELD = "attributes"; 130 131 private final String cipher; 132 private final int bitLength; 133 private final String description; 134 private final Date created; 135 private int versions; 136 private Map<String, String> attributes; 137 138 protected Metadata(String cipher, int bitLength, String description, 139 Map<String, String> attributes, Date created, int versions) { 140 this.cipher = cipher; 141 this.bitLength = bitLength; 142 this.description = description; 143 this.attributes = (attributes == null || attributes.isEmpty()) 144 ? null : attributes; 145 this.created = created; 146 this.versions = versions; 147 } 148 149 public String toString() { 150 final StringBuilder metaSB = new StringBuilder(); 151 metaSB.append("cipher: ").append(cipher).append(", "); 152 metaSB.append("length: ").append(bitLength).append(", "); 153 metaSB.append("description: ").append(description).append(", "); 154 metaSB.append("created: ").append(created).append(", "); 155 metaSB.append("version: ").append(versions).append(", "); 156 metaSB.append("attributes: "); 157 if ((attributes != null) && !attributes.isEmpty()) { 158 for (Map.Entry<String, String> attribute : attributes.entrySet()) { 159 metaSB.append("["); 160 metaSB.append(attribute.getKey()); 161 metaSB.append("="); 162 metaSB.append(attribute.getValue()); 163 metaSB.append("], "); 164 } 165 metaSB.deleteCharAt(metaSB.length() - 2); // remove last ', ' 166 } else { 167 metaSB.append("null"); 168 } 169 return metaSB.toString(); 170 } 171 172 public String getDescription() { 173 return description; 174 } 175 176 public Date getCreated() { 177 return created; 178 } 179 180 public String getCipher() { 181 return cipher; 182 } 183 184 @SuppressWarnings("unchecked") 185 public Map<String, String> getAttributes() { 186 return (attributes == null) ? Collections.EMPTY_MAP : attributes; 187 } 188 189 /** 190 * Get the algorithm from the cipher. 191 * @return the algorithm name 192 */ 193 public String getAlgorithm() { 194 int slash = cipher.indexOf('/'); 195 if (slash == - 1) { 196 return cipher; 197 } else { 198 return cipher.substring(0, slash); 199 } 200 } 201 202 public int getBitLength() { 203 return bitLength; 204 } 205 206 public int getVersions() { 207 return versions; 208 } 209 210 protected int addVersion() { 211 return versions++; 212 } 213 214 /** 215 * Serialize the metadata to a set of bytes. 216 * @return the serialized bytes 217 * @throws IOException 218 */ 219 protected byte[] serialize() throws IOException { 220 ByteArrayOutputStream buffer = new ByteArrayOutputStream(); 221 JsonWriter writer = new JsonWriter( 222 new OutputStreamWriter(buffer, Charsets.UTF_8)); 223 try { 224 writer.beginObject(); 225 if (cipher != null) { 226 writer.name(CIPHER_FIELD).value(cipher); 227 } 228 if (bitLength != 0) { 229 writer.name(BIT_LENGTH_FIELD).value(bitLength); 230 } 231 if (created != null) { 232 writer.name(CREATED_FIELD).value(created.getTime()); 233 } 234 if (description != null) { 235 writer.name(DESCRIPTION_FIELD).value(description); 236 } 237 if (attributes != null && attributes.size() > 0) { 238 writer.name(ATTRIBUTES_FIELD).beginObject(); 239 for (Map.Entry<String, String> attribute : attributes.entrySet()) { 240 writer.name(attribute.getKey()).value(attribute.getValue()); 241 } 242 writer.endObject(); 243 } 244 writer.name(VERSIONS_FIELD).value(versions); 245 writer.endObject(); 246 writer.flush(); 247 } finally { 248 writer.close(); 249 } 250 return buffer.toByteArray(); 251 } 252 253 /** 254 * Deserialize a new metadata object from a set of bytes. 255 * @param bytes the serialized metadata 256 * @throws IOException 257 */ 258 protected Metadata(byte[] bytes) throws IOException { 259 String cipher = null; 260 int bitLength = 0; 261 Date created = null; 262 int versions = 0; 263 String description = null; 264 Map<String, String> attributes = null; 265 JsonReader reader = new JsonReader(new InputStreamReader 266 (new ByteArrayInputStream(bytes), Charsets.UTF_8)); 267 try { 268 reader.beginObject(); 269 while (reader.hasNext()) { 270 String field = reader.nextName(); 271 if (CIPHER_FIELD.equals(field)) { 272 cipher = reader.nextString(); 273 } else if (BIT_LENGTH_FIELD.equals(field)) { 274 bitLength = reader.nextInt(); 275 } else if (CREATED_FIELD.equals(field)) { 276 created = new Date(reader.nextLong()); 277 } else if (VERSIONS_FIELD.equals(field)) { 278 versions = reader.nextInt(); 279 } else if (DESCRIPTION_FIELD.equals(field)) { 280 description = reader.nextString(); 281 } else if (ATTRIBUTES_FIELD.equalsIgnoreCase(field)) { 282 reader.beginObject(); 283 attributes = new HashMap<String, String>(); 284 while (reader.hasNext()) { 285 attributes.put(reader.nextName(), reader.nextString()); 286 } 287 reader.endObject(); 288 } 289 } 290 reader.endObject(); 291 } finally { 292 reader.close(); 293 } 294 this.cipher = cipher; 295 this.bitLength = bitLength; 296 this.created = created; 297 this.description = description; 298 this.attributes = attributes; 299 this.versions = versions; 300 } 301 } 302 303 /** 304 * Options when creating key objects. 305 */ 306 public static class Options { 307 private String cipher; 308 private int bitLength; 309 private String description; 310 private Map<String, String> attributes; 311 312 public Options(Configuration conf) { 313 cipher = conf.get(DEFAULT_CIPHER_NAME, DEFAULT_CIPHER); 314 bitLength = conf.getInt(DEFAULT_BITLENGTH_NAME, DEFAULT_BITLENGTH); 315 } 316 317 public Options setCipher(String cipher) { 318 this.cipher = cipher; 319 return this; 320 } 321 322 public Options setBitLength(int bitLength) { 323 this.bitLength = bitLength; 324 return this; 325 } 326 327 public Options setDescription(String description) { 328 this.description = description; 329 return this; 330 } 331 332 public Options setAttributes(Map<String, String> attributes) { 333 if (attributes != null) { 334 if (attributes.containsKey(null)) { 335 throw new IllegalArgumentException("attributes cannot have a NULL key"); 336 } 337 this.attributes = new HashMap<String, String>(attributes); 338 } 339 return this; 340 } 341 342 public String getCipher() { 343 return cipher; 344 } 345 346 public int getBitLength() { 347 return bitLength; 348 } 349 350 public String getDescription() { 351 return description; 352 } 353 354 @SuppressWarnings("unchecked") 355 public Map<String, String> getAttributes() { 356 return (attributes == null) ? Collections.EMPTY_MAP : attributes; 357 } 358 359 @Override 360 public String toString() { 361 return "Options{" + 362 "cipher='" + cipher + '\'' + 363 ", bitLength=" + bitLength + 364 ", description='" + description + '\'' + 365 ", attributes=" + attributes + 366 '}'; 367 } 368 } 369 370 /** 371 * Constructor. 372 * 373 * @param conf configuration for the provider 374 */ 375 public KeyProvider(Configuration conf) { 376 this.conf = new Configuration(conf); 377 // Added for HADOOP-15473. Configured serialFilter property fixes 378 // java.security.UnrecoverableKeyException in JDK 8u171. 379 if(System.getProperty(JCEKS_KEY_SERIAL_FILTER) == null) { 380 String serialFilter = 381 conf.get(HADOOP_SECURITY_CRYPTO_JCEKS_KEY_SERIALFILTER, 382 JCEKS_KEY_SERIALFILTER_DEFAULT); 383 System.setProperty(JCEKS_KEY_SERIAL_FILTER, serialFilter); 384 } 385 } 386 387 /** 388 * Return the provider configuration. 389 * 390 * @return the provider configuration 391 */ 392 public Configuration getConf() { 393 return conf; 394 } 395 396 /** 397 * A helper function to create an options object. 398 * @param conf the configuration to use 399 * @return a new options object 400 */ 401 public static Options options(Configuration conf) { 402 return new Options(conf); 403 } 404 405 /** 406 * Indicates whether this provider represents a store 407 * that is intended for transient use - such as the UserProvider 408 * is. These providers are generally used to provide access to 409 * keying material rather than for long term storage. 410 * @return true if transient, false otherwise 411 */ 412 public boolean isTransient() { 413 return false; 414 } 415 416 /** 417 * Get the key material for a specific version of the key. This method is used 418 * when decrypting data. 419 * @param versionName the name of a specific version of the key 420 * @return the key material 421 * @throws IOException 422 */ 423 public abstract KeyVersion getKeyVersion(String versionName 424 ) throws IOException; 425 426 /** 427 * Get the key names for all keys. 428 * @return the list of key names 429 * @throws IOException 430 */ 431 public abstract List<String> getKeys() throws IOException; 432 433 /** 434 * Get key metadata in bulk. 435 * @param names the names of the keys to get 436 * @throws IOException 437 */ 438 public Metadata[] getKeysMetadata(String... names) throws IOException { 439 Metadata[] result = new Metadata[names.length]; 440 for (int i=0; i < names.length; ++i) { 441 result[i] = getMetadata(names[i]); 442 } 443 return result; 444 } 445 446 /** 447 * Get the key material for all versions of a specific key name. 448 * @return the list of key material 449 * @throws IOException 450 */ 451 public abstract List<KeyVersion> getKeyVersions(String name) throws IOException; 452 453 /** 454 * Get the current version of the key, which should be used for encrypting new 455 * data. 456 * @param name the base name of the key 457 * @return the version name of the current version of the key or null if the 458 * key version doesn't exist 459 * @throws IOException 460 */ 461 public KeyVersion getCurrentKey(String name) throws IOException { 462 Metadata meta = getMetadata(name); 463 if (meta == null) { 464 return null; 465 } 466 return getKeyVersion(buildVersionName(name, meta.getVersions() - 1)); 467 } 468 469 /** 470 * Get metadata about the key. 471 * @param name the basename of the key 472 * @return the key's metadata or null if the key doesn't exist 473 * @throws IOException 474 */ 475 public abstract Metadata getMetadata(String name) throws IOException; 476 477 /** 478 * Create a new key. The given key must not already exist. 479 * @param name the base name of the key 480 * @param material the key material for the first version of the key. 481 * @param options the options for the new key. 482 * @return the version name of the first version of the key. 483 * @throws IOException 484 */ 485 public abstract KeyVersion createKey(String name, byte[] material, 486 Options options) throws IOException; 487 488 /** 489 * Get the algorithm from the cipher. 490 * 491 * @return the algorithm name 492 */ 493 private String getAlgorithm(String cipher) { 494 int slash = cipher.indexOf('/'); 495 if (slash == -1) { 496 return cipher; 497 } else { 498 return cipher.substring(0, slash); 499 } 500 } 501 502 /** 503 * Generates a key material. 504 * 505 * @param size length of the key. 506 * @param algorithm algorithm to use for generating the key. 507 * @return the generated key. 508 * @throws NoSuchAlgorithmException 509 */ 510 protected byte[] generateKey(int size, String algorithm) 511 throws NoSuchAlgorithmException { 512 algorithm = getAlgorithm(algorithm); 513 KeyGenerator keyGenerator = KeyGenerator.getInstance(algorithm); 514 keyGenerator.init(size); 515 byte[] key = keyGenerator.generateKey().getEncoded(); 516 return key; 517 } 518 519 /** 520 * Create a new key generating the material for it. 521 * The given key must not already exist. 522 * <p/> 523 * This implementation generates the key material and calls the 524 * {@link #createKey(String, byte[], Options)} method. 525 * 526 * @param name the base name of the key 527 * @param options the options for the new key. 528 * @return the version name of the first version of the key. 529 * @throws IOException 530 * @throws NoSuchAlgorithmException 531 */ 532 public KeyVersion createKey(String name, Options options) 533 throws NoSuchAlgorithmException, IOException { 534 byte[] material = generateKey(options.getBitLength(), options.getCipher()); 535 return createKey(name, material, options); 536 } 537 538 /** 539 * Delete the given key. 540 * @param name the name of the key to delete 541 * @throws IOException 542 */ 543 public abstract void deleteKey(String name) throws IOException; 544 545 /** 546 * Roll a new version of the given key. 547 * @param name the basename of the key 548 * @param material the new key material 549 * @return the name of the new version of the key 550 * @throws IOException 551 */ 552 public abstract KeyVersion rollNewVersion(String name, 553 byte[] material 554 ) throws IOException; 555 556 /** 557 * Can be used by implementing classes to close any resources 558 * that require closing 559 */ 560 public void close() throws IOException { 561 // NOP 562 } 563 564 /** 565 * Roll a new version of the given key generating the material for it. 566 * <p/> 567 * This implementation generates the key material and calls the 568 * {@link #rollNewVersion(String, byte[])} method. 569 * 570 * @param name the basename of the key 571 * @return the name of the new version of the key 572 * @throws IOException 573 */ 574 public KeyVersion rollNewVersion(String name) throws NoSuchAlgorithmException, 575 IOException { 576 Metadata meta = getMetadata(name); 577 byte[] material = generateKey(meta.getBitLength(), meta.getCipher()); 578 return rollNewVersion(name, material); 579 } 580 581 /** 582 * Ensures that any changes to the keys are written to persistent store. 583 * @throws IOException 584 */ 585 public abstract void flush() throws IOException; 586 587 /** 588 * Split the versionName in to a base name. Converts "/aaa/bbb/3" to 589 * "/aaa/bbb". 590 * @param versionName the version name to split 591 * @return the base name of the key 592 * @throws IOException 593 */ 594 public static String getBaseName(String versionName) throws IOException { 595 int div = versionName.lastIndexOf('@'); 596 if (div == -1) { 597 throw new IOException("No version in key path " + versionName); 598 } 599 return versionName.substring(0, div); 600 } 601 602 /** 603 * Build a version string from a basename and version number. Converts 604 * "/aaa/bbb" and 3 to "/aaa/bbb@3". 605 * @param name the basename of the key 606 * @param version the version of the key 607 * @return the versionName of the key. 608 */ 609 protected static String buildVersionName(String name, int version) { 610 return name + "@" + version; 611 } 612 613 /** 614 * Find the provider with the given key. 615 * @param providerList the list of providers 616 * @param keyName the key name we are looking for 617 * @return the KeyProvider that has the key 618 */ 619 public static KeyProvider findProvider(List<KeyProvider> providerList, 620 String keyName) throws IOException { 621 for(KeyProvider provider: providerList) { 622 if (provider.getMetadata(keyName) != null) { 623 return provider; 624 } 625 } 626 throw new IOException("Can't find KeyProvider for key " + keyName); 627 } 628}