001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    
019    package org.apache.hadoop.conf;
020    
021    import java.io.BufferedInputStream;
022    import java.io.DataInput;
023    import java.io.DataOutput;
024    import java.io.File;
025    import java.io.FileInputStream;
026    import java.io.IOException;
027    import java.io.InputStream;
028    import java.io.InputStreamReader;
029    import java.io.OutputStream;
030    import java.io.OutputStreamWriter;
031    import java.io.Reader;
032    import java.io.Writer;
033    import java.lang.ref.WeakReference;
034    import java.net.InetSocketAddress;
035    import java.net.URL;
036    import java.util.ArrayList;
037    import java.util.Arrays;
038    import java.util.Collection;
039    import java.util.Collections;
040    import java.util.Enumeration;
041    import java.util.HashMap;
042    import java.util.HashSet;
043    import java.util.Iterator;
044    import java.util.LinkedList;
045    import java.util.List;
046    import java.util.ListIterator;
047    import java.util.Map;
048    import java.util.Map.Entry;
049    import java.util.Properties;
050    import java.util.Set;
051    import java.util.StringTokenizer;
052    import java.util.WeakHashMap;
053    import java.util.concurrent.CopyOnWriteArrayList;
054    import java.util.regex.Matcher;
055    import java.util.regex.Pattern;
056    import java.util.regex.PatternSyntaxException;
057    import java.util.concurrent.TimeUnit;
058    import java.util.concurrent.atomic.AtomicBoolean;
059    import java.util.concurrent.atomic.AtomicReference;
060    
061    import javax.xml.parsers.DocumentBuilder;
062    import javax.xml.parsers.DocumentBuilderFactory;
063    import javax.xml.parsers.ParserConfigurationException;
064    import javax.xml.transform.Transformer;
065    import javax.xml.transform.TransformerException;
066    import javax.xml.transform.TransformerFactory;
067    import javax.xml.transform.dom.DOMSource;
068    import javax.xml.transform.stream.StreamResult;
069    
070    import org.apache.commons.collections.map.UnmodifiableMap;
071    import org.apache.commons.logging.Log;
072    import org.apache.commons.logging.LogFactory;
073    import org.apache.hadoop.classification.InterfaceAudience;
074    import org.apache.hadoop.classification.InterfaceStability;
075    import org.apache.hadoop.fs.FileSystem;
076    import org.apache.hadoop.fs.Path;
077    import org.apache.hadoop.fs.CommonConfigurationKeys;
078    import org.apache.hadoop.io.Writable;
079    import org.apache.hadoop.io.WritableUtils;
080    import org.apache.hadoop.net.NetUtils;
081    import org.apache.hadoop.security.alias.CredentialProvider;
082    import org.apache.hadoop.security.alias.CredentialProvider.CredentialEntry;
083    import org.apache.hadoop.security.alias.CredentialProviderFactory;
084    import org.apache.hadoop.util.ReflectionUtils;
085    import org.apache.hadoop.util.StringInterner;
086    import org.apache.hadoop.util.StringUtils;
087    import org.codehaus.jackson.JsonFactory;
088    import org.codehaus.jackson.JsonGenerator;
089    import org.w3c.dom.DOMException;
090    import org.w3c.dom.Document;
091    import org.w3c.dom.Element;
092    import org.w3c.dom.Node;
093    import org.w3c.dom.NodeList;
094    import org.w3c.dom.Text;
095    import org.xml.sax.SAXException;
096    
097    import com.google.common.base.Preconditions;
098    
099    /** 
100     * Provides access to configuration parameters.
101     *
102     * <h4 id="Resources">Resources</h4>
103     *
104     * <p>Configurations are specified by resources. A resource contains a set of
105     * name/value pairs as XML data. Each resource is named by either a 
106     * <code>String</code> or by a {@link Path}. If named by a <code>String</code>, 
107     * then the classpath is examined for a file with that name.  If named by a 
108     * <code>Path</code>, then the local filesystem is examined directly, without 
109     * referring to the classpath.
110     *
111     * <p>Unless explicitly turned off, Hadoop by default specifies two 
112     * resources, loaded in-order from the classpath: <ol>
113     * <li><tt>
114     * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
115     * core-default.xml</a></tt>: Read-only defaults for hadoop.</li>
116     * <li><tt>core-site.xml</tt>: Site-specific configuration for a given hadoop
117     * installation.</li>
118     * </ol>
119     * Applications may add additional resources, which are loaded
120     * subsequent to these resources in the order they are added.
121     * 
122     * <h4 id="FinalParams">Final Parameters</h4>
123     *
124     * <p>Configuration parameters may be declared <i>final</i>. 
125     * Once a resource declares a value final, no subsequently-loaded 
126     * resource can alter that value.  
127     * For example, one might define a final parameter with:
128     * <tt><pre>
129     *  &lt;property&gt;
130     *    &lt;name&gt;dfs.hosts.include&lt;/name&gt;
131     *    &lt;value&gt;/etc/hadoop/conf/hosts.include&lt;/value&gt;
132     *    <b>&lt;final&gt;true&lt;/final&gt;</b>
133     *  &lt;/property&gt;</pre></tt>
134     *
135     * Administrators typically define parameters as final in 
136     * <tt>core-site.xml</tt> for values that user applications may not alter.
137     *
138     * <h4 id="VariableExpansion">Variable Expansion</h4>
139     *
140     * <p>Value strings are first processed for <i>variable expansion</i>. The
141     * available properties are:<ol>
142     * <li>Other properties defined in this Configuration; and, if a name is
143     * undefined here,</li>
144     * <li>Properties in {@link System#getProperties()}.</li>
145     * </ol>
146     *
147     * <p>For example, if a configuration resource contains the following property
148     * definitions: 
149     * <tt><pre>
150     *  &lt;property&gt;
151     *    &lt;name&gt;basedir&lt;/name&gt;
152     *    &lt;value&gt;/user/${<i>user.name</i>}&lt;/value&gt;
153     *  &lt;/property&gt;
154     *  
155     *  &lt;property&gt;
156     *    &lt;name&gt;tempdir&lt;/name&gt;
157     *    &lt;value&gt;${<i>basedir</i>}/tmp&lt;/value&gt;
158     *  &lt;/property&gt;</pre></tt>
159     *
160     * When <tt>conf.get("tempdir")</tt> is called, then <tt>${<i>basedir</i>}</tt>
161     * will be resolved to another property in this Configuration, while
162     * <tt>${<i>user.name</i>}</tt> would then ordinarily be resolved to the value
163     * of the System property with that name.
164     * By default, warnings will be given to any deprecated configuration 
165     * parameters and these are suppressible by configuring
166     * <tt>log4j.logger.org.apache.hadoop.conf.Configuration.deprecation</tt> in
167     * log4j.properties file.
168     */
169    @InterfaceAudience.Public
170    @InterfaceStability.Stable
171    public class Configuration implements Iterable<Map.Entry<String,String>>,
172                                          Writable {
173      private static final Log LOG =
174        LogFactory.getLog(Configuration.class);
175    
176      private static final Log LOG_DEPRECATION =
177        LogFactory.getLog("org.apache.hadoop.conf.Configuration.deprecation");
178    
179      private boolean quietmode = true;
180      
181      private static class Resource {
182        private final Object resource;
183        private final String name;
184        
185        public Resource(Object resource) {
186          this(resource, resource.toString());
187        }
188        
189        public Resource(Object resource, String name) {
190          this.resource = resource;
191          this.name = name;
192        }
193        
194        public String getName(){
195          return name;
196        }
197        
198        public Object getResource() {
199          return resource;
200        }
201        
202        @Override
203        public String toString() {
204          return name;
205        }
206      }
207      
208      /**
209       * List of configuration resources.
210       */
211      private ArrayList<Resource> resources = new ArrayList<Resource>();
212      
213      /**
214       * The value reported as the setting resource when a key is set
215       * by code rather than a file resource by dumpConfiguration.
216       */
217      static final String UNKNOWN_RESOURCE = "Unknown";
218    
219    
220      /**
221       * List of configuration parameters marked <b>final</b>. 
222       */
223      private Set<String> finalParameters = new HashSet<String>();
224      
225      private boolean loadDefaults = true;
226      
227      /**
228       * Configuration objects
229       */
230      private static final WeakHashMap<Configuration,Object> REGISTRY = 
231        new WeakHashMap<Configuration,Object>();
232      
233      /**
234       * List of default Resources. Resources are loaded in the order of the list 
235       * entries
236       */
237      private static final CopyOnWriteArrayList<String> defaultResources =
238        new CopyOnWriteArrayList<String>();
239    
240      private static final Map<ClassLoader, Map<String, WeakReference<Class<?>>>>
241        CACHE_CLASSES = new WeakHashMap<ClassLoader, Map<String, WeakReference<Class<?>>>>();
242    
243      /**
244       * Sentinel value to store negative cache results in {@link #CACHE_CLASSES}.
245       */
246      private static final Class<?> NEGATIVE_CACHE_SENTINEL =
247        NegativeCacheSentinel.class;
248    
249      /**
250       * Stores the mapping of key to the resource which modifies or loads 
251       * the key most recently
252       */
253      private HashMap<String, String[]> updatingResource;
254     
255      /**
256       * Class to keep the information about the keys which replace the deprecated
257       * ones.
258       * 
259       * This class stores the new keys which replace the deprecated keys and also
260       * gives a provision to have a custom message for each of the deprecated key
261       * that is being replaced. It also provides method to get the appropriate
262       * warning message which can be logged whenever the deprecated key is used.
263       */
264      private static class DeprecatedKeyInfo {
265        private final String[] newKeys;
266        private final String customMessage;
267        private final AtomicBoolean accessed = new AtomicBoolean(false);
268    
269        DeprecatedKeyInfo(String[] newKeys, String customMessage) {
270          this.newKeys = newKeys;
271          this.customMessage = customMessage;
272        }
273    
274        /**
275         * Method to provide the warning message. It gives the custom message if
276         * non-null, and default message otherwise.
277         * @param key the associated deprecated key.
278         * @return message that is to be logged when a deprecated key is used.
279         */
280        private final String getWarningMessage(String key) {
281          String warningMessage;
282          if(customMessage == null) {
283            StringBuilder message = new StringBuilder(key);
284            String deprecatedKeySuffix = " is deprecated. Instead, use ";
285            message.append(deprecatedKeySuffix);
286            for (int i = 0; i < newKeys.length; i++) {
287              message.append(newKeys[i]);
288              if(i != newKeys.length-1) {
289                message.append(", ");
290              }
291            }
292            warningMessage = message.toString();
293          }
294          else {
295            warningMessage = customMessage;
296          }
297          return warningMessage;
298        }
299    
300        boolean getAndSetAccessed() {
301          return accessed.getAndSet(true);
302        }
303    
304        public void clearAccessed() {
305          accessed.set(false);
306        }
307      }
308      
309      /**
310       * A pending addition to the global set of deprecated keys.
311       */
312      public static class DeprecationDelta {
313        private final String key;
314        private final String[] newKeys;
315        private final String customMessage;
316    
317        DeprecationDelta(String key, String[] newKeys, String customMessage) {
318          Preconditions.checkNotNull(key);
319          Preconditions.checkNotNull(newKeys);
320          Preconditions.checkArgument(newKeys.length > 0);
321          this.key = key;
322          this.newKeys = newKeys;
323          this.customMessage = customMessage;
324        }
325    
326        public DeprecationDelta(String key, String newKey, String customMessage) {
327          this(key, new String[] { newKey }, customMessage);
328        }
329    
330        public DeprecationDelta(String key, String newKey) {
331          this(key, new String[] { newKey }, null);
332        }
333    
334        public String getKey() {
335          return key;
336        }
337    
338        public String[] getNewKeys() {
339          return newKeys;
340        }
341    
342        public String getCustomMessage() {
343          return customMessage;
344        }
345      }
346    
347      /**
348       * The set of all keys which are deprecated.
349       *
350       * DeprecationContext objects are immutable.
351       */
352      private static class DeprecationContext {
353        /**
354         * Stores the deprecated keys, the new keys which replace the deprecated keys
355         * and custom message(if any provided).
356         */
357        private final Map<String, DeprecatedKeyInfo> deprecatedKeyMap;
358    
359        /**
360         * Stores a mapping from superseding keys to the keys which they deprecate.
361         */
362        private final Map<String, String> reverseDeprecatedKeyMap;
363    
364        /**
365         * Create a new DeprecationContext by copying a previous DeprecationContext
366         * and adding some deltas.
367         *
368         * @param other   The previous deprecation context to copy, or null to start
369         *                from nothing.
370         * @param deltas  The deltas to apply.
371         */
372        @SuppressWarnings("unchecked")
373        DeprecationContext(DeprecationContext other, DeprecationDelta[] deltas) {
374          HashMap<String, DeprecatedKeyInfo> newDeprecatedKeyMap = 
375            new HashMap<String, DeprecatedKeyInfo>();
376          HashMap<String, String> newReverseDeprecatedKeyMap =
377            new HashMap<String, String>();
378          if (other != null) {
379            for (Entry<String, DeprecatedKeyInfo> entry :
380                other.deprecatedKeyMap.entrySet()) {
381              newDeprecatedKeyMap.put(entry.getKey(), entry.getValue());
382            }
383            for (Entry<String, String> entry :
384                other.reverseDeprecatedKeyMap.entrySet()) {
385              newReverseDeprecatedKeyMap.put(entry.getKey(), entry.getValue());
386            }
387          }
388          for (DeprecationDelta delta : deltas) {
389            if (!newDeprecatedKeyMap.containsKey(delta.getKey())) {
390              DeprecatedKeyInfo newKeyInfo =
391                new DeprecatedKeyInfo(delta.getNewKeys(), delta.getCustomMessage());
392              newDeprecatedKeyMap.put(delta.key, newKeyInfo);
393              for (String newKey : delta.getNewKeys()) {
394                newReverseDeprecatedKeyMap.put(newKey, delta.key);
395              }
396            }
397          }
398          this.deprecatedKeyMap =
399            UnmodifiableMap.decorate(newDeprecatedKeyMap);
400          this.reverseDeprecatedKeyMap =
401            UnmodifiableMap.decorate(newReverseDeprecatedKeyMap);
402        }
403    
404        Map<String, DeprecatedKeyInfo> getDeprecatedKeyMap() {
405          return deprecatedKeyMap;
406        }
407    
408        Map<String, String> getReverseDeprecatedKeyMap() {
409          return reverseDeprecatedKeyMap;
410        }
411      }
412      
413      private static DeprecationDelta[] defaultDeprecations = 
414        new DeprecationDelta[] {
415          new DeprecationDelta("topology.script.file.name", 
416            CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY),
417          new DeprecationDelta("topology.script.number.args", 
418            CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY),
419          new DeprecationDelta("hadoop.configured.node.mapping", 
420            CommonConfigurationKeys.NET_TOPOLOGY_CONFIGURED_NODE_MAPPING_KEY),
421          new DeprecationDelta("topology.node.switch.mapping.impl", 
422            CommonConfigurationKeys.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY),
423          new DeprecationDelta("dfs.df.interval", 
424            CommonConfigurationKeys.FS_DF_INTERVAL_KEY),
425          new DeprecationDelta("hadoop.native.lib", 
426            CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY),
427          new DeprecationDelta("fs.default.name", 
428            CommonConfigurationKeys.FS_DEFAULT_NAME_KEY),
429          new DeprecationDelta("dfs.umaskmode",
430            CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY),
431          new DeprecationDelta("dfs.nfs.exports.allowed.hosts",
432              CommonConfigurationKeys.NFS_EXPORTS_ALLOWED_HOSTS_KEY)
433        };
434    
435      /**
436       * The global DeprecationContext.
437       */
438      private static AtomicReference<DeprecationContext> deprecationContext =
439          new AtomicReference<DeprecationContext>(
440              new DeprecationContext(null, defaultDeprecations));
441    
442      /**
443       * Adds a set of deprecated keys to the global deprecations.
444       *
445       * This method is lockless.  It works by means of creating a new
446       * DeprecationContext based on the old one, and then atomically swapping in
447       * the new context.  If someone else updated the context in between us reading
448       * the old context and swapping in the new one, we try again until we win the
449       * race.
450       *
451       * @param deltas   The deprecations to add.
452       */
453      public static void addDeprecations(DeprecationDelta[] deltas) {
454        DeprecationContext prev, next;
455        do {
456          prev = deprecationContext.get();
457          next = new DeprecationContext(prev, deltas);
458        } while (!deprecationContext.compareAndSet(prev, next));
459      }
460    
461      /**
462       * Adds the deprecated key to the global deprecation map.
463       * It does not override any existing entries in the deprecation map.
464       * This is to be used only by the developers in order to add deprecation of
465       * keys, and attempts to call this method after loading resources once,
466       * would lead to <tt>UnsupportedOperationException</tt>
467       * 
468       * If a key is deprecated in favor of multiple keys, they are all treated as 
469       * aliases of each other, and setting any one of them resets all the others 
470       * to the new value.
471       *
472       * If you have multiple deprecation entries to add, it is more efficient to
473       * use #addDeprecations(DeprecationDelta[] deltas) instead.
474       * 
475       * @param key
476       * @param newKeys
477       * @param customMessage
478       * @deprecated use {@link #addDeprecation(String key, String newKey,
479          String customMessage)} instead
480       */
481      @Deprecated
482      public static void addDeprecation(String key, String[] newKeys,
483          String customMessage) {
484        addDeprecations(new DeprecationDelta[] {
485          new DeprecationDelta(key, newKeys, customMessage)
486        });
487      }
488    
489      /**
490       * Adds the deprecated key to the global deprecation map.
491       * It does not override any existing entries in the deprecation map.
492       * This is to be used only by the developers in order to add deprecation of
493       * keys, and attempts to call this method after loading resources once,
494       * would lead to <tt>UnsupportedOperationException</tt>
495       * 
496       * If you have multiple deprecation entries to add, it is more efficient to
497       * use #addDeprecations(DeprecationDelta[] deltas) instead.
498       *
499       * @param key
500       * @param newKey
501       * @param customMessage
502       */
503      public static void addDeprecation(String key, String newKey,
504                  String customMessage) {
505              addDeprecation(key, new String[] {newKey}, customMessage);
506      }
507    
508      /**
509       * Adds the deprecated key to the global deprecation map when no custom
510       * message is provided.
511       * It does not override any existing entries in the deprecation map.
512       * This is to be used only by the developers in order to add deprecation of
513       * keys, and attempts to call this method after loading resources once,
514       * would lead to <tt>UnsupportedOperationException</tt>
515       * 
516       * If a key is deprecated in favor of multiple keys, they are all treated as 
517       * aliases of each other, and setting any one of them resets all the others 
518       * to the new value.
519       * 
520       * If you have multiple deprecation entries to add, it is more efficient to
521       * use #addDeprecations(DeprecationDelta[] deltas) instead.
522       *
523       * @param key Key that is to be deprecated
524       * @param newKeys list of keys that take up the values of deprecated key
525       * @deprecated use {@link #addDeprecation(String key, String newKey)} instead
526       */
527      @Deprecated
528      public static void addDeprecation(String key, String[] newKeys) {
529        addDeprecation(key, newKeys, null);
530      }
531      
532      /**
533       * Adds the deprecated key to the global deprecation map when no custom
534       * message is provided.
535       * It does not override any existing entries in the deprecation map.
536       * This is to be used only by the developers in order to add deprecation of
537       * keys, and attempts to call this method after loading resources once,
538       * would lead to <tt>UnsupportedOperationException</tt>
539       * 
540       * If you have multiple deprecation entries to add, it is more efficient to
541       * use #addDeprecations(DeprecationDelta[] deltas) instead.
542       *
543       * @param key Key that is to be deprecated
544       * @param newKey key that takes up the value of deprecated key
545       */
546      public static void addDeprecation(String key, String newKey) {
547        addDeprecation(key, new String[] {newKey}, null);
548      }
549      
550      /**
551       * checks whether the given <code>key</code> is deprecated.
552       * 
553       * @param key the parameter which is to be checked for deprecation
554       * @return <code>true</code> if the key is deprecated and 
555       *         <code>false</code> otherwise.
556       */
557      public static boolean isDeprecated(String key) {
558        return deprecationContext.get().getDeprecatedKeyMap().containsKey(key);
559      }
560    
561      /**
562       * Sets all deprecated properties that are not currently set but have a
563       * corresponding new property that is set. Useful for iterating the
564       * properties when all deprecated properties for currently set properties
565       * need to be present.
566       */
567      public void setDeprecatedProperties() {
568        DeprecationContext deprecations = deprecationContext.get();
569        Properties props = getProps();
570        Properties overlay = getOverlay();
571        for (Map.Entry<String, DeprecatedKeyInfo> entry :
572            deprecations.getDeprecatedKeyMap().entrySet()) {
573          String depKey = entry.getKey();
574          if (!overlay.contains(depKey)) {
575            for (String newKey : entry.getValue().newKeys) {
576              String val = overlay.getProperty(newKey);
577              if (val != null) {
578                props.setProperty(depKey, val);
579                overlay.setProperty(depKey, val);
580                break;
581              }
582            }
583          }
584        }
585      }
586    
587      /**
588       * Checks for the presence of the property <code>name</code> in the
589       * deprecation map. Returns the first of the list of new keys if present
590       * in the deprecation map or the <code>name</code> itself. If the property
591       * is not presently set but the property map contains an entry for the
592       * deprecated key, the value of the deprecated key is set as the value for
593       * the provided property name.
594       *
595       * @param name the property name
596       * @return the first property in the list of properties mapping
597       *         the <code>name</code> or the <code>name</code> itself.
598       */
599      private String[] handleDeprecation(DeprecationContext deprecations,
600          String name) {
601        if (null != name) {
602          name = name.trim();
603        }
604        ArrayList<String > names = new ArrayList<String>();
605            if (isDeprecated(name)) {
606          DeprecatedKeyInfo keyInfo = deprecations.getDeprecatedKeyMap().get(name);
607          warnOnceIfDeprecated(deprecations, name);
608          for (String newKey : keyInfo.newKeys) {
609            if(newKey != null) {
610              names.add(newKey);
611            }
612          }
613        }
614        if(names.size() == 0) {
615            names.add(name);
616        }
617        for(String n : names) {
618              String deprecatedKey = deprecations.getReverseDeprecatedKeyMap().get(n);
619              if (deprecatedKey != null && !getOverlay().containsKey(n) &&
620                  getOverlay().containsKey(deprecatedKey)) {
621                getProps().setProperty(n, getOverlay().getProperty(deprecatedKey));
622                getOverlay().setProperty(n, getOverlay().getProperty(deprecatedKey));
623              }
624        }
625        return names.toArray(new String[names.size()]);
626      }
627     
628      private void handleDeprecation() {
629        LOG.debug("Handling deprecation for all properties in config...");
630        DeprecationContext deprecations = deprecationContext.get();
631        Set<Object> keys = new HashSet<Object>();
632        keys.addAll(getProps().keySet());
633        for (Object item: keys) {
634          LOG.debug("Handling deprecation for " + (String)item);
635          handleDeprecation(deprecations, (String)item);
636        }
637      }
638     
639      static{
640        //print deprecation warning if hadoop-site.xml is found in classpath
641        ClassLoader cL = Thread.currentThread().getContextClassLoader();
642        if (cL == null) {
643          cL = Configuration.class.getClassLoader();
644        }
645        if(cL.getResource("hadoop-site.xml")!=null) {
646          LOG.warn("DEPRECATED: hadoop-site.xml found in the classpath. " +
647              "Usage of hadoop-site.xml is deprecated. Instead use core-site.xml, "
648              + "mapred-site.xml and hdfs-site.xml to override properties of " +
649              "core-default.xml, mapred-default.xml and hdfs-default.xml " +
650              "respectively");
651        }
652        addDefaultResource("core-default.xml");
653        addDefaultResource("core-site.xml");
654      }
655      
656      private Properties properties;
657      private Properties overlay;
658      private ClassLoader classLoader;
659      {
660        classLoader = Thread.currentThread().getContextClassLoader();
661        if (classLoader == null) {
662          classLoader = Configuration.class.getClassLoader();
663        }
664      }
665      
666      /** A new configuration. */
667      public Configuration() {
668        this(true);
669      }
670    
671      /** A new configuration where the behavior of reading from the default 
672       * resources can be turned off.
673       * 
674       * If the parameter {@code loadDefaults} is false, the new instance
675       * will not load resources from the default files. 
676       * @param loadDefaults specifies whether to load from the default files
677       */
678      public Configuration(boolean loadDefaults) {
679        this.loadDefaults = loadDefaults;
680        updatingResource = new HashMap<String, String[]>();
681        synchronized(Configuration.class) {
682          REGISTRY.put(this, null);
683        }
684      }
685      
686      /** 
687       * A new configuration with the same settings cloned from another.
688       * 
689       * @param other the configuration from which to clone settings.
690       */
691      @SuppressWarnings("unchecked")
692      public Configuration(Configuration other) {
693       this.resources = (ArrayList<Resource>) other.resources.clone();
694       synchronized(other) {
695         if (other.properties != null) {
696           this.properties = (Properties)other.properties.clone();
697         }
698    
699         if (other.overlay!=null) {
700           this.overlay = (Properties)other.overlay.clone();
701         }
702    
703         this.updatingResource = new HashMap<String, String[]>(other.updatingResource);
704         this.finalParameters = new HashSet<String>(other.finalParameters);
705       }
706       
707        synchronized(Configuration.class) {
708          REGISTRY.put(this, null);
709        }
710        this.classLoader = other.classLoader;
711        this.loadDefaults = other.loadDefaults;
712        setQuietMode(other.getQuietMode());
713      }
714      
715      /**
716       * Add a default resource. Resources are loaded in the order of the resources 
717       * added.
718       * @param name file name. File should be present in the classpath.
719       */
720      public static synchronized void addDefaultResource(String name) {
721        if(!defaultResources.contains(name)) {
722          defaultResources.add(name);
723          for(Configuration conf : REGISTRY.keySet()) {
724            if(conf.loadDefaults) {
725              conf.reloadConfiguration();
726            }
727          }
728        }
729      }
730    
731      /**
732       * Add a configuration resource. 
733       * 
734       * The properties of this resource will override properties of previously 
735       * added resources, unless they were marked <a href="#Final">final</a>. 
736       * 
737       * @param name resource to be added, the classpath is examined for a file 
738       *             with that name.
739       */
740      public void addResource(String name) {
741        addResourceObject(new Resource(name));
742      }
743    
744      /**
745       * Add a configuration resource. 
746       * 
747       * The properties of this resource will override properties of previously 
748       * added resources, unless they were marked <a href="#Final">final</a>. 
749       * 
750       * @param url url of the resource to be added, the local filesystem is 
751       *            examined directly to find the resource, without referring to 
752       *            the classpath.
753       */
754      public void addResource(URL url) {
755        addResourceObject(new Resource(url));
756      }
757    
758      /**
759       * Add a configuration resource. 
760       * 
761       * The properties of this resource will override properties of previously 
762       * added resources, unless they were marked <a href="#Final">final</a>. 
763       * 
764       * @param file file-path of resource to be added, the local filesystem is
765       *             examined directly to find the resource, without referring to 
766       *             the classpath.
767       */
768      public void addResource(Path file) {
769        addResourceObject(new Resource(file));
770      }
771    
772      /**
773       * Add a configuration resource. 
774       * 
775       * The properties of this resource will override properties of previously 
776       * added resources, unless they were marked <a href="#Final">final</a>. 
777       * 
778       * WARNING: The contents of the InputStream will be cached, by this method. 
779       * So use this sparingly because it does increase the memory consumption.
780       * 
781       * @param in InputStream to deserialize the object from. In will be read from
782       * when a get or set is called next.  After it is read the stream will be
783       * closed. 
784       */
785      public void addResource(InputStream in) {
786        addResourceObject(new Resource(in));
787      }
788    
789      /**
790       * Add a configuration resource. 
791       * 
792       * The properties of this resource will override properties of previously 
793       * added resources, unless they were marked <a href="#Final">final</a>. 
794       * 
795       * @param in InputStream to deserialize the object from.
796       * @param name the name of the resource because InputStream.toString is not
797       * very descriptive some times.  
798       */
799      public void addResource(InputStream in, String name) {
800        addResourceObject(new Resource(in, name));
801      }
802      
803      /**
804       * Add a configuration resource.
805       *
806       * The properties of this resource will override properties of previously
807       * added resources, unless they were marked <a href="#Final">final</a>.
808       *
809       * @param conf Configuration object from which to load properties
810       */
811      public void addResource(Configuration conf) {
812        addResourceObject(new Resource(conf.getProps()));
813      }
814    
815      
816      
817      /**
818       * Reload configuration from previously added resources.
819       *
820       * This method will clear all the configuration read from the added 
821       * resources, and final parameters. This will make the resources to 
822       * be read again before accessing the values. Values that are added
823       * via set methods will overlay values read from the resources.
824       */
825      public synchronized void reloadConfiguration() {
826        properties = null;                            // trigger reload
827        finalParameters.clear();                      // clear site-limits
828      }
829      
830      private synchronized void addResourceObject(Resource resource) {
831        resources.add(resource);                      // add to resources
832        reloadConfiguration();
833      }
834      
835      private static final Pattern VAR_PATTERN =
836          Pattern.compile("\\$\\{[^\\}\\$\u0020]+\\}");
837    
838      private static final int MAX_SUBST = 20;
839    
840      private String substituteVars(String expr) {
841        if (expr == null) {
842          return null;
843        }
844        Matcher match = VAR_PATTERN.matcher("");
845        String eval = expr;
846        for(int s=0; s<MAX_SUBST; s++) {
847          match.reset(eval);
848          if (!match.find()) {
849            return eval;
850          }
851          String var = match.group();
852          var = var.substring(2, var.length()-1); // remove ${ .. }
853          String val = null;
854          try {
855            val = System.getProperty(var);
856          } catch(SecurityException se) {
857            LOG.warn("Unexpected SecurityException in Configuration", se);
858          }
859          if (val == null) {
860            val = getRaw(var);
861          }
862          if (val == null) {
863            return eval; // return literal ${var}: var is unbound
864          }
865          // substitute
866          eval = eval.substring(0, match.start())+val+eval.substring(match.end());
867        }
868        throw new IllegalStateException("Variable substitution depth too large: " 
869                                        + MAX_SUBST + " " + expr);
870      }
871      
872      /**
873       * Get the value of the <code>name</code> property, <code>null</code> if
874       * no such property exists. If the key is deprecated, it returns the value of
875       * the first key which replaces the deprecated key and is not null.
876       * 
877       * Values are processed for <a href="#VariableExpansion">variable expansion</a> 
878       * before being returned. 
879       * 
880       * @param name the property name, will be trimmed before get value.
881       * @return the value of the <code>name</code> or its replacing property, 
882       *         or null if no such property exists.
883       */
884      public String get(String name) {
885        String[] names = handleDeprecation(deprecationContext.get(), name);
886        String result = null;
887        for(String n : names) {
888          result = substituteVars(getProps().getProperty(n));
889        }
890        return result;
891      }
892      
893      /**
894       * Get the value of the <code>name</code> property as a trimmed <code>String</code>, 
895       * <code>null</code> if no such property exists. 
896       * If the key is deprecated, it returns the value of
897       * the first key which replaces the deprecated key and is not null
898       * 
899       * Values are processed for <a href="#VariableExpansion">variable expansion</a> 
900       * before being returned. 
901       * 
902       * @param name the property name.
903       * @return the value of the <code>name</code> or its replacing property, 
904       *         or null if no such property exists.
905       */
906      public String getTrimmed(String name) {
907        String value = get(name);
908        
909        if (null == value) {
910          return null;
911        } else {
912          return value.trim();
913        }
914      }
915      
916      /**
917       * Get the value of the <code>name</code> property as a trimmed <code>String</code>, 
918       * <code>defaultValue</code> if no such property exists. 
919       * See @{Configuration#getTrimmed} for more details.
920       * 
921       * @param name          the property name.
922       * @param defaultValue  the property default value.
923       * @return              the value of the <code>name</code> or defaultValue
924       *                      if it is not set.
925       */
926      public String getTrimmed(String name, String defaultValue) {
927        String ret = getTrimmed(name);
928        return ret == null ? defaultValue : ret;
929      }
930    
931      /**
932       * Get the value of the <code>name</code> property, without doing
933       * <a href="#VariableExpansion">variable expansion</a>.If the key is 
934       * deprecated, it returns the value of the first key which replaces 
935       * the deprecated key and is not null.
936       * 
937       * @param name the property name.
938       * @return the value of the <code>name</code> property or 
939       *         its replacing property and null if no such property exists.
940       */
941      public String getRaw(String name) {
942        String[] names = handleDeprecation(deprecationContext.get(), name);
943        String result = null;
944        for(String n : names) {
945          result = getProps().getProperty(n);
946        }
947        return result;
948      }
949    
950      /**
951       * Returns alternative names (non-deprecated keys or previously-set deprecated keys)
952       * for a given non-deprecated key.
953       * If the given key is deprecated, return null.
954       *
955       * @param name property name.
956       * @return alternative names.
957       */
958      private String[] getAlternativeNames(String name) {
959        String altNames[] = null;
960        DeprecatedKeyInfo keyInfo = null;
961        DeprecationContext cur = deprecationContext.get();
962        String depKey = cur.getReverseDeprecatedKeyMap().get(name);
963        if(depKey != null) {
964          keyInfo = cur.getDeprecatedKeyMap().get(depKey);
965          if(keyInfo.newKeys.length > 0) {
966            if(getProps().containsKey(depKey)) {
967              //if deprecated key is previously set explicitly
968              List<String> list = new ArrayList<String>();
969              list.addAll(Arrays.asList(keyInfo.newKeys));
970              list.add(depKey);
971              altNames = list.toArray(new String[list.size()]);
972            }
973            else {
974              altNames = keyInfo.newKeys;
975            }
976          }
977        }
978        return altNames;
979      }
980    
981      /** 
982       * Set the <code>value</code> of the <code>name</code> property. If 
983       * <code>name</code> is deprecated or there is a deprecated name associated to it,
984       * it sets the value to both names. Name will be trimmed before put into
985       * configuration.
986       * 
987       * @param name property name.
988       * @param value property value.
989       */
990      public void set(String name, String value) {
991        set(name, value, null);
992      }
993      
994      /** 
995       * Set the <code>value</code> of the <code>name</code> property. If 
996       * <code>name</code> is deprecated, it also sets the <code>value</code> to
997       * the keys that replace the deprecated key. Name will be trimmed before put
998       * into configuration.
999       *
1000       * @param name property name.
1001       * @param value property value.
1002       * @param source the place that this configuration value came from 
1003       * (For debugging).
1004       * @throws IllegalArgumentException when the value or name is null.
1005       */
1006      public void set(String name, String value, String source) {
1007        Preconditions.checkArgument(
1008            name != null,
1009            "Property name must not be null");
1010        Preconditions.checkArgument(
1011            value != null,
1012            "The value of property " + name + " must not be null");
1013        name = name.trim();
1014        DeprecationContext deprecations = deprecationContext.get();
1015        if (deprecations.getDeprecatedKeyMap().isEmpty()) {
1016          getProps();
1017        }
1018        getOverlay().setProperty(name, value);
1019        getProps().setProperty(name, value);
1020        String newSource = (source == null ? "programatically" : source);
1021    
1022        if (!isDeprecated(name)) {
1023          updatingResource.put(name, new String[] {newSource});
1024          String[] altNames = getAlternativeNames(name);
1025          if(altNames != null) {
1026            for(String n: altNames) {
1027              if(!n.equals(name)) {
1028                getOverlay().setProperty(n, value);
1029                getProps().setProperty(n, value);
1030                updatingResource.put(n, new String[] {newSource});
1031              }
1032            }
1033          }
1034        }
1035        else {
1036          String[] names = handleDeprecation(deprecationContext.get(), name);
1037          String altSource = "because " + name + " is deprecated";
1038          for(String n : names) {
1039            getOverlay().setProperty(n, value);
1040            getProps().setProperty(n, value);
1041            updatingResource.put(n, new String[] {altSource});
1042          }
1043        }
1044      }
1045    
1046      private void warnOnceIfDeprecated(DeprecationContext deprecations, String name) {
1047        DeprecatedKeyInfo keyInfo = deprecations.getDeprecatedKeyMap().get(name);
1048        if (keyInfo != null && !keyInfo.getAndSetAccessed()) {
1049          LOG_DEPRECATION.info(keyInfo.getWarningMessage(name));
1050        }
1051      }
1052    
1053      /**
1054       * Unset a previously set property.
1055       */
1056      public synchronized void unset(String name) {
1057        String[] names = null;
1058        if (!isDeprecated(name)) {
1059          names = getAlternativeNames(name);
1060          if(names == null) {
1061              names = new String[]{name};
1062          }
1063        }
1064        else {
1065          names = handleDeprecation(deprecationContext.get(), name);
1066        }
1067    
1068        for(String n: names) {
1069          getOverlay().remove(n);
1070          getProps().remove(n);
1071        }
1072      }
1073    
1074      /**
1075       * Sets a property if it is currently unset.
1076       * @param name the property name
1077       * @param value the new value
1078       */
1079      public synchronized void setIfUnset(String name, String value) {
1080        if (get(name) == null) {
1081          set(name, value);
1082        }
1083      }
1084      
1085      private synchronized Properties getOverlay() {
1086        if (overlay==null){
1087          overlay=new Properties();
1088        }
1089        return overlay;
1090      }
1091    
1092      /** 
1093       * Get the value of the <code>name</code>. If the key is deprecated,
1094       * it returns the value of the first key which replaces the deprecated key
1095       * and is not null.
1096       * If no such property exists,
1097       * then <code>defaultValue</code> is returned.
1098       * 
1099       * @param name property name, will be trimmed before get value.
1100       * @param defaultValue default value.
1101       * @return property value, or <code>defaultValue</code> if the property 
1102       *         doesn't exist.                    
1103       */
1104      public String get(String name, String defaultValue) {
1105        String[] names = handleDeprecation(deprecationContext.get(), name);
1106        String result = null;
1107        for(String n : names) {
1108          result = substituteVars(getProps().getProperty(n, defaultValue));
1109        }
1110        return result;
1111      }
1112    
1113      /** 
1114       * Get the value of the <code>name</code> property as an <code>int</code>.
1115       *   
1116       * If no such property exists, the provided default value is returned,
1117       * or if the specified value is not a valid <code>int</code>,
1118       * then an error is thrown.
1119       * 
1120       * @param name property name.
1121       * @param defaultValue default value.
1122       * @throws NumberFormatException when the value is invalid
1123       * @return property value as an <code>int</code>, 
1124       *         or <code>defaultValue</code>. 
1125       */
1126      public int getInt(String name, int defaultValue) {
1127        String valueString = getTrimmed(name);
1128        if (valueString == null)
1129          return defaultValue;
1130        String hexString = getHexDigits(valueString);
1131        if (hexString != null) {
1132          return Integer.parseInt(hexString, 16);
1133        }
1134        return Integer.parseInt(valueString);
1135      }
1136      
1137      /**
1138       * Get the value of the <code>name</code> property as a set of comma-delimited
1139       * <code>int</code> values.
1140       * 
1141       * If no such property exists, an empty array is returned.
1142       * 
1143       * @param name property name
1144       * @return property value interpreted as an array of comma-delimited
1145       *         <code>int</code> values
1146       */
1147      public int[] getInts(String name) {
1148        String[] strings = getTrimmedStrings(name);
1149        int[] ints = new int[strings.length];
1150        for (int i = 0; i < strings.length; i++) {
1151          ints[i] = Integer.parseInt(strings[i]);
1152        }
1153        return ints;
1154      }
1155    
1156      /** 
1157       * Set the value of the <code>name</code> property to an <code>int</code>.
1158       * 
1159       * @param name property name.
1160       * @param value <code>int</code> value of the property.
1161       */
1162      public void setInt(String name, int value) {
1163        set(name, Integer.toString(value));
1164      }
1165    
1166    
1167      /** 
1168       * Get the value of the <code>name</code> property as a <code>long</code>.  
1169       * If no such property exists, the provided default value is returned,
1170       * or if the specified value is not a valid <code>long</code>,
1171       * then an error is thrown.
1172       * 
1173       * @param name property name.
1174       * @param defaultValue default value.
1175       * @throws NumberFormatException when the value is invalid
1176       * @return property value as a <code>long</code>, 
1177       *         or <code>defaultValue</code>. 
1178       */
1179      public long getLong(String name, long defaultValue) {
1180        String valueString = getTrimmed(name);
1181        if (valueString == null)
1182          return defaultValue;
1183        String hexString = getHexDigits(valueString);
1184        if (hexString != null) {
1185          return Long.parseLong(hexString, 16);
1186        }
1187        return Long.parseLong(valueString);
1188      }
1189    
1190      /**
1191       * Get the value of the <code>name</code> property as a <code>long</code> or
1192       * human readable format. If no such property exists, the provided default
1193       * value is returned, or if the specified value is not a valid
1194       * <code>long</code> or human readable format, then an error is thrown. You
1195       * can use the following suffix (case insensitive): k(kilo), m(mega), g(giga),
1196       * t(tera), p(peta), e(exa)
1197       *
1198       * @param name property name.
1199       * @param defaultValue default value.
1200       * @throws NumberFormatException when the value is invalid
1201       * @return property value as a <code>long</code>,
1202       *         or <code>defaultValue</code>.
1203       */
1204      public long getLongBytes(String name, long defaultValue) {
1205        String valueString = getTrimmed(name);
1206        if (valueString == null)
1207          return defaultValue;
1208        return StringUtils.TraditionalBinaryPrefix.string2long(valueString);
1209      }
1210    
1211      private String getHexDigits(String value) {
1212        boolean negative = false;
1213        String str = value;
1214        String hexString = null;
1215        if (value.startsWith("-")) {
1216          negative = true;
1217          str = value.substring(1);
1218        }
1219        if (str.startsWith("0x") || str.startsWith("0X")) {
1220          hexString = str.substring(2);
1221          if (negative) {
1222            hexString = "-" + hexString;
1223          }
1224          return hexString;
1225        }
1226        return null;
1227      }
1228      
1229      /** 
1230       * Set the value of the <code>name</code> property to a <code>long</code>.
1231       * 
1232       * @param name property name.
1233       * @param value <code>long</code> value of the property.
1234       */
1235      public void setLong(String name, long value) {
1236        set(name, Long.toString(value));
1237      }
1238    
1239      /** 
1240       * Get the value of the <code>name</code> property as a <code>float</code>.  
1241       * If no such property exists, the provided default value is returned,
1242       * or if the specified value is not a valid <code>float</code>,
1243       * then an error is thrown.
1244       *
1245       * @param name property name.
1246       * @param defaultValue default value.
1247       * @throws NumberFormatException when the value is invalid
1248       * @return property value as a <code>float</code>, 
1249       *         or <code>defaultValue</code>. 
1250       */
1251      public float getFloat(String name, float defaultValue) {
1252        String valueString = getTrimmed(name);
1253        if (valueString == null)
1254          return defaultValue;
1255        return Float.parseFloat(valueString);
1256      }
1257    
1258      /**
1259       * Set the value of the <code>name</code> property to a <code>float</code>.
1260       * 
1261       * @param name property name.
1262       * @param value property value.
1263       */
1264      public void setFloat(String name, float value) {
1265        set(name,Float.toString(value));
1266      }
1267    
1268      /** 
1269       * Get the value of the <code>name</code> property as a <code>double</code>.  
1270       * If no such property exists, the provided default value is returned,
1271       * or if the specified value is not a valid <code>double</code>,
1272       * then an error is thrown.
1273       *
1274       * @param name property name.
1275       * @param defaultValue default value.
1276       * @throws NumberFormatException when the value is invalid
1277       * @return property value as a <code>double</code>, 
1278       *         or <code>defaultValue</code>. 
1279       */
1280      public double getDouble(String name, double defaultValue) {
1281        String valueString = getTrimmed(name);
1282        if (valueString == null)
1283          return defaultValue;
1284        return Double.parseDouble(valueString);
1285      }
1286    
1287      /**
1288       * Set the value of the <code>name</code> property to a <code>double</code>.
1289       * 
1290       * @param name property name.
1291       * @param value property value.
1292       */
1293      public void setDouble(String name, double value) {
1294        set(name,Double.toString(value));
1295      }
1296     
1297      /** 
1298       * Get the value of the <code>name</code> property as a <code>boolean</code>.  
1299       * If no such property is specified, or if the specified value is not a valid
1300       * <code>boolean</code>, then <code>defaultValue</code> is returned.
1301       * 
1302       * @param name property name.
1303       * @param defaultValue default value.
1304       * @return property value as a <code>boolean</code>, 
1305       *         or <code>defaultValue</code>. 
1306       */
1307      public boolean getBoolean(String name, boolean defaultValue) {
1308        String valueString = getTrimmed(name);
1309        if (null == valueString || valueString.isEmpty()) {
1310          return defaultValue;
1311        }
1312    
1313        valueString = valueString.toLowerCase();
1314    
1315        if ("true".equals(valueString))
1316          return true;
1317        else if ("false".equals(valueString))
1318          return false;
1319        else return defaultValue;
1320      }
1321    
1322      /** 
1323       * Set the value of the <code>name</code> property to a <code>boolean</code>.
1324       * 
1325       * @param name property name.
1326       * @param value <code>boolean</code> value of the property.
1327       */
1328      public void setBoolean(String name, boolean value) {
1329        set(name, Boolean.toString(value));
1330      }
1331    
1332      /**
1333       * Set the given property, if it is currently unset.
1334       * @param name property name
1335       * @param value new value
1336       */
1337      public void setBooleanIfUnset(String name, boolean value) {
1338        setIfUnset(name, Boolean.toString(value));
1339      }
1340    
1341      /**
1342       * Set the value of the <code>name</code> property to the given type. This
1343       * is equivalent to <code>set(&lt;name&gt;, value.toString())</code>.
1344       * @param name property name
1345       * @param value new value
1346       */
1347      public <T extends Enum<T>> void setEnum(String name, T value) {
1348        set(name, value.toString());
1349      }
1350    
1351      /**
1352       * Return value matching this enumerated type.
1353       * @param name Property name
1354       * @param defaultValue Value returned if no mapping exists
1355       * @throws IllegalArgumentException If mapping is illegal for the type
1356       * provided
1357       */
1358      public <T extends Enum<T>> T getEnum(String name, T defaultValue) {
1359        final String val = get(name);
1360        return null == val
1361          ? defaultValue
1362          : Enum.valueOf(defaultValue.getDeclaringClass(), val);
1363      }
1364    
1365      enum ParsedTimeDuration {
1366        NS {
1367          TimeUnit unit() { return TimeUnit.NANOSECONDS; }
1368          String suffix() { return "ns"; }
1369        },
1370        US {
1371          TimeUnit unit() { return TimeUnit.MICROSECONDS; }
1372          String suffix() { return "us"; }
1373        },
1374        MS {
1375          TimeUnit unit() { return TimeUnit.MILLISECONDS; }
1376          String suffix() { return "ms"; }
1377        },
1378        S {
1379          TimeUnit unit() { return TimeUnit.SECONDS; }
1380          String suffix() { return "s"; }
1381        },
1382        M {
1383          TimeUnit unit() { return TimeUnit.MINUTES; }
1384          String suffix() { return "m"; }
1385        },
1386        H {
1387          TimeUnit unit() { return TimeUnit.HOURS; }
1388          String suffix() { return "h"; }
1389        },
1390        D {
1391          TimeUnit unit() { return TimeUnit.DAYS; }
1392          String suffix() { return "d"; }
1393        };
1394        abstract TimeUnit unit();
1395        abstract String suffix();
1396        static ParsedTimeDuration unitFor(String s) {
1397          for (ParsedTimeDuration ptd : values()) {
1398            // iteration order is in decl order, so SECONDS matched last
1399            if (s.endsWith(ptd.suffix())) {
1400              return ptd;
1401            }
1402          }
1403          return null;
1404        }
1405        static ParsedTimeDuration unitFor(TimeUnit unit) {
1406          for (ParsedTimeDuration ptd : values()) {
1407            if (ptd.unit() == unit) {
1408              return ptd;
1409            }
1410          }
1411          return null;
1412        }
1413      }
1414    
1415      /**
1416       * Set the value of <code>name</code> to the given time duration. This
1417       * is equivalent to <code>set(&lt;name&gt;, value + &lt;time suffix&gt;)</code>.
1418       * @param name Property name
1419       * @param value Time duration
1420       * @param unit Unit of time
1421       */
1422      public void setTimeDuration(String name, long value, TimeUnit unit) {
1423        set(name, value + ParsedTimeDuration.unitFor(unit).suffix());
1424      }
1425    
1426      /**
1427       * Return time duration in the given time unit. Valid units are encoded in
1428       * properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds
1429       * (ms), seconds (s), minutes (m), hours (h), and days (d).
1430       * @param name Property name
1431       * @param defaultValue Value returned if no mapping exists.
1432       * @param unit Unit to convert the stored property, if it exists.
1433       * @throws NumberFormatException If the property stripped of its unit is not
1434       *         a number
1435       */
1436      public long getTimeDuration(String name, long defaultValue, TimeUnit unit) {
1437        String vStr = get(name);
1438        if (null == vStr) {
1439          return defaultValue;
1440        }
1441        vStr = vStr.trim();
1442        ParsedTimeDuration vUnit = ParsedTimeDuration.unitFor(vStr);
1443        if (null == vUnit) {
1444          LOG.warn("No unit for " + name + "(" + vStr + ") assuming " + unit);
1445          vUnit = ParsedTimeDuration.unitFor(unit);
1446        } else {
1447          vStr = vStr.substring(0, vStr.lastIndexOf(vUnit.suffix()));
1448        }
1449        return unit.convert(Long.parseLong(vStr), vUnit.unit());
1450      }
1451    
1452      /**
1453       * Get the value of the <code>name</code> property as a <code>Pattern</code>.
1454       * If no such property is specified, or if the specified value is not a valid
1455       * <code>Pattern</code>, then <code>DefaultValue</code> is returned.
1456       *
1457       * @param name property name
1458       * @param defaultValue default value
1459       * @return property value as a compiled Pattern, or defaultValue
1460       */
1461      public Pattern getPattern(String name, Pattern defaultValue) {
1462        String valString = get(name);
1463        if (null == valString || valString.isEmpty()) {
1464          return defaultValue;
1465        }
1466        try {
1467          return Pattern.compile(valString);
1468        } catch (PatternSyntaxException pse) {
1469          LOG.warn("Regular expression '" + valString + "' for property '" +
1470                   name + "' not valid. Using default", pse);
1471          return defaultValue;
1472        }
1473      }
1474    
1475      /**
1476       * Set the given property to <code>Pattern</code>.
1477       * If the pattern is passed as null, sets the empty pattern which results in
1478       * further calls to getPattern(...) returning the default value.
1479       *
1480       * @param name property name
1481       * @param pattern new value
1482       */
1483      public void setPattern(String name, Pattern pattern) {
1484        if (null == pattern) {
1485          set(name, null);
1486        } else {
1487          set(name, pattern.pattern());
1488        }
1489      }
1490    
1491      /**
1492       * Gets information about why a property was set.  Typically this is the 
1493       * path to the resource objects (file, URL, etc.) the property came from, but
1494       * it can also indicate that it was set programatically, or because of the
1495       * command line.
1496       *
1497       * @param name - The property name to get the source of.
1498       * @return null - If the property or its source wasn't found. Otherwise, 
1499       * returns a list of the sources of the resource.  The older sources are
1500       * the first ones in the list.  So for example if a configuration is set from
1501       * the command line, and then written out to a file that is read back in the
1502       * first entry would indicate that it was set from the command line, while
1503       * the second one would indicate the file that the new configuration was read
1504       * in from.
1505       */
1506      @InterfaceStability.Unstable
1507      public synchronized String[] getPropertySources(String name) {
1508        if (properties == null) {
1509          // If properties is null, it means a resource was newly added
1510          // but the props were cleared so as to load it upon future
1511          // requests. So lets force a load by asking a properties list.
1512          getProps();
1513        }
1514        // Return a null right away if our properties still
1515        // haven't loaded or the resource mapping isn't defined
1516        if (properties == null || updatingResource == null) {
1517          return null;
1518        } else {
1519          String[] source = updatingResource.get(name);
1520          if(source == null) {
1521            return null;
1522          } else {
1523            return Arrays.copyOf(source, source.length);
1524          }
1525        }
1526      }
1527    
1528      /**
1529       * A class that represents a set of positive integer ranges. It parses 
1530       * strings of the form: "2-3,5,7-" where ranges are separated by comma and 
1531       * the lower/upper bounds are separated by dash. Either the lower or upper 
1532       * bound may be omitted meaning all values up to or over. So the string 
1533       * above means 2, 3, 5, and 7, 8, 9, ...
1534       */
1535      public static class IntegerRanges implements Iterable<Integer>{
1536        private static class Range {
1537          int start;
1538          int end;
1539        }
1540        
1541        private static class RangeNumberIterator implements Iterator<Integer> {
1542          Iterator<Range> internal;
1543          int at;
1544          int end;
1545    
1546          public RangeNumberIterator(List<Range> ranges) {
1547            if (ranges != null) {
1548              internal = ranges.iterator();
1549            }
1550            at = -1;
1551            end = -2;
1552          }
1553          
1554          @Override
1555          public boolean hasNext() {
1556            if (at <= end) {
1557              return true;
1558            } else if (internal != null){
1559              return internal.hasNext();
1560            }
1561            return false;
1562          }
1563    
1564          @Override
1565          public Integer next() {
1566            if (at <= end) {
1567              at++;
1568              return at - 1;
1569            } else if (internal != null){
1570              Range found = internal.next();
1571              if (found != null) {
1572                at = found.start;
1573                end = found.end;
1574                at++;
1575                return at - 1;
1576              }
1577            }
1578            return null;
1579          }
1580    
1581          @Override
1582          public void remove() {
1583            throw new UnsupportedOperationException();
1584          }
1585        };
1586    
1587        List<Range> ranges = new ArrayList<Range>();
1588        
1589        public IntegerRanges() {
1590        }
1591        
1592        public IntegerRanges(String newValue) {
1593          StringTokenizer itr = new StringTokenizer(newValue, ",");
1594          while (itr.hasMoreTokens()) {
1595            String rng = itr.nextToken().trim();
1596            String[] parts = rng.split("-", 3);
1597            if (parts.length < 1 || parts.length > 2) {
1598              throw new IllegalArgumentException("integer range badly formed: " + 
1599                                                 rng);
1600            }
1601            Range r = new Range();
1602            r.start = convertToInt(parts[0], 0);
1603            if (parts.length == 2) {
1604              r.end = convertToInt(parts[1], Integer.MAX_VALUE);
1605            } else {
1606              r.end = r.start;
1607            }
1608            if (r.start > r.end) {
1609              throw new IllegalArgumentException("IntegerRange from " + r.start + 
1610                                                 " to " + r.end + " is invalid");
1611            }
1612            ranges.add(r);
1613          }
1614        }
1615    
1616        /**
1617         * Convert a string to an int treating empty strings as the default value.
1618         * @param value the string value
1619         * @param defaultValue the value for if the string is empty
1620         * @return the desired integer
1621         */
1622        private static int convertToInt(String value, int defaultValue) {
1623          String trim = value.trim();
1624          if (trim.length() == 0) {
1625            return defaultValue;
1626          }
1627          return Integer.parseInt(trim);
1628        }
1629    
1630        /**
1631         * Is the given value in the set of ranges
1632         * @param value the value to check
1633         * @return is the value in the ranges?
1634         */
1635        public boolean isIncluded(int value) {
1636          for(Range r: ranges) {
1637            if (r.start <= value && value <= r.end) {
1638              return true;
1639            }
1640          }
1641          return false;
1642        }
1643        
1644        /**
1645         * @return true if there are no values in this range, else false.
1646         */
1647        public boolean isEmpty() {
1648          return ranges == null || ranges.isEmpty();
1649        }
1650        
1651        @Override
1652        public String toString() {
1653          StringBuilder result = new StringBuilder();
1654          boolean first = true;
1655          for(Range r: ranges) {
1656            if (first) {
1657              first = false;
1658            } else {
1659              result.append(',');
1660            }
1661            result.append(r.start);
1662            result.append('-');
1663            result.append(r.end);
1664          }
1665          return result.toString();
1666        }
1667    
1668        @Override
1669        public Iterator<Integer> iterator() {
1670          return new RangeNumberIterator(ranges);
1671        }
1672        
1673      }
1674    
1675      /**
1676       * Parse the given attribute as a set of integer ranges
1677       * @param name the attribute name
1678       * @param defaultValue the default value if it is not set
1679       * @return a new set of ranges from the configured value
1680       */
1681      public IntegerRanges getRange(String name, String defaultValue) {
1682        return new IntegerRanges(get(name, defaultValue));
1683      }
1684    
1685      /** 
1686       * Get the comma delimited values of the <code>name</code> property as 
1687       * a collection of <code>String</code>s.  
1688       * If no such property is specified then empty collection is returned.
1689       * <p>
1690       * This is an optimized version of {@link #getStrings(String)}
1691       * 
1692       * @param name property name.
1693       * @return property value as a collection of <code>String</code>s. 
1694       */
1695      public Collection<String> getStringCollection(String name) {
1696        String valueString = get(name);
1697        return StringUtils.getStringCollection(valueString);
1698      }
1699    
1700      /** 
1701       * Get the comma delimited values of the <code>name</code> property as 
1702       * an array of <code>String</code>s.  
1703       * If no such property is specified then <code>null</code> is returned.
1704       * 
1705       * @param name property name.
1706       * @return property value as an array of <code>String</code>s, 
1707       *         or <code>null</code>. 
1708       */
1709      public String[] getStrings(String name) {
1710        String valueString = get(name);
1711        return StringUtils.getStrings(valueString);
1712      }
1713    
1714      /** 
1715       * Get the comma delimited values of the <code>name</code> property as 
1716       * an array of <code>String</code>s.  
1717       * If no such property is specified then default value is returned.
1718       * 
1719       * @param name property name.
1720       * @param defaultValue The default value
1721       * @return property value as an array of <code>String</code>s, 
1722       *         or default value. 
1723       */
1724      public String[] getStrings(String name, String... defaultValue) {
1725        String valueString = get(name);
1726        if (valueString == null) {
1727          return defaultValue;
1728        } else {
1729          return StringUtils.getStrings(valueString);
1730        }
1731      }
1732      
1733      /** 
1734       * Get the comma delimited values of the <code>name</code> property as 
1735       * a collection of <code>String</code>s, trimmed of the leading and trailing whitespace.  
1736       * If no such property is specified then empty <code>Collection</code> is returned.
1737       *
1738       * @param name property name.
1739       * @return property value as a collection of <code>String</code>s, or empty <code>Collection</code> 
1740       */
1741      public Collection<String> getTrimmedStringCollection(String name) {
1742        String valueString = get(name);
1743        if (null == valueString) {
1744          Collection<String> empty = new ArrayList<String>();
1745          return empty;
1746        }
1747        return StringUtils.getTrimmedStringCollection(valueString);
1748      }
1749      
1750      /** 
1751       * Get the comma delimited values of the <code>name</code> property as 
1752       * an array of <code>String</code>s, trimmed of the leading and trailing whitespace.
1753       * If no such property is specified then an empty array is returned.
1754       * 
1755       * @param name property name.
1756       * @return property value as an array of trimmed <code>String</code>s, 
1757       *         or empty array. 
1758       */
1759      public String[] getTrimmedStrings(String name) {
1760        String valueString = get(name);
1761        return StringUtils.getTrimmedStrings(valueString);
1762      }
1763    
1764      /** 
1765       * Get the comma delimited values of the <code>name</code> property as 
1766       * an array of <code>String</code>s, trimmed of the leading and trailing whitespace.
1767       * If no such property is specified then default value is returned.
1768       * 
1769       * @param name property name.
1770       * @param defaultValue The default value
1771       * @return property value as an array of trimmed <code>String</code>s, 
1772       *         or default value. 
1773       */
1774      public String[] getTrimmedStrings(String name, String... defaultValue) {
1775        String valueString = get(name);
1776        if (null == valueString) {
1777          return defaultValue;
1778        } else {
1779          return StringUtils.getTrimmedStrings(valueString);
1780        }
1781      }
1782    
1783      /** 
1784       * Set the array of string values for the <code>name</code> property as 
1785       * as comma delimited values.  
1786       * 
1787       * @param name property name.
1788       * @param values The values
1789       */
1790      public void setStrings(String name, String... values) {
1791        set(name, StringUtils.arrayToString(values));
1792      }
1793    
1794      /**
1795       * Get the value for a known password configuration element.
1796       * In order to enable the elimination of clear text passwords in config,
1797       * this method attempts to resolve the property name as an alias through
1798       * the CredentialProvider API and conditionally fallsback to config.
1799       * @param name property name
1800       * @return password
1801       */
1802      public char[] getPassword(String name) throws IOException {
1803        char[] pass = null;
1804    
1805        pass = getPasswordFromCredentialProviders(name);
1806    
1807        if (pass == null) {
1808          pass = getPasswordFromConfig(name);
1809        }
1810    
1811        return pass;
1812      }
1813    
1814      /**
1815       * Try and resolve the provided element name as a credential provider
1816       * alias.
1817       * @param name alias of the provisioned credential
1818       * @return password or null if not found
1819       * @throws IOException
1820       */
1821      protected char[] getPasswordFromCredentialProviders(String name)
1822          throws IOException {
1823        char[] pass = null;
1824        try {
1825          List<CredentialProvider> providers =
1826              CredentialProviderFactory.getProviders(this);
1827    
1828          if (providers != null) {
1829            for (CredentialProvider provider : providers) {
1830              try {
1831                CredentialEntry entry = provider.getCredentialEntry(name);
1832                if (entry != null) {
1833                  pass = entry.getCredential();
1834                  break;
1835                }
1836              }
1837              catch (IOException ioe) {
1838                throw new IOException("Can't get key " + name + " from key provider" +
1839                            "of type: " + provider.getClass().getName() + ".", ioe);
1840              }
1841            }
1842          }
1843        }
1844        catch (IOException ioe) {
1845          throw new IOException("Configuration problem with provider path.", ioe);
1846        }
1847    
1848        return pass;
1849      }
1850    
1851      /**
1852       * Fallback to clear text passwords in configuration.
1853       * @param name
1854       * @return clear text password or null
1855       */
1856      protected char[] getPasswordFromConfig(String name) {
1857        char[] pass = null;
1858        if (getBoolean(CredentialProvider.CLEAR_TEXT_FALLBACK, true)) {
1859          String passStr = get(name);
1860          if (passStr != null) {
1861            pass = passStr.toCharArray();
1862          }
1863        }
1864        return pass;
1865      }
1866    
1867      /**
1868       * Get the socket address for <code>hostProperty</code> as a
1869       * <code>InetSocketAddress</code>. If <code>hostProperty</code> is
1870       * <code>null</code>, <code>addressProperty</code> will be used. This
1871       * is useful for cases where we want to differentiate between host
1872       * bind address and address clients should use to establish connection.
1873       *
1874       * @param hostProperty bind host property name.
1875       * @param addressProperty address property name.
1876       * @param defaultAddressValue the default value
1877       * @param defaultPort the default port
1878       * @return InetSocketAddress
1879       */
1880      public InetSocketAddress getSocketAddr(
1881          String hostProperty,
1882          String addressProperty,
1883          String defaultAddressValue,
1884          int defaultPort) {
1885    
1886        InetSocketAddress bindAddr = getSocketAddr(
1887          addressProperty, defaultAddressValue, defaultPort);
1888    
1889        final String host = get(hostProperty);
1890    
1891        if (host == null || host.isEmpty()) {
1892          return bindAddr;
1893        }
1894    
1895        return NetUtils.createSocketAddr(
1896            host, bindAddr.getPort(), hostProperty);
1897      }
1898    
1899      /**
1900       * Get the socket address for <code>name</code> property as a
1901       * <code>InetSocketAddress</code>.
1902       * @param name property name.
1903       * @param defaultAddress the default value
1904       * @param defaultPort the default port
1905       * @return InetSocketAddress
1906       */
1907      public InetSocketAddress getSocketAddr(
1908          String name, String defaultAddress, int defaultPort) {
1909        final String address = get(name, defaultAddress);
1910        return NetUtils.createSocketAddr(address, defaultPort, name);
1911      }
1912    
1913      /**
1914       * Set the socket address for the <code>name</code> property as
1915       * a <code>host:port</code>.
1916       */
1917      public void setSocketAddr(String name, InetSocketAddress addr) {
1918        set(name, NetUtils.getHostPortString(addr));
1919      }
1920    
1921      /**
1922       * Set the socket address a client can use to connect for the
1923       * <code>name</code> property as a <code>host:port</code>.  The wildcard
1924       * address is replaced with the local host's address. If the host and address
1925       * properties are configured the host component of the address will be combined
1926       * with the port component of the addr to generate the address.  This is to allow
1927       * optional control over which host name is used in multi-home bind-host
1928       * cases where a host can have multiple names
1929       * @param hostProperty the bind-host configuration name
1930       * @param addressProperty the service address configuration name
1931       * @param defaultAddressValue the service default address configuration value
1932       * @param addr InetSocketAddress of the service listener
1933       * @return InetSocketAddress for clients to connect
1934       */
1935      public InetSocketAddress updateConnectAddr(
1936          String hostProperty,
1937          String addressProperty,
1938          String defaultAddressValue,
1939          InetSocketAddress addr) {
1940    
1941        final String host = get(hostProperty);
1942        final String connectHostPort = getTrimmed(addressProperty, defaultAddressValue);
1943    
1944        if (host == null || host.isEmpty() || connectHostPort == null || connectHostPort.isEmpty()) {
1945          //not our case, fall back to original logic
1946          return updateConnectAddr(addressProperty, addr);
1947        }
1948    
1949        final String connectHost = connectHostPort.split(":")[0];
1950        // Create connect address using client address hostname and server port.
1951        return updateConnectAddr(addressProperty, NetUtils.createSocketAddrForHost(
1952            connectHost, addr.getPort()));
1953      }
1954      
1955      /**
1956       * Set the socket address a client can use to connect for the
1957       * <code>name</code> property as a <code>host:port</code>.  The wildcard
1958       * address is replaced with the local host's address.
1959       * @param name property name.
1960       * @param addr InetSocketAddress of a listener to store in the given property
1961       * @return InetSocketAddress for clients to connect
1962       */
1963      public InetSocketAddress updateConnectAddr(String name,
1964                                                 InetSocketAddress addr) {
1965        final InetSocketAddress connectAddr = NetUtils.getConnectAddress(addr);
1966        setSocketAddr(name, connectAddr);
1967        return connectAddr;
1968      }
1969      
1970      /**
1971       * Load a class by name.
1972       * 
1973       * @param name the class name.
1974       * @return the class object.
1975       * @throws ClassNotFoundException if the class is not found.
1976       */
1977      public Class<?> getClassByName(String name) throws ClassNotFoundException {
1978        Class<?> ret = getClassByNameOrNull(name);
1979        if (ret == null) {
1980          throw new ClassNotFoundException("Class " + name + " not found");
1981        }
1982        return ret;
1983      }
1984      
1985      /**
1986       * Load a class by name, returning null rather than throwing an exception
1987       * if it couldn't be loaded. This is to avoid the overhead of creating
1988       * an exception.
1989       * 
1990       * @param name the class name
1991       * @return the class object, or null if it could not be found.
1992       */
1993      public Class<?> getClassByNameOrNull(String name) {
1994        Map<String, WeakReference<Class<?>>> map;
1995        
1996        synchronized (CACHE_CLASSES) {
1997          map = CACHE_CLASSES.get(classLoader);
1998          if (map == null) {
1999            map = Collections.synchronizedMap(
2000              new WeakHashMap<String, WeakReference<Class<?>>>());
2001            CACHE_CLASSES.put(classLoader, map);
2002          }
2003        }
2004    
2005        Class<?> clazz = null;
2006        WeakReference<Class<?>> ref = map.get(name); 
2007        if (ref != null) {
2008           clazz = ref.get();
2009        }
2010         
2011        if (clazz == null) {
2012          try {
2013            clazz = Class.forName(name, true, classLoader);
2014          } catch (ClassNotFoundException e) {
2015            // Leave a marker that the class isn't found
2016            map.put(name, new WeakReference<Class<?>>(NEGATIVE_CACHE_SENTINEL));
2017            return null;
2018          }
2019          // two putters can race here, but they'll put the same class
2020          map.put(name, new WeakReference<Class<?>>(clazz));
2021          return clazz;
2022        } else if (clazz == NEGATIVE_CACHE_SENTINEL) {
2023          return null; // not found
2024        } else {
2025          // cache hit
2026          return clazz;
2027        }
2028      }
2029    
2030      /** 
2031       * Get the value of the <code>name</code> property
2032       * as an array of <code>Class</code>.
2033       * The value of the property specifies a list of comma separated class names.  
2034       * If no such property is specified, then <code>defaultValue</code> is 
2035       * returned.
2036       * 
2037       * @param name the property name.
2038       * @param defaultValue default value.
2039       * @return property value as a <code>Class[]</code>, 
2040       *         or <code>defaultValue</code>. 
2041       */
2042      public Class<?>[] getClasses(String name, Class<?> ... defaultValue) {
2043        String[] classnames = getTrimmedStrings(name);
2044        if (classnames == null)
2045          return defaultValue;
2046        try {
2047          Class<?>[] classes = new Class<?>[classnames.length];
2048          for(int i = 0; i < classnames.length; i++) {
2049            classes[i] = getClassByName(classnames[i]);
2050          }
2051          return classes;
2052        } catch (ClassNotFoundException e) {
2053          throw new RuntimeException(e);
2054        }
2055      }
2056    
2057      /** 
2058       * Get the value of the <code>name</code> property as a <code>Class</code>.  
2059       * If no such property is specified, then <code>defaultValue</code> is 
2060       * returned.
2061       * 
2062       * @param name the class name.
2063       * @param defaultValue default value.
2064       * @return property value as a <code>Class</code>, 
2065       *         or <code>defaultValue</code>. 
2066       */
2067      public Class<?> getClass(String name, Class<?> defaultValue) {
2068        String valueString = getTrimmed(name);
2069        if (valueString == null)
2070          return defaultValue;
2071        try {
2072          return getClassByName(valueString);
2073        } catch (ClassNotFoundException e) {
2074          throw new RuntimeException(e);
2075        }
2076      }
2077    
2078      /** 
2079       * Get the value of the <code>name</code> property as a <code>Class</code>
2080       * implementing the interface specified by <code>xface</code>.
2081       *   
2082       * If no such property is specified, then <code>defaultValue</code> is 
2083       * returned.
2084       * 
2085       * An exception is thrown if the returned class does not implement the named
2086       * interface. 
2087       * 
2088       * @param name the class name.
2089       * @param defaultValue default value.
2090       * @param xface the interface implemented by the named class.
2091       * @return property value as a <code>Class</code>, 
2092       *         or <code>defaultValue</code>.
2093       */
2094      public <U> Class<? extends U> getClass(String name, 
2095                                             Class<? extends U> defaultValue, 
2096                                             Class<U> xface) {
2097        try {
2098          Class<?> theClass = getClass(name, defaultValue);
2099          if (theClass != null && !xface.isAssignableFrom(theClass))
2100            throw new RuntimeException(theClass+" not "+xface.getName());
2101          else if (theClass != null)
2102            return theClass.asSubclass(xface);
2103          else
2104            return null;
2105        } catch (Exception e) {
2106          throw new RuntimeException(e);
2107        }
2108      }
2109    
2110      /**
2111       * Get the value of the <code>name</code> property as a <code>List</code>
2112       * of objects implementing the interface specified by <code>xface</code>.
2113       * 
2114       * An exception is thrown if any of the classes does not exist, or if it does
2115       * not implement the named interface.
2116       * 
2117       * @param name the property name.
2118       * @param xface the interface implemented by the classes named by
2119       *        <code>name</code>.
2120       * @return a <code>List</code> of objects implementing <code>xface</code>.
2121       */
2122      @SuppressWarnings("unchecked")
2123      public <U> List<U> getInstances(String name, Class<U> xface) {
2124        List<U> ret = new ArrayList<U>();
2125        Class<?>[] classes = getClasses(name);
2126        for (Class<?> cl: classes) {
2127          if (!xface.isAssignableFrom(cl)) {
2128            throw new RuntimeException(cl + " does not implement " + xface);
2129          }
2130          ret.add((U)ReflectionUtils.newInstance(cl, this));
2131        }
2132        return ret;
2133      }
2134    
2135      /** 
2136       * Set the value of the <code>name</code> property to the name of a 
2137       * <code>theClass</code> implementing the given interface <code>xface</code>.
2138       * 
2139       * An exception is thrown if <code>theClass</code> does not implement the 
2140       * interface <code>xface</code>. 
2141       * 
2142       * @param name property name.
2143       * @param theClass property value.
2144       * @param xface the interface implemented by the named class.
2145       */
2146      public void setClass(String name, Class<?> theClass, Class<?> xface) {
2147        if (!xface.isAssignableFrom(theClass))
2148          throw new RuntimeException(theClass+" not "+xface.getName());
2149        set(name, theClass.getName());
2150      }
2151    
2152      /** 
2153       * Get a local file under a directory named by <i>dirsProp</i> with
2154       * the given <i>path</i>.  If <i>dirsProp</i> contains multiple directories,
2155       * then one is chosen based on <i>path</i>'s hash code.  If the selected
2156       * directory does not exist, an attempt is made to create it.
2157       * 
2158       * @param dirsProp directory in which to locate the file.
2159       * @param path file-path.
2160       * @return local file under the directory with the given path.
2161       */
2162      public Path getLocalPath(String dirsProp, String path)
2163        throws IOException {
2164        String[] dirs = getTrimmedStrings(dirsProp);
2165        int hashCode = path.hashCode();
2166        FileSystem fs = FileSystem.getLocal(this);
2167        for (int i = 0; i < dirs.length; i++) {  // try each local dir
2168          int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length;
2169          Path file = new Path(dirs[index], path);
2170          Path dir = file.getParent();
2171          if (fs.mkdirs(dir) || fs.exists(dir)) {
2172            return file;
2173          }
2174        }
2175        LOG.warn("Could not make " + path + 
2176                 " in local directories from " + dirsProp);
2177        for(int i=0; i < dirs.length; i++) {
2178          int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length;
2179          LOG.warn(dirsProp + "[" + index + "]=" + dirs[index]);
2180        }
2181        throw new IOException("No valid local directories in property: "+dirsProp);
2182      }
2183    
2184      /** 
2185       * Get a local file name under a directory named in <i>dirsProp</i> with
2186       * the given <i>path</i>.  If <i>dirsProp</i> contains multiple directories,
2187       * then one is chosen based on <i>path</i>'s hash code.  If the selected
2188       * directory does not exist, an attempt is made to create it.
2189       * 
2190       * @param dirsProp directory in which to locate the file.
2191       * @param path file-path.
2192       * @return local file under the directory with the given path.
2193       */
2194      public File getFile(String dirsProp, String path)
2195        throws IOException {
2196        String[] dirs = getTrimmedStrings(dirsProp);
2197        int hashCode = path.hashCode();
2198        for (int i = 0; i < dirs.length; i++) {  // try each local dir
2199          int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length;
2200          File file = new File(dirs[index], path);
2201          File dir = file.getParentFile();
2202          if (dir.exists() || dir.mkdirs()) {
2203            return file;
2204          }
2205        }
2206        throw new IOException("No valid local directories in property: "+dirsProp);
2207      }
2208    
2209      /** 
2210       * Get the {@link URL} for the named resource.
2211       * 
2212       * @param name resource name.
2213       * @return the url for the named resource.
2214       */
2215      public URL getResource(String name) {
2216        return classLoader.getResource(name);
2217      }
2218      
2219      /** 
2220       * Get an input stream attached to the configuration resource with the
2221       * given <code>name</code>.
2222       * 
2223       * @param name configuration resource name.
2224       * @return an input stream attached to the resource.
2225       */
2226      public InputStream getConfResourceAsInputStream(String name) {
2227        try {
2228          URL url= getResource(name);
2229    
2230          if (url == null) {
2231            LOG.info(name + " not found");
2232            return null;
2233          } else {
2234            LOG.info("found resource " + name + " at " + url);
2235          }
2236    
2237          return url.openStream();
2238        } catch (Exception e) {
2239          return null;
2240        }
2241      }
2242    
2243      /** 
2244       * Get a {@link Reader} attached to the configuration resource with the
2245       * given <code>name</code>.
2246       * 
2247       * @param name configuration resource name.
2248       * @return a reader attached to the resource.
2249       */
2250      public Reader getConfResourceAsReader(String name) {
2251        try {
2252          URL url= getResource(name);
2253    
2254          if (url == null) {
2255            LOG.info(name + " not found");
2256            return null;
2257          } else {
2258            LOG.info("found resource " + name + " at " + url);
2259          }
2260    
2261          return new InputStreamReader(url.openStream());
2262        } catch (Exception e) {
2263          return null;
2264        }
2265      }
2266    
2267      /**
2268       * Get the set of parameters marked final.
2269       *
2270       * @return final parameter set.
2271       */
2272      public Set<String> getFinalParameters() {
2273        return new HashSet<String>(finalParameters);
2274      }
2275    
2276      protected synchronized Properties getProps() {
2277        if (properties == null) {
2278          properties = new Properties();
2279          HashMap<String, String[]> backup = 
2280            new HashMap<String, String[]>(updatingResource);
2281          loadResources(properties, resources, quietmode);
2282          if (overlay!= null) {
2283            properties.putAll(overlay);
2284            for (Map.Entry<Object,Object> item: overlay.entrySet()) {
2285              String key = (String)item.getKey();
2286              updatingResource.put(key, backup.get(key));
2287            }
2288          }
2289        }
2290        return properties;
2291      }
2292    
2293      /**
2294       * Return the number of keys in the configuration.
2295       *
2296       * @return number of keys in the configuration.
2297       */
2298      public int size() {
2299        return getProps().size();
2300      }
2301    
2302      /**
2303       * Clears all keys from the configuration.
2304       */
2305      public void clear() {
2306        getProps().clear();
2307        getOverlay().clear();
2308      }
2309    
2310      /**
2311       * Get an {@link Iterator} to go through the list of <code>String</code> 
2312       * key-value pairs in the configuration.
2313       * 
2314       * @return an iterator over the entries.
2315       */
2316      @Override
2317      public Iterator<Map.Entry<String, String>> iterator() {
2318        // Get a copy of just the string to string pairs. After the old object
2319        // methods that allow non-strings to be put into configurations are removed,
2320        // we could replace properties with a Map<String,String> and get rid of this
2321        // code.
2322        Map<String,String> result = new HashMap<String,String>();
2323        for(Map.Entry<Object,Object> item: getProps().entrySet()) {
2324          if (item.getKey() instanceof String && 
2325              item.getValue() instanceof String) {
2326            result.put((String) item.getKey(), (String) item.getValue());
2327          }
2328        }
2329        return result.entrySet().iterator();
2330      }
2331    
2332      private Document parse(DocumentBuilder builder, URL url)
2333          throws IOException, SAXException {
2334        if (!quietmode) {
2335          LOG.debug("parsing URL " + url);
2336        }
2337        if (url == null) {
2338          return null;
2339        }
2340        return parse(builder, url.openStream(), url.toString());
2341      }
2342    
2343      private Document parse(DocumentBuilder builder, InputStream is,
2344          String systemId) throws IOException, SAXException {
2345        if (!quietmode) {
2346          LOG.debug("parsing input stream " + is);
2347        }
2348        if (is == null) {
2349          return null;
2350        }
2351        try {
2352          return (systemId == null) ? builder.parse(is) : builder.parse(is,
2353              systemId);
2354        } finally {
2355          is.close();
2356        }
2357      }
2358    
2359      private void loadResources(Properties properties,
2360                                 ArrayList<Resource> resources,
2361                                 boolean quiet) {
2362        if(loadDefaults) {
2363          for (String resource : defaultResources) {
2364            loadResource(properties, new Resource(resource), quiet);
2365          }
2366        
2367          //support the hadoop-site.xml as a deprecated case
2368          if(getResource("hadoop-site.xml")!=null) {
2369            loadResource(properties, new Resource("hadoop-site.xml"), quiet);
2370          }
2371        }
2372        
2373        for (int i = 0; i < resources.size(); i++) {
2374          Resource ret = loadResource(properties, resources.get(i), quiet);
2375          if (ret != null) {
2376            resources.set(i, ret);
2377          }
2378        }
2379      }
2380      
2381      private Resource loadResource(Properties properties, Resource wrapper, boolean quiet) {
2382        String name = UNKNOWN_RESOURCE;
2383        try {
2384          Object resource = wrapper.getResource();
2385          name = wrapper.getName();
2386          
2387          DocumentBuilderFactory docBuilderFactory 
2388            = DocumentBuilderFactory.newInstance();
2389          //ignore all comments inside the xml file
2390          docBuilderFactory.setIgnoringComments(true);
2391    
2392          //allow includes in the xml file
2393          docBuilderFactory.setNamespaceAware(true);
2394          try {
2395              docBuilderFactory.setXIncludeAware(true);
2396          } catch (UnsupportedOperationException e) {
2397            LOG.error("Failed to set setXIncludeAware(true) for parser "
2398                    + docBuilderFactory
2399                    + ":" + e,
2400                    e);
2401          }
2402          DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
2403          Document doc = null;
2404          Element root = null;
2405          boolean returnCachedProperties = false;
2406          
2407          if (resource instanceof URL) {                  // an URL resource
2408            doc = parse(builder, (URL)resource);
2409          } else if (resource instanceof String) {        // a CLASSPATH resource
2410            URL url = getResource((String)resource);
2411            doc = parse(builder, url);
2412          } else if (resource instanceof Path) {          // a file resource
2413            // Can't use FileSystem API or we get an infinite loop
2414            // since FileSystem uses Configuration API.  Use java.io.File instead.
2415            File file = new File(((Path)resource).toUri().getPath())
2416              .getAbsoluteFile();
2417            if (file.exists()) {
2418              if (!quiet) {
2419                LOG.debug("parsing File " + file);
2420              }
2421              doc = parse(builder, new BufferedInputStream(
2422                  new FileInputStream(file)), ((Path)resource).toString());
2423            }
2424          } else if (resource instanceof InputStream) {
2425            doc = parse(builder, (InputStream) resource, null);
2426            returnCachedProperties = true;
2427          } else if (resource instanceof Properties) {
2428            overlay(properties, (Properties)resource);
2429          } else if (resource instanceof Element) {
2430            root = (Element)resource;
2431          }
2432    
2433          if (root == null) {
2434            if (doc == null) {
2435              if (quiet) {
2436                return null;
2437              }
2438              throw new RuntimeException(resource + " not found");
2439            }
2440            root = doc.getDocumentElement();
2441          }
2442          Properties toAddTo = properties;
2443          if(returnCachedProperties) {
2444            toAddTo = new Properties();
2445          }
2446          if (!"configuration".equals(root.getTagName()))
2447            LOG.fatal("bad conf file: top-level element not <configuration>");
2448          NodeList props = root.getChildNodes();
2449          DeprecationContext deprecations = deprecationContext.get();
2450          for (int i = 0; i < props.getLength(); i++) {
2451            Node propNode = props.item(i);
2452            if (!(propNode instanceof Element))
2453              continue;
2454            Element prop = (Element)propNode;
2455            if ("configuration".equals(prop.getTagName())) {
2456              loadResource(toAddTo, new Resource(prop, name), quiet);
2457              continue;
2458            }
2459            if (!"property".equals(prop.getTagName()))
2460              LOG.warn("bad conf file: element not <property>");
2461            NodeList fields = prop.getChildNodes();
2462            String attr = null;
2463            String value = null;
2464            boolean finalParameter = false;
2465            LinkedList<String> source = new LinkedList<String>();
2466            for (int j = 0; j < fields.getLength(); j++) {
2467              Node fieldNode = fields.item(j);
2468              if (!(fieldNode instanceof Element))
2469                continue;
2470              Element field = (Element)fieldNode;
2471              if ("name".equals(field.getTagName()) && field.hasChildNodes())
2472                attr = StringInterner.weakIntern(
2473                    ((Text)field.getFirstChild()).getData().trim());
2474              if ("value".equals(field.getTagName()) && field.hasChildNodes())
2475                value = StringInterner.weakIntern(
2476                    ((Text)field.getFirstChild()).getData());
2477              if ("final".equals(field.getTagName()) && field.hasChildNodes())
2478                finalParameter = "true".equals(((Text)field.getFirstChild()).getData());
2479              if ("source".equals(field.getTagName()) && field.hasChildNodes())
2480                source.add(StringInterner.weakIntern(
2481                    ((Text)field.getFirstChild()).getData()));
2482            }
2483            source.add(name);
2484            
2485            // Ignore this parameter if it has already been marked as 'final'
2486            if (attr != null) {
2487              if (deprecations.getDeprecatedKeyMap().containsKey(attr)) {
2488                DeprecatedKeyInfo keyInfo =
2489                    deprecations.getDeprecatedKeyMap().get(attr);
2490                keyInfo.clearAccessed();
2491                for (String key:keyInfo.newKeys) {
2492                  // update new keys with deprecated key's value 
2493                  loadProperty(toAddTo, name, key, value, finalParameter, 
2494                      source.toArray(new String[source.size()]));
2495                }
2496              }
2497              else {
2498                loadProperty(toAddTo, name, attr, value, finalParameter, 
2499                    source.toArray(new String[source.size()]));
2500              }
2501            }
2502          }
2503          
2504          if (returnCachedProperties) {
2505            overlay(properties, toAddTo);
2506            return new Resource(toAddTo, name);
2507          }
2508          return null;
2509        } catch (IOException e) {
2510          LOG.fatal("error parsing conf " + name, e);
2511          throw new RuntimeException(e);
2512        } catch (DOMException e) {
2513          LOG.fatal("error parsing conf " + name, e);
2514          throw new RuntimeException(e);
2515        } catch (SAXException e) {
2516          LOG.fatal("error parsing conf " + name, e);
2517          throw new RuntimeException(e);
2518        } catch (ParserConfigurationException e) {
2519          LOG.fatal("error parsing conf " + name , e);
2520          throw new RuntimeException(e);
2521        }
2522      }
2523    
2524      private void overlay(Properties to, Properties from) {
2525        for (Entry<Object, Object> entry: from.entrySet()) {
2526          to.put(entry.getKey(), entry.getValue());
2527        }
2528      }
2529      
2530      private void loadProperty(Properties properties, String name, String attr,
2531          String value, boolean finalParameter, String[] source) {
2532        if (value != null) {
2533          if (!finalParameters.contains(attr)) {
2534            properties.setProperty(attr, value);
2535            updatingResource.put(attr, source);
2536          } else if (!value.equals(properties.getProperty(attr))) {
2537            LOG.warn(name+":an attempt to override final parameter: "+attr
2538                +";  Ignoring.");
2539          }
2540        }
2541        if (finalParameter) {
2542          finalParameters.add(attr);
2543        }
2544      }
2545    
2546      /** 
2547       * Write out the non-default properties in this configuration to the given
2548       * {@link OutputStream} using UTF-8 encoding.
2549       * 
2550       * @param out the output stream to write to.
2551       */
2552      public void writeXml(OutputStream out) throws IOException {
2553        writeXml(new OutputStreamWriter(out, "UTF-8"));
2554      }
2555    
2556      /** 
2557       * Write out the non-default properties in this configuration to the given
2558       * {@link Writer}.
2559       * 
2560       * @param out the writer to write to.
2561       */
2562      public void writeXml(Writer out) throws IOException {
2563        Document doc = asXmlDocument();
2564    
2565        try {
2566          DOMSource source = new DOMSource(doc);
2567          StreamResult result = new StreamResult(out);
2568          TransformerFactory transFactory = TransformerFactory.newInstance();
2569          Transformer transformer = transFactory.newTransformer();
2570    
2571          // Important to not hold Configuration log while writing result, since
2572          // 'out' may be an HDFS stream which needs to lock this configuration
2573          // from another thread.
2574          transformer.transform(source, result);
2575        } catch (TransformerException te) {
2576          throw new IOException(te);
2577        }
2578      }
2579    
2580      /**
2581       * Return the XML DOM corresponding to this Configuration.
2582       */
2583      private synchronized Document asXmlDocument() throws IOException {
2584        Document doc;
2585        try {
2586          doc =
2587            DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument();
2588        } catch (ParserConfigurationException pe) {
2589          throw new IOException(pe);
2590        }
2591        Element conf = doc.createElement("configuration");
2592        doc.appendChild(conf);
2593        conf.appendChild(doc.createTextNode("\n"));
2594        handleDeprecation(); //ensure properties is set and deprecation is handled
2595        for (Enumeration<Object> e = properties.keys(); e.hasMoreElements();) {
2596          String name = (String)e.nextElement();
2597          Object object = properties.get(name);
2598          String value = null;
2599          if (object instanceof String) {
2600            value = (String) object;
2601          }else {
2602            continue;
2603          }
2604          Element propNode = doc.createElement("property");
2605          conf.appendChild(propNode);
2606    
2607          Element nameNode = doc.createElement("name");
2608          nameNode.appendChild(doc.createTextNode(name));
2609          propNode.appendChild(nameNode);
2610    
2611          Element valueNode = doc.createElement("value");
2612          valueNode.appendChild(doc.createTextNode(value));
2613          propNode.appendChild(valueNode);
2614    
2615          if (updatingResource != null) {
2616            String[] sources = updatingResource.get(name);
2617            if(sources != null) {
2618              for(String s : sources) {
2619                Element sourceNode = doc.createElement("source");
2620                sourceNode.appendChild(doc.createTextNode(s));
2621                propNode.appendChild(sourceNode);
2622              }
2623            }
2624          }
2625          
2626          conf.appendChild(doc.createTextNode("\n"));
2627        }
2628        return doc;
2629      }
2630    
2631      /**
2632       *  Writes out all the parameters and their properties (final and resource) to
2633       *  the given {@link Writer}
2634       *  The format of the output would be 
2635       *  { "properties" : [ {key1,value1,key1.isFinal,key1.resource}, {key2,value2,
2636       *  key2.isFinal,key2.resource}... ] } 
2637       *  It does not output the parameters of the configuration object which is 
2638       *  loaded from an input stream.
2639       * @param out the Writer to write to
2640       * @throws IOException
2641       */
2642      public static void dumpConfiguration(Configuration config,
2643          Writer out) throws IOException {
2644        JsonFactory dumpFactory = new JsonFactory();
2645        JsonGenerator dumpGenerator = dumpFactory.createJsonGenerator(out);
2646        dumpGenerator.writeStartObject();
2647        dumpGenerator.writeFieldName("properties");
2648        dumpGenerator.writeStartArray();
2649        dumpGenerator.flush();
2650        synchronized (config) {
2651          for (Map.Entry<Object,Object> item: config.getProps().entrySet()) {
2652            dumpGenerator.writeStartObject();
2653            dumpGenerator.writeStringField("key", (String) item.getKey());
2654            dumpGenerator.writeStringField("value", 
2655                                           config.get((String) item.getKey()));
2656            dumpGenerator.writeBooleanField("isFinal",
2657                                            config.finalParameters.contains(item.getKey()));
2658            String[] resources = config.updatingResource.get(item.getKey());
2659            String resource = UNKNOWN_RESOURCE;
2660            if(resources != null && resources.length > 0) {
2661              resource = resources[0];
2662            }
2663            dumpGenerator.writeStringField("resource", resource);
2664            dumpGenerator.writeEndObject();
2665          }
2666        }
2667        dumpGenerator.writeEndArray();
2668        dumpGenerator.writeEndObject();
2669        dumpGenerator.flush();
2670      }
2671      
2672      /**
2673       * Get the {@link ClassLoader} for this job.
2674       * 
2675       * @return the correct class loader.
2676       */
2677      public ClassLoader getClassLoader() {
2678        return classLoader;
2679      }
2680      
2681      /**
2682       * Set the class loader that will be used to load the various objects.
2683       * 
2684       * @param classLoader the new class loader.
2685       */
2686      public void setClassLoader(ClassLoader classLoader) {
2687        this.classLoader = classLoader;
2688      }
2689      
2690      @Override
2691      public String toString() {
2692        StringBuilder sb = new StringBuilder();
2693        sb.append("Configuration: ");
2694        if(loadDefaults) {
2695          toString(defaultResources, sb);
2696          if(resources.size()>0) {
2697            sb.append(", ");
2698          }
2699        }
2700        toString(resources, sb);
2701        return sb.toString();
2702      }
2703      
2704      private <T> void toString(List<T> resources, StringBuilder sb) {
2705        ListIterator<T> i = resources.listIterator();
2706        while (i.hasNext()) {
2707          if (i.nextIndex() != 0) {
2708            sb.append(", ");
2709          }
2710          sb.append(i.next());
2711        }
2712      }
2713    
2714      /** 
2715       * Set the quietness-mode. 
2716       * 
2717       * In the quiet-mode, error and informational messages might not be logged.
2718       * 
2719       * @param quietmode <code>true</code> to set quiet-mode on, <code>false</code>
2720       *              to turn it off.
2721       */
2722      public synchronized void setQuietMode(boolean quietmode) {
2723        this.quietmode = quietmode;
2724      }
2725    
2726      synchronized boolean getQuietMode() {
2727        return this.quietmode;
2728      }
2729      
2730      /** For debugging.  List non-default properties to the terminal and exit. */
2731      public static void main(String[] args) throws Exception {
2732        new Configuration().writeXml(System.out);
2733      }
2734    
2735      @Override
2736      public void readFields(DataInput in) throws IOException {
2737        clear();
2738        int size = WritableUtils.readVInt(in);
2739        for(int i=0; i < size; ++i) {
2740          String key = org.apache.hadoop.io.Text.readString(in);
2741          String value = org.apache.hadoop.io.Text.readString(in);
2742          set(key, value); 
2743          String sources[] = WritableUtils.readCompressedStringArray(in);
2744          updatingResource.put(key, sources);
2745        }
2746      }
2747    
2748      //@Override
2749      @Override
2750      public void write(DataOutput out) throws IOException {
2751        Properties props = getProps();
2752        WritableUtils.writeVInt(out, props.size());
2753        for(Map.Entry<Object, Object> item: props.entrySet()) {
2754          org.apache.hadoop.io.Text.writeString(out, (String) item.getKey());
2755          org.apache.hadoop.io.Text.writeString(out, (String) item.getValue());
2756          WritableUtils.writeCompressedStringArray(out, 
2757              updatingResource.get(item.getKey()));
2758        }
2759      }
2760      
2761      /**
2762       * get keys matching the the regex 
2763       * @param regex
2764       * @return Map<String,String> with matching keys
2765       */
2766      public Map<String,String> getValByRegex(String regex) {
2767        Pattern p = Pattern.compile(regex);
2768    
2769        Map<String,String> result = new HashMap<String,String>();
2770        Matcher m;
2771    
2772        for(Map.Entry<Object,Object> item: getProps().entrySet()) {
2773          if (item.getKey() instanceof String && 
2774              item.getValue() instanceof String) {
2775            m = p.matcher((String)item.getKey());
2776            if(m.find()) { // match
2777              result.put((String) item.getKey(),
2778                  substituteVars(getProps().getProperty((String) item.getKey())));
2779            }
2780          }
2781        }
2782        return result;
2783      }
2784    
2785      /**
2786       * A unique class which is used as a sentinel value in the caching
2787       * for getClassByName. {@see Configuration#getClassByNameOrNull(String)}
2788       */
2789      private static abstract class NegativeCacheSentinel {}
2790    
2791      public static void dumpDeprecatedKeys() {
2792        DeprecationContext deprecations = deprecationContext.get();
2793        for (Map.Entry<String, DeprecatedKeyInfo> entry :
2794            deprecations.getDeprecatedKeyMap().entrySet()) {
2795          StringBuilder newKeys = new StringBuilder();
2796          for (String newKey : entry.getValue().newKeys) {
2797            newKeys.append(newKey).append("\t");
2798          }
2799          System.out.println(entry.getKey() + "\t" + newKeys.toString());
2800        }
2801      }
2802    
2803      /**
2804       * Returns whether or not a deprecated name has been warned. If the name is not
2805       * deprecated then always return false
2806       */
2807      public static boolean hasWarnedDeprecation(String name) {
2808        DeprecationContext deprecations = deprecationContext.get();
2809        if(deprecations.getDeprecatedKeyMap().containsKey(name)) {
2810          if(deprecations.getDeprecatedKeyMap().get(name).accessed.get()) {
2811            return true;
2812          }
2813        }
2814        return false;
2815      }
2816    }