001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.conf;
020
021import com.google.common.annotations.VisibleForTesting;
022
023import java.io.BufferedInputStream;
024import java.io.DataInput;
025import java.io.DataOutput;
026import java.io.File;
027import java.io.FileInputStream;
028import java.io.IOException;
029import java.io.InputStream;
030import java.io.InputStreamReader;
031import java.io.OutputStream;
032import java.io.OutputStreamWriter;
033import java.io.Reader;
034import java.io.Writer;
035import java.lang.ref.WeakReference;
036import java.net.InetSocketAddress;
037import java.net.URL;
038import java.util.ArrayList;
039import java.util.Arrays;
040import java.util.Collection;
041import java.util.Collections;
042import java.util.Enumeration;
043import java.util.HashMap;
044import java.util.HashSet;
045import java.util.Iterator;
046import java.util.LinkedList;
047import java.util.List;
048import java.util.ListIterator;
049import java.util.Map;
050import java.util.Map.Entry;
051import java.util.Properties;
052import java.util.Set;
053import java.util.StringTokenizer;
054import java.util.WeakHashMap;
055import java.util.concurrent.ConcurrentHashMap;
056import java.util.concurrent.CopyOnWriteArrayList;
057import java.util.regex.Matcher;
058import java.util.regex.Pattern;
059import java.util.regex.PatternSyntaxException;
060import java.util.concurrent.TimeUnit;
061import java.util.concurrent.atomic.AtomicBoolean;
062import java.util.concurrent.atomic.AtomicReference;
063
064import javax.xml.parsers.DocumentBuilder;
065import javax.xml.parsers.DocumentBuilderFactory;
066import javax.xml.parsers.ParserConfigurationException;
067import javax.xml.transform.Transformer;
068import javax.xml.transform.TransformerException;
069import javax.xml.transform.TransformerFactory;
070import javax.xml.transform.dom.DOMSource;
071import javax.xml.transform.stream.StreamResult;
072
073import com.google.common.base.Charsets;
074import org.apache.commons.collections.map.UnmodifiableMap;
075import org.apache.commons.logging.Log;
076import org.apache.commons.logging.LogFactory;
077import org.apache.hadoop.classification.InterfaceAudience;
078import org.apache.hadoop.classification.InterfaceStability;
079import org.apache.hadoop.fs.FileSystem;
080import org.apache.hadoop.fs.Path;
081import org.apache.hadoop.fs.CommonConfigurationKeys;
082import org.apache.hadoop.io.Writable;
083import org.apache.hadoop.io.WritableUtils;
084import org.apache.hadoop.net.NetUtils;
085import org.apache.hadoop.security.UserGroupInformation;
086import org.apache.hadoop.security.alias.CredentialProvider;
087import org.apache.hadoop.security.alias.CredentialProvider.CredentialEntry;
088import org.apache.hadoop.security.alias.CredentialProviderFactory;
089import org.apache.hadoop.util.ReflectionUtils;
090import org.apache.hadoop.util.StringInterner;
091import org.apache.hadoop.util.StringUtils;
092import org.codehaus.jackson.JsonFactory;
093import org.codehaus.jackson.JsonGenerator;
094import org.w3c.dom.DOMException;
095import org.w3c.dom.Document;
096import org.w3c.dom.Element;
097import org.w3c.dom.Node;
098import org.w3c.dom.NodeList;
099import org.w3c.dom.Text;
100import org.xml.sax.SAXException;
101
102import com.google.common.base.Preconditions;
103
104/** 
105 * Provides access to configuration parameters.
106 *
107 * <h4 id="Resources">Resources</h4>
108 *
109 * <p>Configurations are specified by resources. A resource contains a set of
110 * name/value pairs as XML data. Each resource is named by either a 
111 * <code>String</code> or by a {@link Path}. If named by a <code>String</code>, 
112 * then the classpath is examined for a file with that name.  If named by a 
113 * <code>Path</code>, then the local filesystem is examined directly, without 
114 * referring to the classpath.
115 *
116 * <p>Unless explicitly turned off, Hadoop by default specifies two 
117 * resources, loaded in-order from the classpath: <ol>
118 * <li><tt>
119 * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
120 * core-default.xml</a></tt>: Read-only defaults for hadoop.</li>
121 * <li><tt>core-site.xml</tt>: Site-specific configuration for a given hadoop
122 * installation.</li>
123 * </ol>
124 * Applications may add additional resources, which are loaded
125 * subsequent to these resources in the order they are added.
126 * 
127 * <h4 id="FinalParams">Final Parameters</h4>
128 *
129 * <p>Configuration parameters may be declared <i>final</i>. 
130 * Once a resource declares a value final, no subsequently-loaded 
131 * resource can alter that value.  
132 * For example, one might define a final parameter with:
133 * <tt><pre>
134 *  &lt;property&gt;
135 *    &lt;name&gt;dfs.hosts.include&lt;/name&gt;
136 *    &lt;value&gt;/etc/hadoop/conf/hosts.include&lt;/value&gt;
137 *    <b>&lt;final&gt;true&lt;/final&gt;</b>
138 *  &lt;/property&gt;</pre></tt>
139 *
140 * Administrators typically define parameters as final in 
141 * <tt>core-site.xml</tt> for values that user applications may not alter.
142 *
143 * <h4 id="VariableExpansion">Variable Expansion</h4>
144 *
145 * <p>Value strings are first processed for <i>variable expansion</i>. The
146 * available properties are:<ol>
147 * <li>Other properties defined in this Configuration; and, if a name is
148 * undefined here,</li>
149 * <li>Properties in {@link System#getProperties()}.</li>
150 * </ol>
151 *
152 * <p>For example, if a configuration resource contains the following property
153 * definitions: 
154 * <tt><pre>
155 *  &lt;property&gt;
156 *    &lt;name&gt;basedir&lt;/name&gt;
157 *    &lt;value&gt;/user/${<i>user.name</i>}&lt;/value&gt;
158 *  &lt;/property&gt;
159 *  
160 *  &lt;property&gt;
161 *    &lt;name&gt;tempdir&lt;/name&gt;
162 *    &lt;value&gt;${<i>basedir</i>}/tmp&lt;/value&gt;
163 *  &lt;/property&gt;</pre></tt>
164 *
165 * When <tt>conf.get("tempdir")</tt> is called, then <tt>${<i>basedir</i>}</tt>
166 * will be resolved to another property in this Configuration, while
167 * <tt>${<i>user.name</i>}</tt> would then ordinarily be resolved to the value
168 * of the System property with that name.
169 * By default, warnings will be given to any deprecated configuration 
170 * parameters and these are suppressible by configuring
171 * <tt>log4j.logger.org.apache.hadoop.conf.Configuration.deprecation</tt> in
172 * log4j.properties file.
173 */
174@InterfaceAudience.Public
175@InterfaceStability.Stable
176public class Configuration implements Iterable<Map.Entry<String,String>>,
177                                      Writable {
178  private static final Log LOG =
179    LogFactory.getLog(Configuration.class);
180
181  private static final Log LOG_DEPRECATION =
182    LogFactory.getLog("org.apache.hadoop.conf.Configuration.deprecation");
183
184  private boolean quietmode = true;
185
186  private static final String DEFAULT_STRING_CHECK =
187    "testingforemptydefaultvalue";
188
189  private static final String XINCLUDE_NS_URI =
190      "http://www.w3.org/2001/XInclude";
191
192  private static boolean restrictSystemPropsDefault = false;
193  private boolean restrictSystemProps = restrictSystemPropsDefault;
194  private boolean allowNullValueProperties = false;
195
196  private static class Resource {
197    private final Object resource;
198    private final String name;
199    private final boolean restrictParser;
200    
201    public Resource(Object resource) {
202      this(resource, resource.toString());
203    }
204
205    public Resource(Object resource, boolean useRestrictedParser) {
206      this(resource, resource.toString(), useRestrictedParser);
207    }
208
209    public Resource(Object resource, String name) {
210      this(resource, name, getRestrictParserDefault(resource));
211    }
212
213    public Resource(Object resource, String name, boolean restrictParser) {
214      this.resource = resource;
215      this.name = name;
216      this.restrictParser = restrictParser;
217    }
218    
219    public String getName(){
220      return name;
221    }
222    
223    public Object getResource() {
224      return resource;
225    }
226
227    public boolean isParserRestricted() {
228      return restrictParser;
229    }
230
231    @Override
232    public String toString() {
233      return name;
234    }
235
236    private static boolean getRestrictParserDefault(Object resource) {
237      if (resource instanceof String) {
238        return false;
239      }
240      UserGroupInformation user;
241      try {
242        user = UserGroupInformation.getCurrentUser();
243      } catch (IOException e) {
244        throw new RuntimeException("Unable to determine current user", e);
245      }
246      return user.getRealUser() != null;
247    }
248  }
249  
250  /**
251   * List of configuration resources.
252   */
253  private ArrayList<Resource> resources = new ArrayList<Resource>();
254  
255  /**
256   * The value reported as the setting resource when a key is set
257   * by code rather than a file resource by dumpConfiguration.
258   */
259  static final String UNKNOWN_RESOURCE = "Unknown";
260
261
262  /**
263   * List of configuration parameters marked <b>final</b>. 
264   */
265  private Set<String> finalParameters = Collections.newSetFromMap(
266      new ConcurrentHashMap<String, Boolean>());
267  
268  private boolean loadDefaults = true;
269
270  /**
271   * Configuration objects
272   */
273  private static final WeakHashMap<Configuration,Object> REGISTRY = 
274    new WeakHashMap<Configuration,Object>();
275  
276  /**
277   * List of default Resources. Resources are loaded in the order of the list 
278   * entries
279   */
280  private static final CopyOnWriteArrayList<String> defaultResources =
281    new CopyOnWriteArrayList<String>();
282
283  private static final Map<ClassLoader, Map<String, WeakReference<Class<?>>>>
284    CACHE_CLASSES = new WeakHashMap<ClassLoader, Map<String, WeakReference<Class<?>>>>();
285
286  /**
287   * Sentinel value to store negative cache results in {@link #CACHE_CLASSES}.
288   */
289  private static final Class<?> NEGATIVE_CACHE_SENTINEL =
290    NegativeCacheSentinel.class;
291
292  /**
293   * Stores the mapping of key to the resource which modifies or loads 
294   * the key most recently
295   */
296  private Map<String, String[]> updatingResource;
297 
298  /**
299   * Class to keep the information about the keys which replace the deprecated
300   * ones.
301   * 
302   * This class stores the new keys which replace the deprecated keys and also
303   * gives a provision to have a custom message for each of the deprecated key
304   * that is being replaced. It also provides method to get the appropriate
305   * warning message which can be logged whenever the deprecated key is used.
306   */
307  private static class DeprecatedKeyInfo {
308    private final String[] newKeys;
309    private final String customMessage;
310    private final AtomicBoolean accessed = new AtomicBoolean(false);
311
312    DeprecatedKeyInfo(String[] newKeys, String customMessage) {
313      this.newKeys = newKeys;
314      this.customMessage = customMessage;
315    }
316
317    /**
318     * Method to provide the warning message. It gives the custom message if
319     * non-null, and default message otherwise.
320     * @param key the associated deprecated key.
321     * @return message that is to be logged when a deprecated key is used.
322     */
323    private final String getWarningMessage(String key) {
324      String warningMessage;
325      if(customMessage == null) {
326        StringBuilder message = new StringBuilder(key);
327        String deprecatedKeySuffix = " is deprecated. Instead, use ";
328        message.append(deprecatedKeySuffix);
329        for (int i = 0; i < newKeys.length; i++) {
330          message.append(newKeys[i]);
331          if(i != newKeys.length-1) {
332            message.append(", ");
333          }
334        }
335        warningMessage = message.toString();
336      }
337      else {
338        warningMessage = customMessage;
339      }
340      return warningMessage;
341    }
342
343    boolean getAndSetAccessed() {
344      return accessed.getAndSet(true);
345    }
346
347    public void clearAccessed() {
348      accessed.set(false);
349    }
350  }
351  
352  /**
353   * A pending addition to the global set of deprecated keys.
354   */
355  public static class DeprecationDelta {
356    private final String key;
357    private final String[] newKeys;
358    private final String customMessage;
359
360    DeprecationDelta(String key, String[] newKeys, String customMessage) {
361      Preconditions.checkNotNull(key);
362      Preconditions.checkNotNull(newKeys);
363      Preconditions.checkArgument(newKeys.length > 0);
364      this.key = key;
365      this.newKeys = newKeys;
366      this.customMessage = customMessage;
367    }
368
369    public DeprecationDelta(String key, String newKey, String customMessage) {
370      this(key, new String[] { newKey }, customMessage);
371    }
372
373    public DeprecationDelta(String key, String newKey) {
374      this(key, new String[] { newKey }, null);
375    }
376
377    public String getKey() {
378      return key;
379    }
380
381    public String[] getNewKeys() {
382      return newKeys;
383    }
384
385    public String getCustomMessage() {
386      return customMessage;
387    }
388  }
389
390  /**
391   * The set of all keys which are deprecated.
392   *
393   * DeprecationContext objects are immutable.
394   */
395  private static class DeprecationContext {
396    /**
397     * Stores the deprecated keys, the new keys which replace the deprecated keys
398     * and custom message(if any provided).
399     */
400    private final Map<String, DeprecatedKeyInfo> deprecatedKeyMap;
401
402    /**
403     * Stores a mapping from superseding keys to the keys which they deprecate.
404     */
405    private final Map<String, String> reverseDeprecatedKeyMap;
406
407    /**
408     * Create a new DeprecationContext by copying a previous DeprecationContext
409     * and adding some deltas.
410     *
411     * @param other   The previous deprecation context to copy, or null to start
412     *                from nothing.
413     * @param deltas  The deltas to apply.
414     */
415    @SuppressWarnings("unchecked")
416    DeprecationContext(DeprecationContext other, DeprecationDelta[] deltas) {
417      HashMap<String, DeprecatedKeyInfo> newDeprecatedKeyMap = 
418        new HashMap<String, DeprecatedKeyInfo>();
419      HashMap<String, String> newReverseDeprecatedKeyMap =
420        new HashMap<String, String>();
421      if (other != null) {
422        for (Entry<String, DeprecatedKeyInfo> entry :
423            other.deprecatedKeyMap.entrySet()) {
424          newDeprecatedKeyMap.put(entry.getKey(), entry.getValue());
425        }
426        for (Entry<String, String> entry :
427            other.reverseDeprecatedKeyMap.entrySet()) {
428          newReverseDeprecatedKeyMap.put(entry.getKey(), entry.getValue());
429        }
430      }
431      for (DeprecationDelta delta : deltas) {
432        if (!newDeprecatedKeyMap.containsKey(delta.getKey())) {
433          DeprecatedKeyInfo newKeyInfo =
434            new DeprecatedKeyInfo(delta.getNewKeys(), delta.getCustomMessage());
435          newDeprecatedKeyMap.put(delta.key, newKeyInfo);
436          for (String newKey : delta.getNewKeys()) {
437            newReverseDeprecatedKeyMap.put(newKey, delta.key);
438          }
439        }
440      }
441      this.deprecatedKeyMap =
442        UnmodifiableMap.decorate(newDeprecatedKeyMap);
443      this.reverseDeprecatedKeyMap =
444        UnmodifiableMap.decorate(newReverseDeprecatedKeyMap);
445    }
446
447    Map<String, DeprecatedKeyInfo> getDeprecatedKeyMap() {
448      return deprecatedKeyMap;
449    }
450
451    Map<String, String> getReverseDeprecatedKeyMap() {
452      return reverseDeprecatedKeyMap;
453    }
454  }
455  
456  private static DeprecationDelta[] defaultDeprecations = 
457    new DeprecationDelta[] {
458      new DeprecationDelta("topology.script.file.name", 
459        CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY),
460      new DeprecationDelta("topology.script.number.args", 
461        CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY),
462      new DeprecationDelta("hadoop.configured.node.mapping", 
463        CommonConfigurationKeys.NET_TOPOLOGY_CONFIGURED_NODE_MAPPING_KEY),
464      new DeprecationDelta("topology.node.switch.mapping.impl", 
465        CommonConfigurationKeys.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY),
466      new DeprecationDelta("dfs.df.interval", 
467        CommonConfigurationKeys.FS_DF_INTERVAL_KEY),
468      new DeprecationDelta("hadoop.native.lib", 
469        CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY),
470      new DeprecationDelta("fs.default.name", 
471        CommonConfigurationKeys.FS_DEFAULT_NAME_KEY),
472      new DeprecationDelta("dfs.umaskmode",
473        CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY),
474      new DeprecationDelta("dfs.nfs.exports.allowed.hosts",
475          CommonConfigurationKeys.NFS_EXPORTS_ALLOWED_HOSTS_KEY)
476    };
477
478  /**
479   * The global DeprecationContext.
480   */
481  private static AtomicReference<DeprecationContext> deprecationContext =
482      new AtomicReference<DeprecationContext>(
483          new DeprecationContext(null, defaultDeprecations));
484
485  /**
486   * Adds a set of deprecated keys to the global deprecations.
487   *
488   * This method is lockless.  It works by means of creating a new
489   * DeprecationContext based on the old one, and then atomically swapping in
490   * the new context.  If someone else updated the context in between us reading
491   * the old context and swapping in the new one, we try again until we win the
492   * race.
493   *
494   * @param deltas   The deprecations to add.
495   */
496  public static void addDeprecations(DeprecationDelta[] deltas) {
497    DeprecationContext prev, next;
498    do {
499      prev = deprecationContext.get();
500      next = new DeprecationContext(prev, deltas);
501    } while (!deprecationContext.compareAndSet(prev, next));
502  }
503
504  /**
505   * Adds the deprecated key to the global deprecation map.
506   * It does not override any existing entries in the deprecation map.
507   * This is to be used only by the developers in order to add deprecation of
508   * keys, and attempts to call this method after loading resources once,
509   * would lead to <tt>UnsupportedOperationException</tt>
510   * 
511   * If a key is deprecated in favor of multiple keys, they are all treated as 
512   * aliases of each other, and setting any one of them resets all the others 
513   * to the new value.
514   *
515   * If you have multiple deprecation entries to add, it is more efficient to
516   * use #addDeprecations(DeprecationDelta[] deltas) instead.
517   * 
518   * @param key
519   * @param newKeys
520   * @param customMessage
521   * @deprecated use {@link #addDeprecation(String key, String newKey,
522      String customMessage)} instead
523   */
524  @Deprecated
525  public static void addDeprecation(String key, String[] newKeys,
526      String customMessage) {
527    addDeprecations(new DeprecationDelta[] {
528      new DeprecationDelta(key, newKeys, customMessage)
529    });
530  }
531
532  /**
533   * Adds the deprecated key to the global deprecation map.
534   * It does not override any existing entries in the deprecation map.
535   * This is to be used only by the developers in order to add deprecation of
536   * keys, and attempts to call this method after loading resources once,
537   * would lead to <tt>UnsupportedOperationException</tt>
538   * 
539   * If you have multiple deprecation entries to add, it is more efficient to
540   * use #addDeprecations(DeprecationDelta[] deltas) instead.
541   *
542   * @param key
543   * @param newKey
544   * @param customMessage
545   */
546  public static void addDeprecation(String key, String newKey,
547              String customMessage) {
548          addDeprecation(key, new String[] {newKey}, customMessage);
549  }
550
551  /**
552   * Adds the deprecated key to the global deprecation map when no custom
553   * message is provided.
554   * It does not override any existing entries in the deprecation map.
555   * This is to be used only by the developers in order to add deprecation of
556   * keys, and attempts to call this method after loading resources once,
557   * would lead to <tt>UnsupportedOperationException</tt>
558   * 
559   * If a key is deprecated in favor of multiple keys, they are all treated as 
560   * aliases of each other, and setting any one of them resets all the others 
561   * to the new value.
562   * 
563   * If you have multiple deprecation entries to add, it is more efficient to
564   * use #addDeprecations(DeprecationDelta[] deltas) instead.
565   *
566   * @param key Key that is to be deprecated
567   * @param newKeys list of keys that take up the values of deprecated key
568   * @deprecated use {@link #addDeprecation(String key, String newKey)} instead
569   */
570  @Deprecated
571  public static void addDeprecation(String key, String[] newKeys) {
572    addDeprecation(key, newKeys, null);
573  }
574  
575  /**
576   * Adds the deprecated key to the global deprecation map when no custom
577   * message is provided.
578   * It does not override any existing entries in the deprecation map.
579   * This is to be used only by the developers in order to add deprecation of
580   * keys, and attempts to call this method after loading resources once,
581   * would lead to <tt>UnsupportedOperationException</tt>
582   * 
583   * If you have multiple deprecation entries to add, it is more efficient to
584   * use #addDeprecations(DeprecationDelta[] deltas) instead.
585   *
586   * @param key Key that is to be deprecated
587   * @param newKey key that takes up the value of deprecated key
588   */
589  public static void addDeprecation(String key, String newKey) {
590    addDeprecation(key, new String[] {newKey}, null);
591  }
592  
593  /**
594   * checks whether the given <code>key</code> is deprecated.
595   * 
596   * @param key the parameter which is to be checked for deprecation
597   * @return <code>true</code> if the key is deprecated and 
598   *         <code>false</code> otherwise.
599   */
600  public static boolean isDeprecated(String key) {
601    return deprecationContext.get().getDeprecatedKeyMap().containsKey(key);
602  }
603
604  /**
605   * Sets all deprecated properties that are not currently set but have a
606   * corresponding new property that is set. Useful for iterating the
607   * properties when all deprecated properties for currently set properties
608   * need to be present.
609   */
610  public void setDeprecatedProperties() {
611    DeprecationContext deprecations = deprecationContext.get();
612    Properties props = getProps();
613    Properties overlay = getOverlay();
614    for (Map.Entry<String, DeprecatedKeyInfo> entry :
615        deprecations.getDeprecatedKeyMap().entrySet()) {
616      String depKey = entry.getKey();
617      if (!overlay.contains(depKey)) {
618        for (String newKey : entry.getValue().newKeys) {
619          String val = overlay.getProperty(newKey);
620          if (val != null) {
621            props.setProperty(depKey, val);
622            overlay.setProperty(depKey, val);
623            break;
624          }
625        }
626      }
627    }
628  }
629
630  /**
631   * Checks for the presence of the property <code>name</code> in the
632   * deprecation map. Returns the first of the list of new keys if present
633   * in the deprecation map or the <code>name</code> itself. If the property
634   * is not presently set but the property map contains an entry for the
635   * deprecated key, the value of the deprecated key is set as the value for
636   * the provided property name.
637   *
638   * @param name the property name
639   * @return the first property in the list of properties mapping
640   *         the <code>name</code> or the <code>name</code> itself.
641   */
642  private String[] handleDeprecation(DeprecationContext deprecations,
643      String name) {
644    if (null != name) {
645      name = name.trim();
646    }
647    ArrayList<String > names = new ArrayList<String>();
648        if (isDeprecated(name)) {
649      DeprecatedKeyInfo keyInfo = deprecations.getDeprecatedKeyMap().get(name);
650      warnOnceIfDeprecated(deprecations, name);
651      for (String newKey : keyInfo.newKeys) {
652        if(newKey != null) {
653          names.add(newKey);
654        }
655      }
656    }
657    if(names.size() == 0) {
658        names.add(name);
659    }
660    for(String n : names) {
661          String deprecatedKey = deprecations.getReverseDeprecatedKeyMap().get(n);
662          if (deprecatedKey != null && !getOverlay().containsKey(n) &&
663              getOverlay().containsKey(deprecatedKey)) {
664            getProps().setProperty(n, getOverlay().getProperty(deprecatedKey));
665            getOverlay().setProperty(n, getOverlay().getProperty(deprecatedKey));
666          }
667    }
668    return names.toArray(new String[names.size()]);
669  }
670 
671  private void handleDeprecation() {
672    LOG.debug("Handling deprecation for all properties in config...");
673    DeprecationContext deprecations = deprecationContext.get();
674    Set<Object> keys = new HashSet<Object>();
675    keys.addAll(getProps().keySet());
676    for (Object item: keys) {
677      LOG.debug("Handling deprecation for " + (String)item);
678      handleDeprecation(deprecations, (String)item);
679    }
680  }
681 
682  static{
683    //print deprecation warning if hadoop-site.xml is found in classpath
684    ClassLoader cL = Thread.currentThread().getContextClassLoader();
685    if (cL == null) {
686      cL = Configuration.class.getClassLoader();
687    }
688    if(cL.getResource("hadoop-site.xml")!=null) {
689      LOG.warn("DEPRECATED: hadoop-site.xml found in the classpath. " +
690          "Usage of hadoop-site.xml is deprecated. Instead use core-site.xml, "
691          + "mapred-site.xml and hdfs-site.xml to override properties of " +
692          "core-default.xml, mapred-default.xml and hdfs-default.xml " +
693          "respectively");
694    }
695    addDefaultResource("core-default.xml");
696    addDefaultResource("core-site.xml");
697  }
698  
699  private Properties properties;
700  private Properties overlay;
701  private ClassLoader classLoader;
702  {
703    classLoader = Thread.currentThread().getContextClassLoader();
704    if (classLoader == null) {
705      classLoader = Configuration.class.getClassLoader();
706    }
707  }
708  
709  /** A new configuration. */
710  public Configuration() {
711    this(true);
712  }
713
714  /** A new configuration where the behavior of reading from the default 
715   * resources can be turned off.
716   * 
717   * If the parameter {@code loadDefaults} is false, the new instance
718   * will not load resources from the default files. 
719   * @param loadDefaults specifies whether to load from the default files
720   */
721  public Configuration(boolean loadDefaults) {
722    this.loadDefaults = loadDefaults;
723    updatingResource = new ConcurrentHashMap<String, String[]>();
724    synchronized(Configuration.class) {
725      REGISTRY.put(this, null);
726    }
727  }
728  
729  /** 
730   * A new configuration with the same settings cloned from another.
731   * 
732   * @param other the configuration from which to clone settings.
733   */
734  @SuppressWarnings("unchecked")
735  public Configuration(Configuration other) {
736   this.resources = (ArrayList<Resource>) other.resources.clone();
737   synchronized(other) {
738     if (other.properties != null) {
739       this.properties = (Properties)other.properties.clone();
740     }
741
742     if (other.overlay!=null) {
743       this.overlay = (Properties)other.overlay.clone();
744     }
745
746     this.restrictSystemProps = other.restrictSystemProps;
747     this.updatingResource = new ConcurrentHashMap<String, String[]>(
748         other.updatingResource);
749     this.finalParameters = Collections.newSetFromMap(
750         new ConcurrentHashMap<String, Boolean>());
751     this.finalParameters.addAll(other.finalParameters);
752   }
753   
754    synchronized(Configuration.class) {
755      REGISTRY.put(this, null);
756    }
757    this.classLoader = other.classLoader;
758    this.loadDefaults = other.loadDefaults;
759    setQuietMode(other.getQuietMode());
760  }
761  
762  /**
763   * Add a default resource. Resources are loaded in the order of the resources 
764   * added.
765   * @param name file name. File should be present in the classpath.
766   */
767  public static synchronized void addDefaultResource(String name) {
768    if(!defaultResources.contains(name)) {
769      defaultResources.add(name);
770      for(Configuration conf : REGISTRY.keySet()) {
771        if(conf.loadDefaults) {
772          conf.reloadConfiguration();
773        }
774      }
775    }
776  }
777
778  public static void setRestrictSystemPropertiesDefault(boolean val) {
779    restrictSystemPropsDefault = val;
780  }
781
782  public void setRestrictSystemProperties(boolean val) {
783    this.restrictSystemProps = val;
784  }
785
786  /**
787   * Add a configuration resource. 
788   * 
789   * The properties of this resource will override properties of previously 
790   * added resources, unless they were marked <a href="#Final">final</a>. 
791   * 
792   * @param name resource to be added, the classpath is examined for a file 
793   *             with that name.
794   */
795  public void addResource(String name) {
796    addResourceObject(new Resource(name));
797  }
798
799  public void addResource(String name, boolean restrictedParser) {
800    addResourceObject(new Resource(name, restrictedParser));
801  }
802
803  /**
804   * Add a configuration resource. 
805   * 
806   * The properties of this resource will override properties of previously 
807   * added resources, unless they were marked <a href="#Final">final</a>. 
808   * 
809   * @param url url of the resource to be added, the local filesystem is 
810   *            examined directly to find the resource, without referring to 
811   *            the classpath.
812   */
813  public void addResource(URL url) {
814    addResourceObject(new Resource(url));
815  }
816
817  public void addResource(URL url, boolean restrictedParser) {
818    addResourceObject(new Resource(url, restrictedParser));
819  }
820
821  /**
822   * Add a configuration resource. 
823   * 
824   * The properties of this resource will override properties of previously 
825   * added resources, unless they were marked <a href="#Final">final</a>. 
826   * 
827   * @param file file-path of resource to be added, the local filesystem is
828   *             examined directly to find the resource, without referring to 
829   *             the classpath.
830   */
831  public void addResource(Path file) {
832    addResourceObject(new Resource(file));
833  }
834
835  public void addResource(Path file, boolean restrictedParser) {
836    addResourceObject(new Resource(file, restrictedParser));
837  }
838
839  /**
840   * Add a configuration resource. 
841   * 
842   * The properties of this resource will override properties of previously 
843   * added resources, unless they were marked <a href="#Final">final</a>. 
844   * 
845   * WARNING: The contents of the InputStream will be cached, by this method. 
846   * So use this sparingly because it does increase the memory consumption.
847   * 
848   * @param in InputStream to deserialize the object from. In will be read from
849   * when a get or set is called next.  After it is read the stream will be
850   * closed. 
851   */
852  public void addResource(InputStream in) {
853    addResourceObject(new Resource(in));
854  }
855
856  public void addResource(InputStream in, boolean restrictedParser) {
857    addResourceObject(new Resource(in, restrictedParser));
858  }
859
860  /**
861   * Add a configuration resource. 
862   * 
863   * The properties of this resource will override properties of previously 
864   * added resources, unless they were marked <a href="#Final">final</a>. 
865   * 
866   * @param in InputStream to deserialize the object from.
867   * @param name the name of the resource because InputStream.toString is not
868   * very descriptive some times.  
869   */
870  public void addResource(InputStream in, String name) {
871    addResourceObject(new Resource(in, name));
872  }
873
874  public void addResource(InputStream in, String name,
875      boolean restrictedParser) {
876    addResourceObject(new Resource(in, name, restrictedParser));
877  }
878
879  /**
880   * Add a configuration resource.
881   *
882   * The properties of this resource will override properties of previously
883   * added resources, unless they were marked <a href="#Final">final</a>.
884   *
885   * @param conf Configuration object from which to load properties
886   */
887  public void addResource(Configuration conf) {
888    addResourceObject(new Resource(conf.getProps()));
889  }
890
891  
892  
893  /**
894   * Reload configuration from previously added resources.
895   *
896   * This method will clear all the configuration read from the added 
897   * resources, and final parameters. This will make the resources to 
898   * be read again before accessing the values. Values that are added
899   * via set methods will overlay values read from the resources.
900   */
901  public synchronized void reloadConfiguration() {
902    properties = null;                            // trigger reload
903    finalParameters.clear();                      // clear site-limits
904  }
905  
906  private synchronized void addResourceObject(Resource resource) {
907    resources.add(resource);                      // add to resources
908    restrictSystemProps |= resource.isParserRestricted();
909    reloadConfiguration();
910  }
911
912  private static final int MAX_SUBST = 20;
913
914  private static final int SUB_START_IDX = 0;
915  private static final int SUB_END_IDX = SUB_START_IDX + 1;
916
917  /**
918   * This is a manual implementation of the following regex
919   * "\\$\\{[^\\}\\$\u0020]+\\}". It can be 15x more efficient than
920   * a regex matcher as demonstrated by HADOOP-11506. This is noticeable with
921   * Hadoop apps building on the assumption Configuration#get is an O(1)
922   * hash table lookup, especially when the eval is a long string.
923   *
924   * @param eval a string that may contain variables requiring expansion.
925   * @return a 2-element int array res such that
926   * eval.substring(res[0], res[1]) is "var" for the left-most occurrence of
927   * ${var} in eval. If no variable is found -1, -1 is returned.
928   */
929  private static int[] findSubVariable(String eval) {
930    int[] result = {-1, -1};
931
932    int matchStart;
933    int leftBrace;
934
935    // scanning for a brace first because it's less frequent than $
936    // that can occur in nested class names
937    //
938    match_loop:
939    for (matchStart = 1, leftBrace = eval.indexOf('{', matchStart);
940         // minimum left brace position (follows '$')
941         leftBrace > 0
942         // right brace of a smallest valid expression "${c}"
943         && leftBrace + "{c".length() < eval.length();
944         leftBrace = eval.indexOf('{', matchStart)) {
945      int matchedLen = 0;
946      if (eval.charAt(leftBrace - 1) == '$') {
947        int subStart = leftBrace + 1; // after '{'
948        for (int i = subStart; i < eval.length(); i++) {
949          switch (eval.charAt(i)) {
950            case '}':
951              if (matchedLen > 0) { // match
952                result[SUB_START_IDX] = subStart;
953                result[SUB_END_IDX] = subStart + matchedLen;
954                break match_loop;
955              }
956              // fall through to skip 1 char
957            case ' ':
958            case '$':
959              matchStart = i + 1;
960              continue match_loop;
961            default:
962              matchedLen++;
963          }
964        }
965        // scanned from "${"  to the end of eval, and no reset via ' ', '$':
966        //    no match!
967        break match_loop;
968      } else {
969        // not a start of a variable
970        //
971        matchStart = leftBrace + 1;
972      }
973    }
974    return result;
975  }
976
977  /**
978   * Attempts to repeatedly expand the value {@code expr} by replacing the
979   * left-most substring of the form "${var}" in the following precedence order
980   * <ol>
981   *   <li>by the value of the Java system property "var" if defined</li>
982   *   <li>by the value of the configuration key "var" if defined</li>
983   * </ol>
984   *
985   * If var is unbounded the current state of expansion "prefix${var}suffix" is
986   * returned.
987   *
988   * @param expr the literal value of a config key
989   * @return null if expr is null, otherwise the value resulting from expanding
990   * expr using the algorithm above.
991   * @throws IllegalArgumentException when more than
992   * {@link Configuration#MAX_SUBST} replacements are required
993   */
994  private String substituteVars(String expr) {
995    if (expr == null) {
996      return null;
997    }
998    String eval = expr;
999    for (int s = 0; s < MAX_SUBST; s++) {
1000      final int[] varBounds = findSubVariable(eval);
1001      if (varBounds[SUB_START_IDX] == -1) {
1002        return eval;
1003      }
1004      final String var = eval.substring(varBounds[SUB_START_IDX],
1005          varBounds[SUB_END_IDX]);
1006      String val = null;
1007      if (!restrictSystemProps) {
1008        try {
1009          val = System.getProperty(var);
1010        } catch (SecurityException se) {
1011          LOG.warn("Unexpected SecurityException in Configuration", se);
1012        }
1013      }
1014      if (val == null) {
1015        val = getRaw(var);
1016      }
1017      if (val == null) {
1018        return eval; // return literal ${var}: var is unbound
1019      }
1020      final int dollar = varBounds[SUB_START_IDX] - "${".length();
1021      final int afterRightBrace = varBounds[SUB_END_IDX] + "}".length();
1022      // substitute
1023      eval = eval.substring(0, dollar)
1024             + val
1025             + eval.substring(afterRightBrace);
1026    }
1027    throw new IllegalStateException("Variable substitution depth too large: " 
1028                                    + MAX_SUBST + " " + expr);
1029  }
1030  
1031  /**
1032   * Get the value of the <code>name</code> property, <code>null</code> if
1033   * no such property exists. If the key is deprecated, it returns the value of
1034   * the first key which replaces the deprecated key and is not null.
1035   * 
1036   * Values are processed for <a href="#VariableExpansion">variable expansion</a> 
1037   * before being returned. 
1038   * 
1039   * @param name the property name, will be trimmed before get value.
1040   * @return the value of the <code>name</code> or its replacing property, 
1041   *         or null if no such property exists.
1042   */
1043  public String get(String name) {
1044    String[] names = handleDeprecation(deprecationContext.get(), name);
1045    String result = null;
1046    for(String n : names) {
1047      result = substituteVars(getProps().getProperty(n));
1048    }
1049    return result;
1050  }
1051
1052  /**
1053   * Set Configuration to allow keys without values during setup.  Intended
1054   * for use during testing.
1055   *
1056   * @param val If true, will allow Configuration to store keys without values
1057   */
1058  @VisibleForTesting
1059  public void setAllowNullValueProperties( boolean val ) {
1060    this.allowNullValueProperties = val;
1061  }
1062
1063  public void setRestrictSystemProps(boolean val) {
1064    this.restrictSystemProps = val;
1065  }
1066
1067  /**
1068   * Return existence of the <code>name</code> property, but only for
1069   * names which have no valid value, usually non-existent or commented
1070   * out in XML.
1071   *
1072   * @param name the property name
1073   * @return true if the property <code>name</code> exists without value
1074   */
1075  @VisibleForTesting
1076  public boolean onlyKeyExists(String name) {
1077    String[] names = handleDeprecation(deprecationContext.get(), name);
1078    for(String n : names) {
1079      if ( getProps().getProperty(n,DEFAULT_STRING_CHECK)
1080               .equals(DEFAULT_STRING_CHECK) ) {
1081        return true;
1082      }
1083    }
1084    return false;
1085  }
1086
1087  /**
1088   * Get the value of the <code>name</code> property as a trimmed <code>String</code>, 
1089   * <code>null</code> if no such property exists. 
1090   * If the key is deprecated, it returns the value of
1091   * the first key which replaces the deprecated key and is not null
1092   * 
1093   * Values are processed for <a href="#VariableExpansion">variable expansion</a> 
1094   * before being returned. 
1095   * 
1096   * @param name the property name.
1097   * @return the value of the <code>name</code> or its replacing property, 
1098   *         or null if no such property exists.
1099   */
1100  public String getTrimmed(String name) {
1101    String value = get(name);
1102    
1103    if (null == value) {
1104      return null;
1105    } else {
1106      return value.trim();
1107    }
1108  }
1109  
1110  /**
1111   * Get the value of the <code>name</code> property as a trimmed <code>String</code>, 
1112   * <code>defaultValue</code> if no such property exists. 
1113   * See @{Configuration#getTrimmed} for more details.
1114   * 
1115   * @param name          the property name.
1116   * @param defaultValue  the property default value.
1117   * @return              the value of the <code>name</code> or defaultValue
1118   *                      if it is not set.
1119   */
1120  public String getTrimmed(String name, String defaultValue) {
1121    String ret = getTrimmed(name);
1122    return ret == null ? defaultValue : ret;
1123  }
1124
1125  /**
1126   * Get the value of the <code>name</code> property, without doing
1127   * <a href="#VariableExpansion">variable expansion</a>.If the key is 
1128   * deprecated, it returns the value of the first key which replaces 
1129   * the deprecated key and is not null.
1130   * 
1131   * @param name the property name.
1132   * @return the value of the <code>name</code> property or 
1133   *         its replacing property and null if no such property exists.
1134   */
1135  public String getRaw(String name) {
1136    String[] names = handleDeprecation(deprecationContext.get(), name);
1137    String result = null;
1138    for(String n : names) {
1139      result = getProps().getProperty(n);
1140    }
1141    return result;
1142  }
1143
1144  /**
1145   * Returns alternative names (non-deprecated keys or previously-set deprecated keys)
1146   * for a given non-deprecated key.
1147   * If the given key is deprecated, return null.
1148   *
1149   * @param name property name.
1150   * @return alternative names.
1151   */
1152  private String[] getAlternativeNames(String name) {
1153    String altNames[] = null;
1154    DeprecatedKeyInfo keyInfo = null;
1155    DeprecationContext cur = deprecationContext.get();
1156    String depKey = cur.getReverseDeprecatedKeyMap().get(name);
1157    if(depKey != null) {
1158      keyInfo = cur.getDeprecatedKeyMap().get(depKey);
1159      if(keyInfo.newKeys.length > 0) {
1160        if(getProps().containsKey(depKey)) {
1161          //if deprecated key is previously set explicitly
1162          List<String> list = new ArrayList<String>();
1163          list.addAll(Arrays.asList(keyInfo.newKeys));
1164          list.add(depKey);
1165          altNames = list.toArray(new String[list.size()]);
1166        }
1167        else {
1168          altNames = keyInfo.newKeys;
1169        }
1170      }
1171    }
1172    return altNames;
1173  }
1174
1175  /** 
1176   * Set the <code>value</code> of the <code>name</code> property. If 
1177   * <code>name</code> is deprecated or there is a deprecated name associated to it,
1178   * it sets the value to both names. Name will be trimmed before put into
1179   * configuration.
1180   * 
1181   * @param name property name.
1182   * @param value property value.
1183   */
1184  public void set(String name, String value) {
1185    set(name, value, null);
1186  }
1187  
1188  /** 
1189   * Set the <code>value</code> of the <code>name</code> property. If 
1190   * <code>name</code> is deprecated, it also sets the <code>value</code> to
1191   * the keys that replace the deprecated key. Name will be trimmed before put
1192   * into configuration.
1193   *
1194   * @param name property name.
1195   * @param value property value.
1196   * @param source the place that this configuration value came from 
1197   * (For debugging).
1198   * @throws IllegalArgumentException when the value or name is null.
1199   */
1200  public void set(String name, String value, String source) {
1201    Preconditions.checkArgument(
1202        name != null,
1203        "Property name must not be null");
1204    Preconditions.checkArgument(
1205        value != null,
1206        "The value of property " + name + " must not be null");
1207    name = name.trim();
1208    DeprecationContext deprecations = deprecationContext.get();
1209    if (deprecations.getDeprecatedKeyMap().isEmpty()) {
1210      getProps();
1211    }
1212    getOverlay().setProperty(name, value);
1213    getProps().setProperty(name, value);
1214    String newSource = (source == null ? "programatically" : source);
1215
1216    if (!isDeprecated(name)) {
1217      updatingResource.put(name, new String[] {newSource});
1218      String[] altNames = getAlternativeNames(name);
1219      if(altNames != null) {
1220        for(String n: altNames) {
1221          if(!n.equals(name)) {
1222            getOverlay().setProperty(n, value);
1223            getProps().setProperty(n, value);
1224            updatingResource.put(n, new String[] {newSource});
1225          }
1226        }
1227      }
1228    }
1229    else {
1230      String[] names = handleDeprecation(deprecationContext.get(), name);
1231      String altSource = "because " + name + " is deprecated";
1232      for(String n : names) {
1233        getOverlay().setProperty(n, value);
1234        getProps().setProperty(n, value);
1235        updatingResource.put(n, new String[] {altSource});
1236      }
1237    }
1238  }
1239
1240  private void warnOnceIfDeprecated(DeprecationContext deprecations, String name) {
1241    DeprecatedKeyInfo keyInfo = deprecations.getDeprecatedKeyMap().get(name);
1242    if (keyInfo != null && !keyInfo.getAndSetAccessed()) {
1243      LOG_DEPRECATION.info(keyInfo.getWarningMessage(name));
1244    }
1245  }
1246
1247  /**
1248   * Unset a previously set property.
1249   */
1250  public synchronized void unset(String name) {
1251    String[] names = null;
1252    if (!isDeprecated(name)) {
1253      names = getAlternativeNames(name);
1254      if(names == null) {
1255          names = new String[]{name};
1256      }
1257    }
1258    else {
1259      names = handleDeprecation(deprecationContext.get(), name);
1260    }
1261
1262    for(String n: names) {
1263      getOverlay().remove(n);
1264      getProps().remove(n);
1265    }
1266  }
1267
1268  /**
1269   * Sets a property if it is currently unset.
1270   * @param name the property name
1271   * @param value the new value
1272   */
1273  public synchronized void setIfUnset(String name, String value) {
1274    if (get(name) == null) {
1275      set(name, value);
1276    }
1277  }
1278  
1279  private synchronized Properties getOverlay() {
1280    if (overlay==null){
1281      overlay=new Properties();
1282    }
1283    return overlay;
1284  }
1285
1286  /** 
1287   * Get the value of the <code>name</code>. If the key is deprecated,
1288   * it returns the value of the first key which replaces the deprecated key
1289   * and is not null.
1290   * If no such property exists,
1291   * then <code>defaultValue</code> is returned.
1292   * 
1293   * @param name property name, will be trimmed before get value.
1294   * @param defaultValue default value.
1295   * @return property value, or <code>defaultValue</code> if the property 
1296   *         doesn't exist.                    
1297   */
1298  public String get(String name, String defaultValue) {
1299    String[] names = handleDeprecation(deprecationContext.get(), name);
1300    String result = null;
1301    for(String n : names) {
1302      result = substituteVars(getProps().getProperty(n, defaultValue));
1303    }
1304    return result;
1305  }
1306
1307  /** 
1308   * Get the value of the <code>name</code> property as an <code>int</code>.
1309   *   
1310   * If no such property exists, the provided default value is returned,
1311   * or if the specified value is not a valid <code>int</code>,
1312   * then an error is thrown.
1313   * 
1314   * @param name property name.
1315   * @param defaultValue default value.
1316   * @throws NumberFormatException when the value is invalid
1317   * @return property value as an <code>int</code>, 
1318   *         or <code>defaultValue</code>. 
1319   */
1320  public int getInt(String name, int defaultValue) {
1321    String valueString = getTrimmed(name);
1322    if (valueString == null)
1323      return defaultValue;
1324    String hexString = getHexDigits(valueString);
1325    if (hexString != null) {
1326      return Integer.parseInt(hexString, 16);
1327    }
1328    return Integer.parseInt(valueString);
1329  }
1330  
1331  /**
1332   * Get the value of the <code>name</code> property as a set of comma-delimited
1333   * <code>int</code> values.
1334   * 
1335   * If no such property exists, an empty array is returned.
1336   * 
1337   * @param name property name
1338   * @return property value interpreted as an array of comma-delimited
1339   *         <code>int</code> values
1340   */
1341  public int[] getInts(String name) {
1342    String[] strings = getTrimmedStrings(name);
1343    int[] ints = new int[strings.length];
1344    for (int i = 0; i < strings.length; i++) {
1345      ints[i] = Integer.parseInt(strings[i]);
1346    }
1347    return ints;
1348  }
1349
1350  /** 
1351   * Set the value of the <code>name</code> property to an <code>int</code>.
1352   * 
1353   * @param name property name.
1354   * @param value <code>int</code> value of the property.
1355   */
1356  public void setInt(String name, int value) {
1357    set(name, Integer.toString(value));
1358  }
1359
1360
1361  /** 
1362   * Get the value of the <code>name</code> property as a <code>long</code>.  
1363   * If no such property exists, the provided default value is returned,
1364   * or if the specified value is not a valid <code>long</code>,
1365   * then an error is thrown.
1366   * 
1367   * @param name property name.
1368   * @param defaultValue default value.
1369   * @throws NumberFormatException when the value is invalid
1370   * @return property value as a <code>long</code>, 
1371   *         or <code>defaultValue</code>. 
1372   */
1373  public long getLong(String name, long defaultValue) {
1374    String valueString = getTrimmed(name);
1375    if (valueString == null)
1376      return defaultValue;
1377    String hexString = getHexDigits(valueString);
1378    if (hexString != null) {
1379      return Long.parseLong(hexString, 16);
1380    }
1381    return Long.parseLong(valueString);
1382  }
1383
1384  /**
1385   * Get the value of the <code>name</code> property as a <code>long</code> or
1386   * human readable format. If no such property exists, the provided default
1387   * value is returned, or if the specified value is not a valid
1388   * <code>long</code> or human readable format, then an error is thrown. You
1389   * can use the following suffix (case insensitive): k(kilo), m(mega), g(giga),
1390   * t(tera), p(peta), e(exa)
1391   *
1392   * @param name property name.
1393   * @param defaultValue default value.
1394   * @throws NumberFormatException when the value is invalid
1395   * @return property value as a <code>long</code>,
1396   *         or <code>defaultValue</code>.
1397   */
1398  public long getLongBytes(String name, long defaultValue) {
1399    String valueString = getTrimmed(name);
1400    if (valueString == null)
1401      return defaultValue;
1402    return StringUtils.TraditionalBinaryPrefix.string2long(valueString);
1403  }
1404
1405  private String getHexDigits(String value) {
1406    boolean negative = false;
1407    String str = value;
1408    String hexString = null;
1409    if (value.startsWith("-")) {
1410      negative = true;
1411      str = value.substring(1);
1412    }
1413    if (str.startsWith("0x") || str.startsWith("0X")) {
1414      hexString = str.substring(2);
1415      if (negative) {
1416        hexString = "-" + hexString;
1417      }
1418      return hexString;
1419    }
1420    return null;
1421  }
1422  
1423  /** 
1424   * Set the value of the <code>name</code> property to a <code>long</code>.
1425   * 
1426   * @param name property name.
1427   * @param value <code>long</code> value of the property.
1428   */
1429  public void setLong(String name, long value) {
1430    set(name, Long.toString(value));
1431  }
1432
1433  /** 
1434   * Get the value of the <code>name</code> property as a <code>float</code>.  
1435   * If no such property exists, the provided default value is returned,
1436   * or if the specified value is not a valid <code>float</code>,
1437   * then an error is thrown.
1438   *
1439   * @param name property name.
1440   * @param defaultValue default value.
1441   * @throws NumberFormatException when the value is invalid
1442   * @return property value as a <code>float</code>, 
1443   *         or <code>defaultValue</code>. 
1444   */
1445  public float getFloat(String name, float defaultValue) {
1446    String valueString = getTrimmed(name);
1447    if (valueString == null)
1448      return defaultValue;
1449    return Float.parseFloat(valueString);
1450  }
1451
1452  /**
1453   * Set the value of the <code>name</code> property to a <code>float</code>.
1454   * 
1455   * @param name property name.
1456   * @param value property value.
1457   */
1458  public void setFloat(String name, float value) {
1459    set(name,Float.toString(value));
1460  }
1461
1462  /** 
1463   * Get the value of the <code>name</code> property as a <code>double</code>.  
1464   * If no such property exists, the provided default value is returned,
1465   * or if the specified value is not a valid <code>double</code>,
1466   * then an error is thrown.
1467   *
1468   * @param name property name.
1469   * @param defaultValue default value.
1470   * @throws NumberFormatException when the value is invalid
1471   * @return property value as a <code>double</code>, 
1472   *         or <code>defaultValue</code>. 
1473   */
1474  public double getDouble(String name, double defaultValue) {
1475    String valueString = getTrimmed(name);
1476    if (valueString == null)
1477      return defaultValue;
1478    return Double.parseDouble(valueString);
1479  }
1480
1481  /**
1482   * Set the value of the <code>name</code> property to a <code>double</code>.
1483   * 
1484   * @param name property name.
1485   * @param value property value.
1486   */
1487  public void setDouble(String name, double value) {
1488    set(name,Double.toString(value));
1489  }
1490 
1491  /** 
1492   * Get the value of the <code>name</code> property as a <code>boolean</code>.  
1493   * If no such property is specified, or if the specified value is not a valid
1494   * <code>boolean</code>, then <code>defaultValue</code> is returned.
1495   * 
1496   * @param name property name.
1497   * @param defaultValue default value.
1498   * @return property value as a <code>boolean</code>, 
1499   *         or <code>defaultValue</code>. 
1500   */
1501  public boolean getBoolean(String name, boolean defaultValue) {
1502    String valueString = getTrimmed(name);
1503    if (null == valueString || valueString.isEmpty()) {
1504      return defaultValue;
1505    }
1506
1507    if (StringUtils.equalsIgnoreCase("true", valueString))
1508      return true;
1509    else if (StringUtils.equalsIgnoreCase("false", valueString))
1510      return false;
1511    else return defaultValue;
1512  }
1513
1514  /** 
1515   * Set the value of the <code>name</code> property to a <code>boolean</code>.
1516   * 
1517   * @param name property name.
1518   * @param value <code>boolean</code> value of the property.
1519   */
1520  public void setBoolean(String name, boolean value) {
1521    set(name, Boolean.toString(value));
1522  }
1523
1524  /**
1525   * Set the given property, if it is currently unset.
1526   * @param name property name
1527   * @param value new value
1528   */
1529  public void setBooleanIfUnset(String name, boolean value) {
1530    setIfUnset(name, Boolean.toString(value));
1531  }
1532
1533  /**
1534   * Set the value of the <code>name</code> property to the given type. This
1535   * is equivalent to <code>set(&lt;name&gt;, value.toString())</code>.
1536   * @param name property name
1537   * @param value new value
1538   */
1539  public <T extends Enum<T>> void setEnum(String name, T value) {
1540    set(name, value.toString());
1541  }
1542
1543  /**
1544   * Return value matching this enumerated type.
1545   * Note that the returned value is trimmed by this method.
1546   * @param name Property name
1547   * @param defaultValue Value returned if no mapping exists
1548   * @throws IllegalArgumentException If mapping is illegal for the type
1549   * provided
1550   */
1551  public <T extends Enum<T>> T getEnum(String name, T defaultValue) {
1552    final String val = getTrimmed(name);
1553    return null == val
1554      ? defaultValue
1555      : Enum.valueOf(defaultValue.getDeclaringClass(), val);
1556  }
1557
1558  enum ParsedTimeDuration {
1559    NS {
1560      TimeUnit unit() { return TimeUnit.NANOSECONDS; }
1561      String suffix() { return "ns"; }
1562    },
1563    US {
1564      TimeUnit unit() { return TimeUnit.MICROSECONDS; }
1565      String suffix() { return "us"; }
1566    },
1567    MS {
1568      TimeUnit unit() { return TimeUnit.MILLISECONDS; }
1569      String suffix() { return "ms"; }
1570    },
1571    S {
1572      TimeUnit unit() { return TimeUnit.SECONDS; }
1573      String suffix() { return "s"; }
1574    },
1575    M {
1576      TimeUnit unit() { return TimeUnit.MINUTES; }
1577      String suffix() { return "m"; }
1578    },
1579    H {
1580      TimeUnit unit() { return TimeUnit.HOURS; }
1581      String suffix() { return "h"; }
1582    },
1583    D {
1584      TimeUnit unit() { return TimeUnit.DAYS; }
1585      String suffix() { return "d"; }
1586    };
1587    abstract TimeUnit unit();
1588    abstract String suffix();
1589    static ParsedTimeDuration unitFor(String s) {
1590      for (ParsedTimeDuration ptd : values()) {
1591        // iteration order is in decl order, so SECONDS matched last
1592        if (s.endsWith(ptd.suffix())) {
1593          return ptd;
1594        }
1595      }
1596      return null;
1597    }
1598    static ParsedTimeDuration unitFor(TimeUnit unit) {
1599      for (ParsedTimeDuration ptd : values()) {
1600        if (ptd.unit() == unit) {
1601          return ptd;
1602        }
1603      }
1604      return null;
1605    }
1606  }
1607
1608  /**
1609   * Set the value of <code>name</code> to the given time duration. This
1610   * is equivalent to <code>set(&lt;name&gt;, value + &lt;time suffix&gt;)</code>.
1611   * @param name Property name
1612   * @param value Time duration
1613   * @param unit Unit of time
1614   */
1615  public void setTimeDuration(String name, long value, TimeUnit unit) {
1616    set(name, value + ParsedTimeDuration.unitFor(unit).suffix());
1617  }
1618
1619  /**
1620   * Return time duration in the given time unit. Valid units are encoded in
1621   * properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds
1622   * (ms), seconds (s), minutes (m), hours (h), and days (d).
1623   * @param name Property name
1624   * @param defaultValue Value returned if no mapping exists.
1625   * @param unit Unit to convert the stored property, if it exists.
1626   * @throws NumberFormatException If the property stripped of its unit is not
1627   *         a number
1628   */
1629  public long getTimeDuration(String name, long defaultValue, TimeUnit unit) {
1630    String vStr = get(name);
1631    if (null == vStr) {
1632      return defaultValue;
1633    }
1634    vStr = vStr.trim();
1635    ParsedTimeDuration vUnit = ParsedTimeDuration.unitFor(vStr);
1636    if (null == vUnit) {
1637      LOG.warn("No unit for " + name + "(" + vStr + ") assuming " + unit);
1638      vUnit = ParsedTimeDuration.unitFor(unit);
1639    } else {
1640      vStr = vStr.substring(0, vStr.lastIndexOf(vUnit.suffix()));
1641    }
1642    return unit.convert(Long.parseLong(vStr), vUnit.unit());
1643  }
1644
1645  /**
1646   * Get the value of the <code>name</code> property as a <code>Pattern</code>.
1647   * If no such property is specified, or if the specified value is not a valid
1648   * <code>Pattern</code>, then <code>DefaultValue</code> is returned.
1649   * Note that the returned value is NOT trimmed by this method.
1650   *
1651   * @param name property name
1652   * @param defaultValue default value
1653   * @return property value as a compiled Pattern, or defaultValue
1654   */
1655  public Pattern getPattern(String name, Pattern defaultValue) {
1656    String valString = get(name);
1657    if (null == valString || valString.isEmpty()) {
1658      return defaultValue;
1659    }
1660    try {
1661      return Pattern.compile(valString);
1662    } catch (PatternSyntaxException pse) {
1663      LOG.warn("Regular expression '" + valString + "' for property '" +
1664               name + "' not valid. Using default", pse);
1665      return defaultValue;
1666    }
1667  }
1668
1669  /**
1670   * Set the given property to <code>Pattern</code>.
1671   * If the pattern is passed as null, sets the empty pattern which results in
1672   * further calls to getPattern(...) returning the default value.
1673   *
1674   * @param name property name
1675   * @param pattern new value
1676   */
1677  public void setPattern(String name, Pattern pattern) {
1678    assert pattern != null : "Pattern cannot be null";
1679    set(name, pattern.pattern());
1680  }
1681
1682  /**
1683   * Gets information about why a property was set.  Typically this is the 
1684   * path to the resource objects (file, URL, etc.) the property came from, but
1685   * it can also indicate that it was set programatically, or because of the
1686   * command line.
1687   *
1688   * @param name - The property name to get the source of.
1689   * @return null - If the property or its source wasn't found. Otherwise, 
1690   * returns a list of the sources of the resource.  The older sources are
1691   * the first ones in the list.  So for example if a configuration is set from
1692   * the command line, and then written out to a file that is read back in the
1693   * first entry would indicate that it was set from the command line, while
1694   * the second one would indicate the file that the new configuration was read
1695   * in from.
1696   */
1697  @InterfaceStability.Unstable
1698  public synchronized String[] getPropertySources(String name) {
1699    if (properties == null) {
1700      // If properties is null, it means a resource was newly added
1701      // but the props were cleared so as to load it upon future
1702      // requests. So lets force a load by asking a properties list.
1703      getProps();
1704    }
1705    // Return a null right away if our properties still
1706    // haven't loaded or the resource mapping isn't defined
1707    if (properties == null || updatingResource == null) {
1708      return null;
1709    } else {
1710      String[] source = updatingResource.get(name);
1711      if(source == null) {
1712        return null;
1713      } else {
1714        return Arrays.copyOf(source, source.length);
1715      }
1716    }
1717  }
1718
1719  /**
1720   * A class that represents a set of positive integer ranges. It parses 
1721   * strings of the form: "2-3,5,7-" where ranges are separated by comma and 
1722   * the lower/upper bounds are separated by dash. Either the lower or upper 
1723   * bound may be omitted meaning all values up to or over. So the string 
1724   * above means 2, 3, 5, and 7, 8, 9, ...
1725   */
1726  public static class IntegerRanges implements Iterable<Integer>{
1727    private static class Range {
1728      int start;
1729      int end;
1730    }
1731    
1732    private static class RangeNumberIterator implements Iterator<Integer> {
1733      Iterator<Range> internal;
1734      int at;
1735      int end;
1736
1737      public RangeNumberIterator(List<Range> ranges) {
1738        if (ranges != null) {
1739          internal = ranges.iterator();
1740        }
1741        at = -1;
1742        end = -2;
1743      }
1744      
1745      @Override
1746      public boolean hasNext() {
1747        if (at <= end) {
1748          return true;
1749        } else if (internal != null){
1750          return internal.hasNext();
1751        }
1752        return false;
1753      }
1754
1755      @Override
1756      public Integer next() {
1757        if (at <= end) {
1758          at++;
1759          return at - 1;
1760        } else if (internal != null){
1761          Range found = internal.next();
1762          if (found != null) {
1763            at = found.start;
1764            end = found.end;
1765            at++;
1766            return at - 1;
1767          }
1768        }
1769        return null;
1770      }
1771
1772      @Override
1773      public void remove() {
1774        throw new UnsupportedOperationException();
1775      }
1776    };
1777
1778    List<Range> ranges = new ArrayList<Range>();
1779    
1780    public IntegerRanges() {
1781    }
1782    
1783    public IntegerRanges(String newValue) {
1784      StringTokenizer itr = new StringTokenizer(newValue, ",");
1785      while (itr.hasMoreTokens()) {
1786        String rng = itr.nextToken().trim();
1787        String[] parts = rng.split("-", 3);
1788        if (parts.length < 1 || parts.length > 2) {
1789          throw new IllegalArgumentException("integer range badly formed: " + 
1790                                             rng);
1791        }
1792        Range r = new Range();
1793        r.start = convertToInt(parts[0], 0);
1794        if (parts.length == 2) {
1795          r.end = convertToInt(parts[1], Integer.MAX_VALUE);
1796        } else {
1797          r.end = r.start;
1798        }
1799        if (r.start > r.end) {
1800          throw new IllegalArgumentException("IntegerRange from " + r.start + 
1801                                             " to " + r.end + " is invalid");
1802        }
1803        ranges.add(r);
1804      }
1805    }
1806
1807    /**
1808     * Convert a string to an int treating empty strings as the default value.
1809     * @param value the string value
1810     * @param defaultValue the value for if the string is empty
1811     * @return the desired integer
1812     */
1813    private static int convertToInt(String value, int defaultValue) {
1814      String trim = value.trim();
1815      if (trim.length() == 0) {
1816        return defaultValue;
1817      }
1818      return Integer.parseInt(trim);
1819    }
1820
1821    /**
1822     * Is the given value in the set of ranges
1823     * @param value the value to check
1824     * @return is the value in the ranges?
1825     */
1826    public boolean isIncluded(int value) {
1827      for(Range r: ranges) {
1828        if (r.start <= value && value <= r.end) {
1829          return true;
1830        }
1831      }
1832      return false;
1833    }
1834    
1835    /**
1836     * @return true if there are no values in this range, else false.
1837     */
1838    public boolean isEmpty() {
1839      return ranges == null || ranges.isEmpty();
1840    }
1841    
1842    @Override
1843    public String toString() {
1844      StringBuilder result = new StringBuilder();
1845      boolean first = true;
1846      for(Range r: ranges) {
1847        if (first) {
1848          first = false;
1849        } else {
1850          result.append(',');
1851        }
1852        result.append(r.start);
1853        result.append('-');
1854        result.append(r.end);
1855      }
1856      return result.toString();
1857    }
1858
1859    @Override
1860    public Iterator<Integer> iterator() {
1861      return new RangeNumberIterator(ranges);
1862    }
1863    
1864  }
1865
1866  /**
1867   * Parse the given attribute as a set of integer ranges
1868   * @param name the attribute name
1869   * @param defaultValue the default value if it is not set
1870   * @return a new set of ranges from the configured value
1871   */
1872  public IntegerRanges getRange(String name, String defaultValue) {
1873    return new IntegerRanges(get(name, defaultValue));
1874  }
1875
1876  /** 
1877   * Get the comma delimited values of the <code>name</code> property as 
1878   * a collection of <code>String</code>s.  
1879   * If no such property is specified then empty collection is returned.
1880   * <p>
1881   * This is an optimized version of {@link #getStrings(String)}
1882   * 
1883   * @param name property name.
1884   * @return property value as a collection of <code>String</code>s. 
1885   */
1886  public Collection<String> getStringCollection(String name) {
1887    String valueString = get(name);
1888    return StringUtils.getStringCollection(valueString);
1889  }
1890
1891  /** 
1892   * Get the comma delimited values of the <code>name</code> property as 
1893   * an array of <code>String</code>s.  
1894   * If no such property is specified then <code>null</code> is returned.
1895   * 
1896   * @param name property name.
1897   * @return property value as an array of <code>String</code>s, 
1898   *         or <code>null</code>. 
1899   */
1900  public String[] getStrings(String name) {
1901    String valueString = get(name);
1902    return StringUtils.getStrings(valueString);
1903  }
1904
1905  /** 
1906   * Get the comma delimited values of the <code>name</code> property as 
1907   * an array of <code>String</code>s.  
1908   * If no such property is specified then default value is returned.
1909   * 
1910   * @param name property name.
1911   * @param defaultValue The default value
1912   * @return property value as an array of <code>String</code>s, 
1913   *         or default value. 
1914   */
1915  public String[] getStrings(String name, String... defaultValue) {
1916    String valueString = get(name);
1917    if (valueString == null) {
1918      return defaultValue;
1919    } else {
1920      return StringUtils.getStrings(valueString);
1921    }
1922  }
1923  
1924  /** 
1925   * Get the comma delimited values of the <code>name</code> property as 
1926   * a collection of <code>String</code>s, trimmed of the leading and trailing whitespace.  
1927   * If no such property is specified then empty <code>Collection</code> is returned.
1928   *
1929   * @param name property name.
1930   * @return property value as a collection of <code>String</code>s, or empty <code>Collection</code> 
1931   */
1932  public Collection<String> getTrimmedStringCollection(String name) {
1933    String valueString = get(name);
1934    if (null == valueString) {
1935      Collection<String> empty = new ArrayList<String>();
1936      return empty;
1937    }
1938    return StringUtils.getTrimmedStringCollection(valueString);
1939  }
1940  
1941  /** 
1942   * Get the comma delimited values of the <code>name</code> property as 
1943   * an array of <code>String</code>s, trimmed of the leading and trailing whitespace.
1944   * If no such property is specified then an empty array is returned.
1945   * 
1946   * @param name property name.
1947   * @return property value as an array of trimmed <code>String</code>s, 
1948   *         or empty array. 
1949   */
1950  public String[] getTrimmedStrings(String name) {
1951    String valueString = get(name);
1952    return StringUtils.getTrimmedStrings(valueString);
1953  }
1954
1955  /** 
1956   * Get the comma delimited values of the <code>name</code> property as 
1957   * an array of <code>String</code>s, trimmed of the leading and trailing whitespace.
1958   * If no such property is specified then default value is returned.
1959   * 
1960   * @param name property name.
1961   * @param defaultValue The default value
1962   * @return property value as an array of trimmed <code>String</code>s, 
1963   *         or default value. 
1964   */
1965  public String[] getTrimmedStrings(String name, String... defaultValue) {
1966    String valueString = get(name);
1967    if (null == valueString) {
1968      return defaultValue;
1969    } else {
1970      return StringUtils.getTrimmedStrings(valueString);
1971    }
1972  }
1973
1974  /** 
1975   * Set the array of string values for the <code>name</code> property as 
1976   * as comma delimited values.  
1977   * 
1978   * @param name property name.
1979   * @param values The values
1980   */
1981  public void setStrings(String name, String... values) {
1982    set(name, StringUtils.arrayToString(values));
1983  }
1984
1985  /**
1986   * Get the value for a known password configuration element.
1987   * In order to enable the elimination of clear text passwords in config,
1988   * this method attempts to resolve the property name as an alias through
1989   * the CredentialProvider API and conditionally fallsback to config.
1990   * @param name property name
1991   * @return password
1992   */
1993  public char[] getPassword(String name) throws IOException {
1994    char[] pass = null;
1995
1996    pass = getPasswordFromCredentialProviders(name);
1997
1998    if (pass == null) {
1999      pass = getPasswordFromConfig(name);
2000    }
2001
2002    return pass;
2003  }
2004
2005  /**
2006   * Try and resolve the provided element name as a credential provider
2007   * alias.
2008   * @param name alias of the provisioned credential
2009   * @return password or null if not found
2010   * @throws IOException
2011   */
2012  public char[] getPasswordFromCredentialProviders(String name)
2013      throws IOException {
2014    char[] pass = null;
2015    try {
2016      List<CredentialProvider> providers =
2017          CredentialProviderFactory.getProviders(this);
2018
2019      if (providers != null) {
2020        for (CredentialProvider provider : providers) {
2021          try {
2022            CredentialEntry entry = provider.getCredentialEntry(name);
2023            if (entry != null) {
2024              pass = entry.getCredential();
2025              break;
2026            }
2027          }
2028          catch (IOException ioe) {
2029            throw new IOException("Can't get key " + name + " from key provider" +
2030                        "of type: " + provider.getClass().getName() + ".", ioe);
2031          }
2032        }
2033      }
2034    }
2035    catch (IOException ioe) {
2036      throw new IOException("Configuration problem with provider path.", ioe);
2037    }
2038
2039    return pass;
2040  }
2041
2042  /**
2043   * Fallback to clear text passwords in configuration.
2044   * @param name
2045   * @return clear text password or null
2046   */
2047  protected char[] getPasswordFromConfig(String name) {
2048    char[] pass = null;
2049    if (getBoolean(CredentialProvider.CLEAR_TEXT_FALLBACK, true)) {
2050      String passStr = get(name);
2051      if (passStr != null) {
2052        pass = passStr.toCharArray();
2053      }
2054    }
2055    return pass;
2056  }
2057
2058  /**
2059   * Get the socket address for <code>hostProperty</code> as a
2060   * <code>InetSocketAddress</code>. If <code>hostProperty</code> is
2061   * <code>null</code>, <code>addressProperty</code> will be used. This
2062   * is useful for cases where we want to differentiate between host
2063   * bind address and address clients should use to establish connection.
2064   *
2065   * @param hostProperty bind host property name.
2066   * @param addressProperty address property name.
2067   * @param defaultAddressValue the default value
2068   * @param defaultPort the default port
2069   * @return InetSocketAddress
2070   */
2071  public InetSocketAddress getSocketAddr(
2072      String hostProperty,
2073      String addressProperty,
2074      String defaultAddressValue,
2075      int defaultPort) {
2076
2077    InetSocketAddress bindAddr = getSocketAddr(
2078      addressProperty, defaultAddressValue, defaultPort);
2079
2080    final String host = get(hostProperty);
2081
2082    if (host == null || host.isEmpty()) {
2083      return bindAddr;
2084    }
2085
2086    return NetUtils.createSocketAddr(
2087        host, bindAddr.getPort(), hostProperty);
2088  }
2089
2090  /**
2091   * Get the socket address for <code>name</code> property as a
2092   * <code>InetSocketAddress</code>.
2093   * @param name property name.
2094   * @param defaultAddress the default value
2095   * @param defaultPort the default port
2096   * @return InetSocketAddress
2097   */
2098  public InetSocketAddress getSocketAddr(
2099      String name, String defaultAddress, int defaultPort) {
2100    final String address = getTrimmed(name, defaultAddress);
2101    return NetUtils.createSocketAddr(address, defaultPort, name);
2102  }
2103
2104  /**
2105   * Set the socket address for the <code>name</code> property as
2106   * a <code>host:port</code>.
2107   */
2108  public void setSocketAddr(String name, InetSocketAddress addr) {
2109    set(name, NetUtils.getHostPortString(addr));
2110  }
2111
2112  /**
2113   * Set the socket address a client can use to connect for the
2114   * <code>name</code> property as a <code>host:port</code>.  The wildcard
2115   * address is replaced with the local host's address. If the host and address
2116   * properties are configured the host component of the address will be combined
2117   * with the port component of the addr to generate the address.  This is to allow
2118   * optional control over which host name is used in multi-home bind-host
2119   * cases where a host can have multiple names
2120   * @param hostProperty the bind-host configuration name
2121   * @param addressProperty the service address configuration name
2122   * @param defaultAddressValue the service default address configuration value
2123   * @param addr InetSocketAddress of the service listener
2124   * @return InetSocketAddress for clients to connect
2125   */
2126  public InetSocketAddress updateConnectAddr(
2127      String hostProperty,
2128      String addressProperty,
2129      String defaultAddressValue,
2130      InetSocketAddress addr) {
2131
2132    final String host = get(hostProperty);
2133    final String connectHostPort = getTrimmed(addressProperty, defaultAddressValue);
2134
2135    if (host == null || host.isEmpty() || connectHostPort == null || connectHostPort.isEmpty()) {
2136      //not our case, fall back to original logic
2137      return updateConnectAddr(addressProperty, addr);
2138    }
2139
2140    final String connectHost = connectHostPort.split(":")[0];
2141    // Create connect address using client address hostname and server port.
2142    return updateConnectAddr(addressProperty, NetUtils.createSocketAddrForHost(
2143        connectHost, addr.getPort()));
2144  }
2145  
2146  /**
2147   * Set the socket address a client can use to connect for the
2148   * <code>name</code> property as a <code>host:port</code>.  The wildcard
2149   * address is replaced with the local host's address.
2150   * @param name property name.
2151   * @param addr InetSocketAddress of a listener to store in the given property
2152   * @return InetSocketAddress for clients to connect
2153   */
2154  public InetSocketAddress updateConnectAddr(String name,
2155                                             InetSocketAddress addr) {
2156    final InetSocketAddress connectAddr = NetUtils.getConnectAddress(addr);
2157    setSocketAddr(name, connectAddr);
2158    return connectAddr;
2159  }
2160  
2161  /**
2162   * Load a class by name.
2163   * 
2164   * @param name the class name.
2165   * @return the class object.
2166   * @throws ClassNotFoundException if the class is not found.
2167   */
2168  public Class<?> getClassByName(String name) throws ClassNotFoundException {
2169    Class<?> ret = getClassByNameOrNull(name);
2170    if (ret == null) {
2171      throw new ClassNotFoundException("Class " + name + " not found");
2172    }
2173    return ret;
2174  }
2175  
2176  /**
2177   * Load a class by name, returning null rather than throwing an exception
2178   * if it couldn't be loaded. This is to avoid the overhead of creating
2179   * an exception.
2180   * 
2181   * @param name the class name
2182   * @return the class object, or null if it could not be found.
2183   */
2184  public Class<?> getClassByNameOrNull(String name) {
2185    Map<String, WeakReference<Class<?>>> map;
2186    
2187    synchronized (CACHE_CLASSES) {
2188      map = CACHE_CLASSES.get(classLoader);
2189      if (map == null) {
2190        map = Collections.synchronizedMap(
2191          new WeakHashMap<String, WeakReference<Class<?>>>());
2192        CACHE_CLASSES.put(classLoader, map);
2193      }
2194    }
2195
2196    Class<?> clazz = null;
2197    WeakReference<Class<?>> ref = map.get(name); 
2198    if (ref != null) {
2199       clazz = ref.get();
2200    }
2201     
2202    if (clazz == null) {
2203      try {
2204        clazz = Class.forName(name, true, classLoader);
2205      } catch (ClassNotFoundException e) {
2206        // Leave a marker that the class isn't found
2207        map.put(name, new WeakReference<Class<?>>(NEGATIVE_CACHE_SENTINEL));
2208        return null;
2209      }
2210      // two putters can race here, but they'll put the same class
2211      map.put(name, new WeakReference<Class<?>>(clazz));
2212      return clazz;
2213    } else if (clazz == NEGATIVE_CACHE_SENTINEL) {
2214      return null; // not found
2215    } else {
2216      // cache hit
2217      return clazz;
2218    }
2219  }
2220
2221  /** 
2222   * Get the value of the <code>name</code> property
2223   * as an array of <code>Class</code>.
2224   * The value of the property specifies a list of comma separated class names.  
2225   * If no such property is specified, then <code>defaultValue</code> is 
2226   * returned.
2227   * 
2228   * @param name the property name.
2229   * @param defaultValue default value.
2230   * @return property value as a <code>Class[]</code>, 
2231   *         or <code>defaultValue</code>. 
2232   */
2233  public Class<?>[] getClasses(String name, Class<?> ... defaultValue) {
2234    String valueString = getRaw(name);
2235    if (null == valueString) {
2236      return defaultValue;
2237    }
2238    String[] classnames = getTrimmedStrings(name);
2239    try {
2240      Class<?>[] classes = new Class<?>[classnames.length];
2241      for(int i = 0; i < classnames.length; i++) {
2242        classes[i] = getClassByName(classnames[i]);
2243      }
2244      return classes;
2245    } catch (ClassNotFoundException e) {
2246      throw new RuntimeException(e);
2247    }
2248  }
2249
2250  /** 
2251   * Get the value of the <code>name</code> property as a <code>Class</code>.  
2252   * If no such property is specified, then <code>defaultValue</code> is 
2253   * returned.
2254   * 
2255   * @param name the class name.
2256   * @param defaultValue default value.
2257   * @return property value as a <code>Class</code>, 
2258   *         or <code>defaultValue</code>. 
2259   */
2260  public Class<?> getClass(String name, Class<?> defaultValue) {
2261    String valueString = getTrimmed(name);
2262    if (valueString == null)
2263      return defaultValue;
2264    try {
2265      return getClassByName(valueString);
2266    } catch (ClassNotFoundException e) {
2267      throw new RuntimeException(e);
2268    }
2269  }
2270
2271  /** 
2272   * Get the value of the <code>name</code> property as a <code>Class</code>
2273   * implementing the interface specified by <code>xface</code>.
2274   *   
2275   * If no such property is specified, then <code>defaultValue</code> is 
2276   * returned.
2277   * 
2278   * An exception is thrown if the returned class does not implement the named
2279   * interface. 
2280   * 
2281   * @param name the class name.
2282   * @param defaultValue default value.
2283   * @param xface the interface implemented by the named class.
2284   * @return property value as a <code>Class</code>, 
2285   *         or <code>defaultValue</code>.
2286   */
2287  public <U> Class<? extends U> getClass(String name, 
2288                                         Class<? extends U> defaultValue, 
2289                                         Class<U> xface) {
2290    try {
2291      Class<?> theClass = getClass(name, defaultValue);
2292      if (theClass != null && !xface.isAssignableFrom(theClass))
2293        throw new RuntimeException(theClass+" not "+xface.getName());
2294      else if (theClass != null)
2295        return theClass.asSubclass(xface);
2296      else
2297        return null;
2298    } catch (Exception e) {
2299      throw new RuntimeException(e);
2300    }
2301  }
2302
2303  /**
2304   * Get the value of the <code>name</code> property as a <code>List</code>
2305   * of objects implementing the interface specified by <code>xface</code>.
2306   * 
2307   * An exception is thrown if any of the classes does not exist, or if it does
2308   * not implement the named interface.
2309   * 
2310   * @param name the property name.
2311   * @param xface the interface implemented by the classes named by
2312   *        <code>name</code>.
2313   * @return a <code>List</code> of objects implementing <code>xface</code>.
2314   */
2315  @SuppressWarnings("unchecked")
2316  public <U> List<U> getInstances(String name, Class<U> xface) {
2317    List<U> ret = new ArrayList<U>();
2318    Class<?>[] classes = getClasses(name);
2319    for (Class<?> cl: classes) {
2320      if (!xface.isAssignableFrom(cl)) {
2321        throw new RuntimeException(cl + " does not implement " + xface);
2322      }
2323      ret.add((U)ReflectionUtils.newInstance(cl, this));
2324    }
2325    return ret;
2326  }
2327
2328  /** 
2329   * Set the value of the <code>name</code> property to the name of a 
2330   * <code>theClass</code> implementing the given interface <code>xface</code>.
2331   * 
2332   * An exception is thrown if <code>theClass</code> does not implement the 
2333   * interface <code>xface</code>. 
2334   * 
2335   * @param name property name.
2336   * @param theClass property value.
2337   * @param xface the interface implemented by the named class.
2338   */
2339  public void setClass(String name, Class<?> theClass, Class<?> xface) {
2340    if (!xface.isAssignableFrom(theClass))
2341      throw new RuntimeException(theClass+" not "+xface.getName());
2342    set(name, theClass.getName());
2343  }
2344
2345  /** 
2346   * Get a local file under a directory named by <i>dirsProp</i> with
2347   * the given <i>path</i>.  If <i>dirsProp</i> contains multiple directories,
2348   * then one is chosen based on <i>path</i>'s hash code.  If the selected
2349   * directory does not exist, an attempt is made to create it.
2350   * 
2351   * @param dirsProp directory in which to locate the file.
2352   * @param path file-path.
2353   * @return local file under the directory with the given path.
2354   */
2355  public Path getLocalPath(String dirsProp, String path)
2356    throws IOException {
2357    String[] dirs = getTrimmedStrings(dirsProp);
2358    int hashCode = path.hashCode();
2359    FileSystem fs = FileSystem.getLocal(this);
2360    for (int i = 0; i < dirs.length; i++) {  // try each local dir
2361      int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length;
2362      Path file = new Path(dirs[index], path);
2363      Path dir = file.getParent();
2364      if (fs.mkdirs(dir) || fs.exists(dir)) {
2365        return file;
2366      }
2367    }
2368    LOG.warn("Could not make " + path + 
2369             " in local directories from " + dirsProp);
2370    for(int i=0; i < dirs.length; i++) {
2371      int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length;
2372      LOG.warn(dirsProp + "[" + index + "]=" + dirs[index]);
2373    }
2374    throw new IOException("No valid local directories in property: "+dirsProp);
2375  }
2376
2377  /** 
2378   * Get a local file name under a directory named in <i>dirsProp</i> with
2379   * the given <i>path</i>.  If <i>dirsProp</i> contains multiple directories,
2380   * then one is chosen based on <i>path</i>'s hash code.  If the selected
2381   * directory does not exist, an attempt is made to create it.
2382   * 
2383   * @param dirsProp directory in which to locate the file.
2384   * @param path file-path.
2385   * @return local file under the directory with the given path.
2386   */
2387  public File getFile(String dirsProp, String path)
2388    throws IOException {
2389    String[] dirs = getTrimmedStrings(dirsProp);
2390    int hashCode = path.hashCode();
2391    for (int i = 0; i < dirs.length; i++) {  // try each local dir
2392      int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length;
2393      File file = new File(dirs[index], path);
2394      File dir = file.getParentFile();
2395      if (dir.exists() || dir.mkdirs()) {
2396        return file;
2397      }
2398    }
2399    throw new IOException("No valid local directories in property: "+dirsProp);
2400  }
2401
2402  /** 
2403   * Get the {@link URL} for the named resource.
2404   * 
2405   * @param name resource name.
2406   * @return the url for the named resource.
2407   */
2408  public URL getResource(String name) {
2409    return classLoader.getResource(name);
2410  }
2411  
2412  /** 
2413   * Get an input stream attached to the configuration resource with the
2414   * given <code>name</code>.
2415   * 
2416   * @param name configuration resource name.
2417   * @return an input stream attached to the resource.
2418   */
2419  public InputStream getConfResourceAsInputStream(String name) {
2420    try {
2421      URL url= getResource(name);
2422
2423      if (url == null) {
2424        LOG.info(name + " not found");
2425        return null;
2426      } else {
2427        LOG.info("found resource " + name + " at " + url);
2428      }
2429
2430      return url.openStream();
2431    } catch (Exception e) {
2432      return null;
2433    }
2434  }
2435
2436  /** 
2437   * Get a {@link Reader} attached to the configuration resource with the
2438   * given <code>name</code>.
2439   * 
2440   * @param name configuration resource name.
2441   * @return a reader attached to the resource.
2442   */
2443  public Reader getConfResourceAsReader(String name) {
2444    try {
2445      URL url= getResource(name);
2446
2447      if (url == null) {
2448        LOG.info(name + " not found");
2449        return null;
2450      } else {
2451        LOG.info("found resource " + name + " at " + url);
2452      }
2453
2454      return new InputStreamReader(url.openStream(), Charsets.UTF_8);
2455    } catch (Exception e) {
2456      return null;
2457    }
2458  }
2459
2460  /**
2461   * Get the set of parameters marked final.
2462   *
2463   * @return final parameter set.
2464   */
2465  public Set<String> getFinalParameters() {
2466    Set<String> setFinalParams = Collections.newSetFromMap(
2467        new ConcurrentHashMap<String, Boolean>());
2468    setFinalParams.addAll(finalParameters);
2469    return setFinalParams;
2470  }
2471
2472  protected synchronized Properties getProps() {
2473    if (properties == null) {
2474      properties = new Properties();
2475      Map<String, String[]> backup =
2476          new ConcurrentHashMap<String, String[]>(updatingResource);
2477      loadResources(properties, resources, quietmode);
2478
2479      if (overlay != null) {
2480        properties.putAll(overlay);
2481        for (Map.Entry<Object,Object> item: overlay.entrySet()) {
2482          String key = (String)item.getKey();
2483          String[] source = backup.get(key);
2484          if(source != null) {
2485            updatingResource.put(key, source);
2486          }
2487        }
2488      }
2489    }
2490    return properties;
2491  }
2492
2493  /**
2494   * Return the number of keys in the configuration.
2495   *
2496   * @return number of keys in the configuration.
2497   */
2498  public int size() {
2499    return getProps().size();
2500  }
2501
2502  /**
2503   * Clears all keys from the configuration.
2504   */
2505  public void clear() {
2506    getProps().clear();
2507    getOverlay().clear();
2508  }
2509
2510  /**
2511   * Get an {@link Iterator} to go through the list of <code>String</code> 
2512   * key-value pairs in the configuration.
2513   * 
2514   * @return an iterator over the entries.
2515   */
2516  @Override
2517  public Iterator<Map.Entry<String, String>> iterator() {
2518    // Get a copy of just the string to string pairs. After the old object
2519    // methods that allow non-strings to be put into configurations are removed,
2520    // we could replace properties with a Map<String,String> and get rid of this
2521    // code.
2522    Map<String,String> result = new HashMap<String,String>();
2523    for(Map.Entry<Object,Object> item: getProps().entrySet()) {
2524      if (item.getKey() instanceof String &&
2525          item.getValue() instanceof String) {
2526          result.put((String) item.getKey(), (String) item.getValue());
2527      }
2528    }
2529    return result.entrySet().iterator();
2530  }
2531
2532  private Document parse(DocumentBuilder builder, URL url)
2533      throws IOException, SAXException {
2534    if (!quietmode) {
2535      LOG.debug("parsing URL " + url);
2536    }
2537    if (url == null) {
2538      return null;
2539    }
2540    return parse(builder, url.openStream(), url.toString());
2541  }
2542
2543  private Document parse(DocumentBuilder builder, InputStream is,
2544      String systemId) throws IOException, SAXException {
2545    if (!quietmode) {
2546      LOG.debug("parsing input stream " + is);
2547    }
2548    if (is == null) {
2549      return null;
2550    }
2551    try {
2552      return (systemId == null) ? builder.parse(is) : builder.parse(is,
2553          systemId);
2554    } finally {
2555      is.close();
2556    }
2557  }
2558
2559  private void loadResources(Properties properties,
2560                             ArrayList<Resource> resources,
2561                             boolean quiet) {
2562    if(loadDefaults) {
2563      for (String resource : defaultResources) {
2564        loadResource(properties, new Resource(resource, false), quiet);
2565      }
2566    
2567      //support the hadoop-site.xml as a deprecated case
2568      if(getResource("hadoop-site.xml")!=null) {
2569        loadResource(properties, new Resource("hadoop-site.xml", false), quiet);
2570      }
2571    }
2572    
2573    for (int i = 0; i < resources.size(); i++) {
2574      Resource ret = loadResource(properties, resources.get(i), quiet);
2575      if (ret != null) {
2576        resources.set(i, ret);
2577      }
2578    }
2579  }
2580  
2581  private Resource loadResource(Properties properties, Resource wrapper, boolean quiet) {
2582    String name = UNKNOWN_RESOURCE;
2583    try {
2584      Object resource = wrapper.getResource();
2585      name = wrapper.getName();
2586      
2587      DocumentBuilderFactory docBuilderFactory 
2588        = DocumentBuilderFactory.newInstance();
2589      //ignore all comments inside the xml file
2590      docBuilderFactory.setIgnoringComments(true);
2591
2592      //allow includes in the xml file
2593      docBuilderFactory.setNamespaceAware(true);
2594      boolean useXInclude = !wrapper.isParserRestricted();
2595      try {
2596        docBuilderFactory.setXIncludeAware(useXInclude);
2597      } catch (UnsupportedOperationException e) {
2598        LOG.error("Failed to set setXIncludeAware(" + useXInclude
2599                + ") for parser " + docBuilderFactory, e);
2600      }
2601      if (wrapper.isParserRestricted()) {
2602        docBuilderFactory.setFeature(
2603            "http://apache.org/xml/features/disallow-doctype-decl", true);
2604      }
2605      DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
2606      Document doc = null;
2607      Element root = null;
2608      boolean returnCachedProperties = false;
2609      
2610      if (resource instanceof URL) {                  // an URL resource
2611        doc = parse(builder, (URL)resource);
2612      } else if (resource instanceof String) {        // a CLASSPATH resource
2613        URL url = getResource((String)resource);
2614        doc = parse(builder, url);
2615      } else if (resource instanceof Path) {          // a file resource
2616        // Can't use FileSystem API or we get an infinite loop
2617        // since FileSystem uses Configuration API.  Use java.io.File instead.
2618        File file = new File(((Path)resource).toUri().getPath())
2619          .getAbsoluteFile();
2620        if (file.exists()) {
2621          if (!quiet) {
2622            LOG.debug("parsing File " + file);
2623          }
2624          doc = parse(builder, new BufferedInputStream(
2625              new FileInputStream(file)), ((Path)resource).toString());
2626        }
2627      } else if (resource instanceof InputStream) {
2628        doc = parse(builder, (InputStream) resource, null);
2629        returnCachedProperties = true;
2630      } else if (resource instanceof Properties) {
2631        overlay(properties, (Properties)resource);
2632      } else if (resource instanceof Element) {
2633        root = (Element)resource;
2634      }
2635
2636      if (root == null) {
2637        if (doc == null) {
2638          if (quiet) {
2639            return null;
2640          }
2641          throw new RuntimeException(resource + " not found");
2642        }
2643        root = doc.getDocumentElement();
2644      }
2645      Properties toAddTo = properties;
2646      if(returnCachedProperties) {
2647        toAddTo = new Properties();
2648      }
2649      if (!"configuration".equals(root.getTagName()))
2650        LOG.fatal("bad conf file: top-level element not <configuration>");
2651      NodeList props = root.getChildNodes();
2652      DeprecationContext deprecations = deprecationContext.get();
2653      for (int i = 0; i < props.getLength(); i++) {
2654        Node propNode = props.item(i);
2655        if (!(propNode instanceof Element))
2656          continue;
2657        Element prop = (Element)propNode;
2658        if ("configuration".equals(prop.getTagName())) {
2659          loadResource(toAddTo,
2660              new Resource(prop, name, wrapper.isParserRestricted()), quiet);
2661          continue;
2662        }
2663        if (!"property".equals(prop.getTagName())) {
2664          if (wrapper.isParserRestricted()
2665              && XINCLUDE_NS_URI.equals(prop.getNamespaceURI())) {
2666            throw new RuntimeException("Error parsing resource " + wrapper
2667                + ": XInclude is not supported for restricted resources");
2668          }
2669          LOG.warn("Unexpected tag in conf file " + wrapper
2670              + ": expected <property> but found <" + prop.getTagName() + ">");
2671        }
2672        NodeList fields = prop.getChildNodes();
2673        String attr = null;
2674        String value = null;
2675        boolean finalParameter = false;
2676        LinkedList<String> source = new LinkedList<String>();
2677        for (int j = 0; j < fields.getLength(); j++) {
2678          Node fieldNode = fields.item(j);
2679          if (!(fieldNode instanceof Element))
2680            continue;
2681          Element field = (Element)fieldNode;
2682          if ("name".equals(field.getTagName()) && field.hasChildNodes())
2683            attr = StringInterner.weakIntern(
2684                ((Text)field.getFirstChild()).getData().trim());
2685          if ("value".equals(field.getTagName()) && field.hasChildNodes())
2686            value = StringInterner.weakIntern(
2687                ((Text)field.getFirstChild()).getData());
2688          if ("final".equals(field.getTagName()) && field.hasChildNodes())
2689            finalParameter = "true".equals(((Text)field.getFirstChild()).getData());
2690          if ("source".equals(field.getTagName()) && field.hasChildNodes())
2691            source.add(StringInterner.weakIntern(
2692                ((Text)field.getFirstChild()).getData()));
2693        }
2694        source.add(name);
2695        
2696        // Ignore this parameter if it has already been marked as 'final'
2697        if (attr != null) {
2698          if (deprecations.getDeprecatedKeyMap().containsKey(attr)) {
2699            DeprecatedKeyInfo keyInfo =
2700                deprecations.getDeprecatedKeyMap().get(attr);
2701            keyInfo.clearAccessed();
2702            for (String key:keyInfo.newKeys) {
2703              // update new keys with deprecated key's value 
2704              loadProperty(toAddTo, name, key, value, finalParameter, 
2705                  source.toArray(new String[source.size()]));
2706            }
2707          }
2708          else {
2709            loadProperty(toAddTo, name, attr, value, finalParameter, 
2710                source.toArray(new String[source.size()]));
2711          }
2712        }
2713      }
2714      
2715      if (returnCachedProperties) {
2716        overlay(properties, toAddTo);
2717        return new Resource(toAddTo, name, wrapper.isParserRestricted());
2718      }
2719      return null;
2720    } catch (IOException e) {
2721      LOG.fatal("error parsing conf " + name, e);
2722      throw new RuntimeException(e);
2723    } catch (DOMException e) {
2724      LOG.fatal("error parsing conf " + name, e);
2725      throw new RuntimeException(e);
2726    } catch (SAXException e) {
2727      LOG.fatal("error parsing conf " + name, e);
2728      throw new RuntimeException(e);
2729    } catch (ParserConfigurationException e) {
2730      LOG.fatal("error parsing conf " + name , e);
2731      throw new RuntimeException(e);
2732    }
2733  }
2734
2735  private void overlay(Properties to, Properties from) {
2736    for (Entry<Object, Object> entry: from.entrySet()) {
2737      to.put(entry.getKey(), entry.getValue());
2738    }
2739  }
2740  
2741  private void loadProperty(Properties properties, String name, String attr,
2742      String value, boolean finalParameter, String[] source) {
2743    if (value != null || allowNullValueProperties) {
2744      if (!finalParameters.contains(attr)) {
2745        if (value==null && allowNullValueProperties) {
2746          value = DEFAULT_STRING_CHECK;
2747        }
2748        properties.setProperty(attr, value);
2749        if(source != null) {
2750          updatingResource.put(attr, source);
2751        }
2752      } else if (!value.equals(properties.getProperty(attr))) {
2753        LOG.warn(name+":an attempt to override final parameter: "+attr
2754            +";  Ignoring.");
2755      }
2756    }
2757    if (finalParameter && attr != null) {
2758      finalParameters.add(attr);
2759    }
2760  }
2761
2762  /** 
2763   * Write out the non-default properties in this configuration to the given
2764   * {@link OutputStream} using UTF-8 encoding.
2765   * 
2766   * @param out the output stream to write to.
2767   */
2768  public void writeXml(OutputStream out) throws IOException {
2769    writeXml(new OutputStreamWriter(out, "UTF-8"));
2770  }
2771
2772  /** 
2773   * Write out the non-default properties in this configuration to the given
2774   * {@link Writer}.
2775   * 
2776   * @param out the writer to write to.
2777   */
2778  public void writeXml(Writer out) throws IOException {
2779    Document doc = asXmlDocument();
2780
2781    try {
2782      DOMSource source = new DOMSource(doc);
2783      StreamResult result = new StreamResult(out);
2784      TransformerFactory transFactory = TransformerFactory.newInstance();
2785      Transformer transformer = transFactory.newTransformer();
2786
2787      // Important to not hold Configuration log while writing result, since
2788      // 'out' may be an HDFS stream which needs to lock this configuration
2789      // from another thread.
2790      transformer.transform(source, result);
2791    } catch (TransformerException te) {
2792      throw new IOException(te);
2793    }
2794  }
2795
2796  /**
2797   * Return the XML DOM corresponding to this Configuration.
2798   */
2799  private synchronized Document asXmlDocument() throws IOException {
2800    Document doc;
2801    try {
2802      doc =
2803        DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument();
2804    } catch (ParserConfigurationException pe) {
2805      throw new IOException(pe);
2806    }
2807    Element conf = doc.createElement("configuration");
2808    doc.appendChild(conf);
2809    conf.appendChild(doc.createTextNode("\n"));
2810    handleDeprecation(); //ensure properties is set and deprecation is handled
2811    for (Enumeration<Object> e = properties.keys(); e.hasMoreElements();) {
2812      String name = (String)e.nextElement();
2813      Object object = properties.get(name);
2814      String value = null;
2815      if (object instanceof String) {
2816        value = (String) object;
2817      }else {
2818        continue;
2819      }
2820      Element propNode = doc.createElement("property");
2821      conf.appendChild(propNode);
2822
2823      Element nameNode = doc.createElement("name");
2824      nameNode.appendChild(doc.createTextNode(name));
2825      propNode.appendChild(nameNode);
2826
2827      Element valueNode = doc.createElement("value");
2828      valueNode.appendChild(doc.createTextNode(value));
2829      propNode.appendChild(valueNode);
2830
2831      if (updatingResource != null) {
2832        String[] sources = updatingResource.get(name);
2833        if(sources != null) {
2834          for(String s : sources) {
2835            Element sourceNode = doc.createElement("source");
2836            sourceNode.appendChild(doc.createTextNode(s));
2837            propNode.appendChild(sourceNode);
2838          }
2839        }
2840      }
2841      
2842      conf.appendChild(doc.createTextNode("\n"));
2843    }
2844    return doc;
2845  }
2846
2847  /**
2848   *  Writes out all the parameters and their properties (final and resource) to
2849   *  the given {@link Writer}
2850   *  The format of the output would be 
2851   *  { "properties" : [ {key1,value1,key1.isFinal,key1.resource}, {key2,value2,
2852   *  key2.isFinal,key2.resource}... ] } 
2853   *  It does not output the parameters of the configuration object which is 
2854   *  loaded from an input stream.
2855   * @param out the Writer to write to
2856   * @throws IOException
2857   */
2858  public static void dumpConfiguration(Configuration config,
2859      Writer out) throws IOException {
2860    JsonFactory dumpFactory = new JsonFactory();
2861    JsonGenerator dumpGenerator = dumpFactory.createJsonGenerator(out);
2862    dumpGenerator.writeStartObject();
2863    dumpGenerator.writeFieldName("properties");
2864    dumpGenerator.writeStartArray();
2865    dumpGenerator.flush();
2866    synchronized (config) {
2867      for (Map.Entry<Object,Object> item: config.getProps().entrySet()) {
2868        dumpGenerator.writeStartObject();
2869        dumpGenerator.writeStringField("key", (String) item.getKey());
2870        dumpGenerator.writeStringField("value", 
2871                                       config.get((String) item.getKey()));
2872        dumpGenerator.writeBooleanField("isFinal",
2873                                        config.finalParameters.contains(item.getKey()));
2874        String[] resources = config.updatingResource.get(item.getKey());
2875        String resource = UNKNOWN_RESOURCE;
2876        if(resources != null && resources.length > 0) {
2877          resource = resources[0];
2878        }
2879        dumpGenerator.writeStringField("resource", resource);
2880        dumpGenerator.writeEndObject();
2881      }
2882    }
2883    dumpGenerator.writeEndArray();
2884    dumpGenerator.writeEndObject();
2885    dumpGenerator.flush();
2886  }
2887  
2888  /**
2889   * Get the {@link ClassLoader} for this job.
2890   * 
2891   * @return the correct class loader.
2892   */
2893  public ClassLoader getClassLoader() {
2894    return classLoader;
2895  }
2896  
2897  /**
2898   * Set the class loader that will be used to load the various objects.
2899   * 
2900   * @param classLoader the new class loader.
2901   */
2902  public void setClassLoader(ClassLoader classLoader) {
2903    this.classLoader = classLoader;
2904  }
2905  
2906  @Override
2907  public String toString() {
2908    StringBuilder sb = new StringBuilder();
2909    sb.append("Configuration: ");
2910    if(loadDefaults) {
2911      toString(defaultResources, sb);
2912      if(resources.size()>0) {
2913        sb.append(", ");
2914      }
2915    }
2916    toString(resources, sb);
2917    return sb.toString();
2918  }
2919  
2920  private <T> void toString(List<T> resources, StringBuilder sb) {
2921    ListIterator<T> i = resources.listIterator();
2922    while (i.hasNext()) {
2923      if (i.nextIndex() != 0) {
2924        sb.append(", ");
2925      }
2926      sb.append(i.next());
2927    }
2928  }
2929
2930  /** 
2931   * Set the quietness-mode. 
2932   * 
2933   * In the quiet-mode, error and informational messages might not be logged.
2934   * 
2935   * @param quietmode <code>true</code> to set quiet-mode on, <code>false</code>
2936   *              to turn it off.
2937   */
2938  public synchronized void setQuietMode(boolean quietmode) {
2939    this.quietmode = quietmode;
2940  }
2941
2942  synchronized boolean getQuietMode() {
2943    return this.quietmode;
2944  }
2945  
2946  /** For debugging.  List non-default properties to the terminal and exit. */
2947  public static void main(String[] args) throws Exception {
2948    new Configuration().writeXml(System.out);
2949  }
2950
2951  @Override
2952  public void readFields(DataInput in) throws IOException {
2953    clear();
2954    int size = WritableUtils.readVInt(in);
2955    for(int i=0; i < size; ++i) {
2956      String key = org.apache.hadoop.io.Text.readString(in);
2957      String value = org.apache.hadoop.io.Text.readString(in);
2958      set(key, value); 
2959      String sources[] = WritableUtils.readCompressedStringArray(in);
2960      if(sources != null) {
2961        updatingResource.put(key, sources);
2962      }
2963    }
2964  }
2965
2966  //@Override
2967  @Override
2968  public void write(DataOutput out) throws IOException {
2969    Properties props = getProps();
2970    WritableUtils.writeVInt(out, props.size());
2971    for(Map.Entry<Object, Object> item: props.entrySet()) {
2972      org.apache.hadoop.io.Text.writeString(out, (String) item.getKey());
2973      org.apache.hadoop.io.Text.writeString(out, (String) item.getValue());
2974      WritableUtils.writeCompressedStringArray(out, 
2975          updatingResource.get(item.getKey()));
2976    }
2977  }
2978  
2979  /**
2980   * get keys matching the the regex 
2981   * @param regex
2982   * @return Map<String,String> with matching keys
2983   */
2984  public Map<String,String> getValByRegex(String regex) {
2985    Pattern p = Pattern.compile(regex);
2986
2987    Map<String,String> result = new HashMap<String,String>();
2988    Matcher m;
2989
2990    for(Map.Entry<Object,Object> item: getProps().entrySet()) {
2991      if (item.getKey() instanceof String && 
2992          item.getValue() instanceof String) {
2993        m = p.matcher((String)item.getKey());
2994        if(m.find()) { // match
2995          result.put((String) item.getKey(),
2996              substituteVars(getProps().getProperty((String) item.getKey())));
2997        }
2998      }
2999    }
3000    return result;
3001  }
3002
3003  /**
3004   * A unique class which is used as a sentinel value in the caching
3005   * for getClassByName. {@see Configuration#getClassByNameOrNull(String)}
3006   */
3007  private static abstract class NegativeCacheSentinel {}
3008
3009  public static void dumpDeprecatedKeys() {
3010    DeprecationContext deprecations = deprecationContext.get();
3011    for (Map.Entry<String, DeprecatedKeyInfo> entry :
3012        deprecations.getDeprecatedKeyMap().entrySet()) {
3013      StringBuilder newKeys = new StringBuilder();
3014      for (String newKey : entry.getValue().newKeys) {
3015        newKeys.append(newKey).append("\t");
3016      }
3017      System.out.println(entry.getKey() + "\t" + newKeys.toString());
3018    }
3019  }
3020
3021  /**
3022   * Returns whether or not a deprecated name has been warned. If the name is not
3023   * deprecated then always return false
3024   */
3025  public static boolean hasWarnedDeprecation(String name) {
3026    DeprecationContext deprecations = deprecationContext.get();
3027    if(deprecations.getDeprecatedKeyMap().containsKey(name)) {
3028      if(deprecations.getDeprecatedKeyMap().get(name).accessed.get()) {
3029        return true;
3030      }
3031    }
3032    return false;
3033  }
3034}