001    /**
002    * Licensed to the Apache Software Foundation (ASF) under one
003    * or more contributor license agreements.  See the NOTICE file
004    * distributed with this work for additional information
005    * regarding copyright ownership.  The ASF licenses this file
006    * to you under the Apache License, Version 2.0 (the
007    * "License"); you may not use this file except in compliance
008    * with the License.  You may obtain a copy of the License at
009    *
010    *     http://www.apache.org/licenses/LICENSE-2.0
011    *
012    * Unless required by applicable law or agreed to in writing, software
013    * distributed under the License is distributed on an "AS IS" BASIS,
014    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015    * See the License for the specific language governing permissions and
016    * limitations under the License.
017    */
018    
019    package org.apache.hadoop.yarn.conf;
020    
021    import java.net.InetSocketAddress;
022    import java.util.Arrays;
023    import java.util.Collections;
024    import java.util.List;
025    
026    import org.apache.hadoop.HadoopIllegalArgumentException;
027    import org.apache.hadoop.classification.InterfaceAudience.Private;
028    import org.apache.hadoop.classification.InterfaceAudience.Public;
029    import org.apache.hadoop.classification.InterfaceStability.Evolving;
030    import org.apache.hadoop.classification.InterfaceStability.Unstable;
031    import org.apache.hadoop.conf.Configuration;
032    import org.apache.hadoop.http.HttpConfig;
033    import org.apache.hadoop.net.NetUtils;
034    import org.apache.hadoop.util.StringUtils;
035    import org.apache.hadoop.yarn.api.ApplicationConstants;
036    
037    @Public
038    @Evolving
039    public class YarnConfiguration extends Configuration {
040    
041      @Private
042      public static final String CS_CONFIGURATION_FILE= "capacity-scheduler.xml";
043    
044      @Private
045      public static final String HADOOP_POLICY_CONFIGURATION_FILE =
046          "hadoop-policy.xml";
047    
048      @Private
049      public static final String YARN_SITE_CONFIGURATION_FILE = "yarn-site.xml";
050    
051      private static final String YARN_DEFAULT_CONFIGURATION_FILE =
052          "yarn-default.xml";
053    
054      @Private
055      public static final String CORE_SITE_CONFIGURATION_FILE = "core-site.xml";
056    
057      @Private
058      public static final List<String> RM_CONFIGURATION_FILES =
059          Collections.unmodifiableList(Arrays.asList(
060              CS_CONFIGURATION_FILE,
061              HADOOP_POLICY_CONFIGURATION_FILE,
062              YARN_SITE_CONFIGURATION_FILE,
063              CORE_SITE_CONFIGURATION_FILE));
064    
065      @Evolving
066      public static final int APPLICATION_MAX_TAGS = 10;
067    
068      @Evolving
069      public static final int APPLICATION_MAX_TAG_LENGTH = 100;
070    
071      static {
072        Configuration.addDefaultResource(YARN_DEFAULT_CONFIGURATION_FILE);
073        Configuration.addDefaultResource(YARN_SITE_CONFIGURATION_FILE);
074      }
075    
076      //Configurations
077    
078      public static final String YARN_PREFIX = "yarn.";
079    
080      /** Delay before deleting resource to ease debugging of NM issues */
081      public static final String DEBUG_NM_DELETE_DELAY_SEC =
082        YarnConfiguration.NM_PREFIX + "delete.debug-delay-sec";
083      
084      ////////////////////////////////
085      // IPC Configs
086      ////////////////////////////////
087      public static final String IPC_PREFIX = YARN_PREFIX + "ipc.";
088    
089      /** Factory to create client IPC classes.*/
090      public static final String IPC_CLIENT_FACTORY_CLASS =
091        IPC_PREFIX + "client.factory.class";
092      public static final String DEFAULT_IPC_CLIENT_FACTORY_CLASS = 
093          "org.apache.hadoop.yarn.factories.impl.pb.RpcClientFactoryPBImpl";
094    
095      /** Factory to create server IPC classes.*/
096      public static final String IPC_SERVER_FACTORY_CLASS = 
097        IPC_PREFIX + "server.factory.class";
098      public static final String DEFAULT_IPC_SERVER_FACTORY_CLASS = 
099          "org.apache.hadoop.yarn.factories.impl.pb.RpcServerFactoryPBImpl";
100    
101      /** Factory to create serializeable records.*/
102      public static final String IPC_RECORD_FACTORY_CLASS = 
103        IPC_PREFIX + "record.factory.class";
104      public static final String DEFAULT_IPC_RECORD_FACTORY_CLASS = 
105          "org.apache.hadoop.yarn.factories.impl.pb.RecordFactoryPBImpl";
106    
107      /** RPC class implementation*/
108      public static final String IPC_RPC_IMPL =
109        IPC_PREFIX + "rpc.class";
110      public static final String DEFAULT_IPC_RPC_IMPL = 
111        "org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC";
112      
113      ////////////////////////////////
114      // Resource Manager Configs
115      ////////////////////////////////
116      public static final String RM_PREFIX = "yarn.resourcemanager.";
117    
118      public static final String RM_CLUSTER_ID = RM_PREFIX + "cluster-id";
119    
120      public static final String RM_HOSTNAME = RM_PREFIX + "hostname";
121    
122      /** The address of the applications manager interface in the RM.*/
123      public static final String RM_ADDRESS = 
124        RM_PREFIX + "address";
125      public static final int DEFAULT_RM_PORT = 8032;
126      public static final String DEFAULT_RM_ADDRESS =
127        "0.0.0.0:" + DEFAULT_RM_PORT;
128    
129      /** The number of threads used to handle applications manager requests.*/
130      public static final String RM_CLIENT_THREAD_COUNT =
131        RM_PREFIX + "client.thread-count";
132      public static final int DEFAULT_RM_CLIENT_THREAD_COUNT = 50;
133    
134      /** The Kerberos principal for the resource manager.*/
135      public static final String RM_PRINCIPAL =
136        RM_PREFIX + "principal";
137      
138      /** The address of the scheduler interface.*/
139      public static final String RM_SCHEDULER_ADDRESS = 
140        RM_PREFIX + "scheduler.address";
141      public static final int DEFAULT_RM_SCHEDULER_PORT = 8030;
142      public static final String DEFAULT_RM_SCHEDULER_ADDRESS = "0.0.0.0:" +
143        DEFAULT_RM_SCHEDULER_PORT;
144    
145      /** Miniumum request grant-able by the RM scheduler. */
146      public static final String RM_SCHEDULER_MINIMUM_ALLOCATION_MB =
147        YARN_PREFIX + "scheduler.minimum-allocation-mb";
148      public static final int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB = 1024;
149      public static final String RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES =
150          YARN_PREFIX + "scheduler.minimum-allocation-vcores";
151        public static final int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES = 1;
152    
153      /** Maximum request grant-able by the RM scheduler. */
154      public static final String RM_SCHEDULER_MAXIMUM_ALLOCATION_MB =
155        YARN_PREFIX + "scheduler.maximum-allocation-mb";
156      public static final int DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB = 8192;
157      public static final String RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES =
158          YARN_PREFIX + "scheduler.maximum-allocation-vcores";
159      public static final int DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES = 4;
160    
161      /** Number of threads to handle scheduler interface.*/
162      public static final String RM_SCHEDULER_CLIENT_THREAD_COUNT =
163        RM_PREFIX + "scheduler.client.thread-count";
164      public static final int DEFAULT_RM_SCHEDULER_CLIENT_THREAD_COUNT = 50;
165    
166      /** If the port should be included or not in the node name. The node name
167       * is used by the scheduler for resource requests allocation location 
168       * matching. Typically this is just the hostname, using the port is needed
169       * when using minicluster and specific NM are required.*/
170      public static final String RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME =
171          YARN_PREFIX + "scheduler.include-port-in-node-name";
172      public static final boolean DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME = 
173          false;
174    
175      /**
176       * Enable periodic monitor threads.
177       * @see #RM_SCHEDULER_MONITOR_POLICIES
178       */
179      public static final String RM_SCHEDULER_ENABLE_MONITORS =
180        RM_PREFIX + "scheduler.monitor.enable";
181      public static final boolean DEFAULT_RM_SCHEDULER_ENABLE_MONITORS = false;
182    
183      /** List of SchedulingEditPolicy classes affecting the scheduler. */
184      public static final String RM_SCHEDULER_MONITOR_POLICIES =
185        RM_PREFIX + "scheduler.monitor.policies";
186    
187      /** The address of the RM web application.*/
188      public static final String RM_WEBAPP_ADDRESS = 
189        RM_PREFIX + "webapp.address";
190    
191      public static final int DEFAULT_RM_WEBAPP_PORT = 8088;
192      public static final String DEFAULT_RM_WEBAPP_ADDRESS = "0.0.0.0:" +
193        DEFAULT_RM_WEBAPP_PORT;
194      
195      /** The https address of the RM web application.*/
196      public static final String RM_WEBAPP_HTTPS_ADDRESS =
197          RM_PREFIX + "webapp.https.address";
198      public static final boolean YARN_SSL_CLIENT_HTTPS_NEED_AUTH_DEFAULT = false;
199      public static final String YARN_SSL_SERVER_RESOURCE_DEFAULT = "ssl-server.xml";
200      
201      public static final int DEFAULT_RM_WEBAPP_HTTPS_PORT = 8090;
202      public static final String DEFAULT_RM_WEBAPP_HTTPS_ADDRESS = "0.0.0.0:"
203          + DEFAULT_RM_WEBAPP_HTTPS_PORT;
204      
205      public static final String RM_RESOURCE_TRACKER_ADDRESS =
206        RM_PREFIX + "resource-tracker.address";
207      public static final int DEFAULT_RM_RESOURCE_TRACKER_PORT = 8031;
208      public static final String DEFAULT_RM_RESOURCE_TRACKER_ADDRESS =
209        "0.0.0.0:" + DEFAULT_RM_RESOURCE_TRACKER_PORT;
210    
211      /** The expiry interval for application master reporting.*/
212      public static final String RM_AM_EXPIRY_INTERVAL_MS = 
213        YARN_PREFIX  + "am.liveness-monitor.expiry-interval-ms";
214      public static final int DEFAULT_RM_AM_EXPIRY_INTERVAL_MS = 600000;
215    
216      /** How long to wait until a node manager is considered dead.*/
217      public static final String RM_NM_EXPIRY_INTERVAL_MS = 
218        YARN_PREFIX + "nm.liveness-monitor.expiry-interval-ms";
219      public static final int DEFAULT_RM_NM_EXPIRY_INTERVAL_MS = 600000;
220    
221      /** Are acls enabled.*/
222      public static final String YARN_ACL_ENABLE = 
223        YARN_PREFIX + "acl.enable";
224      public static final boolean DEFAULT_YARN_ACL_ENABLE = false;
225      
226      /** ACL of who can be admin of YARN cluster.*/
227      public static final String YARN_ADMIN_ACL = 
228        YARN_PREFIX + "admin.acl";
229      public static final String DEFAULT_YARN_ADMIN_ACL = "*";
230      
231      /** ACL used in case none is found. Allows nothing. */
232      public static final String DEFAULT_YARN_APP_ACL = " ";
233    
234      /** The address of the RM admin interface.*/
235      public static final String RM_ADMIN_ADDRESS = 
236        RM_PREFIX + "admin.address";
237      public static final int DEFAULT_RM_ADMIN_PORT = 8033;
238      public static final String DEFAULT_RM_ADMIN_ADDRESS = "0.0.0.0:" +
239          DEFAULT_RM_ADMIN_PORT;
240      
241      /**Number of threads used to handle RM admin interface.*/
242      public static final String RM_ADMIN_CLIENT_THREAD_COUNT =
243        RM_PREFIX + "admin.client.thread-count";
244      public static final int DEFAULT_RM_ADMIN_CLIENT_THREAD_COUNT = 1;
245      
246      /**
247       * The maximum number of application attempts.
248       * It's a global setting for all application masters.
249       */
250      public static final String RM_AM_MAX_ATTEMPTS =
251        RM_PREFIX + "am.max-attempts";
252      public static final int DEFAULT_RM_AM_MAX_ATTEMPTS = 2;
253      
254      /** The keytab for the resource manager.*/
255      public static final String RM_KEYTAB = 
256        RM_PREFIX + "keytab";
257    
258      /**The kerberos principal to be used for spnego filter for RM.*/
259      public static final String RM_WEBAPP_SPNEGO_USER_NAME_KEY =
260          RM_PREFIX + "webapp.spnego-principal";
261      
262      /**The kerberos keytab to be used for spnego filter for RM.*/
263      public static final String RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY =
264          RM_PREFIX + "webapp.spnego-keytab-file";
265    
266      /** How long to wait until a container is considered dead.*/
267      public static final String RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS = 
268        RM_PREFIX + "rm.container-allocation.expiry-interval-ms";
269      public static final int DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS = 600000;
270      
271      /** Path to file with nodes to include.*/
272      public static final String RM_NODES_INCLUDE_FILE_PATH = 
273        RM_PREFIX + "nodes.include-path";
274      public static final String DEFAULT_RM_NODES_INCLUDE_FILE_PATH = "";
275      
276      /** Path to file with nodes to exclude.*/
277      public static final String RM_NODES_EXCLUDE_FILE_PATH = 
278        RM_PREFIX + "nodes.exclude-path";
279      public static final String DEFAULT_RM_NODES_EXCLUDE_FILE_PATH = "";
280      
281      /** Number of threads to handle resource tracker calls.*/
282      public static final String RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT =
283        RM_PREFIX + "resource-tracker.client.thread-count";
284      public static final int DEFAULT_RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT = 50;
285      
286      /** The class to use as the resource scheduler.*/
287      public static final String RM_SCHEDULER = 
288        RM_PREFIX + "scheduler.class";
289     
290      public static final String DEFAULT_RM_SCHEDULER = 
291          "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler";
292    
293      /** RM set next Heartbeat interval for NM */
294      public static final String RM_NM_HEARTBEAT_INTERVAL_MS =
295          RM_PREFIX + "nodemanagers.heartbeat-interval-ms";
296      public static final long DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS = 1000;
297    
298      /** Number of worker threads that write the history data. */
299      public static final String RM_HISTORY_WRITER_MULTI_THREADED_DISPATCHER_POOL_SIZE =
300          RM_PREFIX + "history-writer.multi-threaded-dispatcher.pool-size";
301      public static final int DEFAULT_RM_HISTORY_WRITER_MULTI_THREADED_DISPATCHER_POOL_SIZE =
302          10;
303    
304      //Delegation token related keys
305      public static final String  DELEGATION_KEY_UPDATE_INTERVAL_KEY = 
306        RM_PREFIX + "delegation.key.update-interval";
307      public static final long    DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT = 
308        24*60*60*1000; // 1 day
309      public static final String  DELEGATION_TOKEN_RENEW_INTERVAL_KEY = 
310        RM_PREFIX + "delegation.token.renew-interval";
311      public static final long    DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT = 
312        24*60*60*1000;  // 1 day
313      public static final String  DELEGATION_TOKEN_MAX_LIFETIME_KEY = 
314         RM_PREFIX + "delegation.token.max-lifetime";
315      public static final long    DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = 
316        7*24*60*60*1000; // 7 days
317      
318      public static final String RECOVERY_ENABLED = RM_PREFIX + "recovery.enabled";
319      public static final boolean DEFAULT_RM_RECOVERY_ENABLED = false;
320    
321      /** Zookeeper interaction configs */
322      public static final String RM_ZK_PREFIX = RM_PREFIX + "zk-";
323    
324      public static final String RM_ZK_ADDRESS = RM_ZK_PREFIX + "address";
325    
326      public static final String RM_ZK_NUM_RETRIES = RM_ZK_PREFIX + "num-retries";
327      public static final int DEFAULT_ZK_RM_NUM_RETRIES = 500;
328    
329      public static final String RM_ZK_RETRY_INTERVAL_MS =
330          RM_ZK_PREFIX + "retry-interval-ms";
331      public static final long DEFAULT_RM_ZK_RETRY_INTERVAL_MS = 2000;
332    
333      public static final String RM_ZK_TIMEOUT_MS = RM_ZK_PREFIX + "timeout-ms";
334      public static final int DEFAULT_RM_ZK_TIMEOUT_MS = 10000;
335    
336      public static final String RM_ZK_ACL = RM_ZK_PREFIX + "acl";
337      public static final String DEFAULT_RM_ZK_ACL = "world:anyone:rwcda";
338    
339      public static final String RM_ZK_AUTH = RM_ZK_PREFIX + "auth";
340    
341      public static final String ZK_STATE_STORE_PREFIX =
342          RM_PREFIX + "zk-state-store.";
343    
344      /** Parent znode path under which ZKRMStateStore will create znodes */
345      public static final String ZK_RM_STATE_STORE_PARENT_PATH =
346          ZK_STATE_STORE_PREFIX + "parent-path";
347      public static final String DEFAULT_ZK_RM_STATE_STORE_PARENT_PATH = "/rmstore";
348    
349      /** Root node ACLs for fencing */
350      public static final String ZK_RM_STATE_STORE_ROOT_NODE_ACL =
351          ZK_STATE_STORE_PREFIX + "root-node.acl";
352    
353      /** HA related configs */
354      public static final String RM_HA_PREFIX = RM_PREFIX + "ha.";
355      public static final String RM_HA_ENABLED = RM_HA_PREFIX + "enabled";
356      public static final boolean DEFAULT_RM_HA_ENABLED = false;
357    
358      public static final String RM_HA_IDS = RM_HA_PREFIX + "rm-ids";
359      public static final String RM_HA_ID = RM_HA_PREFIX + "id";
360    
361      /** Store the related configuration files in File System */
362      public static final String FS_BASED_RM_CONF_STORE = RM_PREFIX
363          + "configuration.file-system-based-store";
364      public static final String DEFAULT_FS_BASED_RM_CONF_STORE = "/yarn/conf";
365    
366      public static final String RM_CONFIGURATION_PROVIDER_CLASS = RM_PREFIX
367          + "configuration.provider-class";
368      public static final String DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS =
369          "org.apache.hadoop.yarn.LocalConfigurationProvider";
370    
371      private static final List<String> RM_SERVICES_ADDRESS_CONF_KEYS_HTTP =
372          Collections.unmodifiableList(Arrays.asList(
373              RM_ADDRESS,
374              RM_SCHEDULER_ADDRESS,
375              RM_ADMIN_ADDRESS,
376              RM_RESOURCE_TRACKER_ADDRESS,
377              RM_WEBAPP_ADDRESS));
378    
379      private static final List<String> RM_SERVICES_ADDRESS_CONF_KEYS_HTTPS =
380          Collections.unmodifiableList(Arrays.asList(
381              RM_ADDRESS,
382              RM_SCHEDULER_ADDRESS,
383              RM_ADMIN_ADDRESS,
384              RM_RESOURCE_TRACKER_ADDRESS,
385              RM_WEBAPP_HTTPS_ADDRESS));
386    
387      public static final String AUTO_FAILOVER_PREFIX =
388          RM_HA_PREFIX + "automatic-failover.";
389    
390      public static final String AUTO_FAILOVER_ENABLED =
391          AUTO_FAILOVER_PREFIX + "enabled";
392      public static final boolean DEFAULT_AUTO_FAILOVER_ENABLED = true;
393    
394      public static final String AUTO_FAILOVER_EMBEDDED =
395          AUTO_FAILOVER_PREFIX + "embedded";
396      public static final boolean DEFAULT_AUTO_FAILOVER_EMBEDDED = true;
397    
398      public static final String AUTO_FAILOVER_ZK_BASE_PATH =
399          AUTO_FAILOVER_PREFIX + "zk-base-path";
400      public static final String DEFAULT_AUTO_FAILOVER_ZK_BASE_PATH =
401          "/yarn-leader-election";
402    
403      public static final String CLIENT_FAILOVER_PREFIX =
404          YARN_PREFIX + "client.failover-";
405      public static final String CLIENT_FAILOVER_PROXY_PROVIDER =
406          CLIENT_FAILOVER_PREFIX + "proxy-provider";
407      public static final String DEFAULT_CLIENT_FAILOVER_PROXY_PROVIDER =
408          "org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider";
409    
410      public static final String CLIENT_FAILOVER_MAX_ATTEMPTS =
411          CLIENT_FAILOVER_PREFIX + "max-attempts";
412    
413      public static final String CLIENT_FAILOVER_SLEEPTIME_BASE_MS =
414          CLIENT_FAILOVER_PREFIX + "sleep-base-ms";
415    
416      public static final String CLIENT_FAILOVER_SLEEPTIME_MAX_MS =
417          CLIENT_FAILOVER_PREFIX + "sleep-max-ms";
418    
419      public static final String CLIENT_FAILOVER_RETRIES =
420          CLIENT_FAILOVER_PREFIX + "retries";
421      public static final int DEFAULT_CLIENT_FAILOVER_RETRIES = 0;
422    
423      public static final String CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS =
424          CLIENT_FAILOVER_PREFIX + "retries-on-socket-timeouts";
425      public static final int
426          DEFAULT_CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS = 0;
427    
428      ////////////////////////////////
429      // RM state store configs
430      ////////////////////////////////
431      /** The class to use as the persistent store.*/
432      public static final String RM_STORE = RM_PREFIX + "store.class";
433      
434      /** URI for FileSystemRMStateStore */
435      public static final String FS_RM_STATE_STORE_URI = RM_PREFIX
436          + "fs.state-store.uri";
437      public static final String FS_RM_STATE_STORE_RETRY_POLICY_SPEC = RM_PREFIX
438          + "fs.state-store.retry-policy-spec";
439      public static final String DEFAULT_FS_RM_STATE_STORE_RETRY_POLICY_SPEC =
440          "2000, 500";
441    
442      /** The maximum number of completed applications RM keeps. */ 
443      public static final String RM_MAX_COMPLETED_APPLICATIONS =
444        RM_PREFIX + "max-completed-applications";
445      public static final int DEFAULT_RM_MAX_COMPLETED_APPLICATIONS = 10000;
446    
447      /**
448       * The maximum number of completed applications RM state store keeps, by
449       * default equals to DEFAULT_RM_MAX_COMPLETED_APPLICATIONS
450       */
451      public static final String RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS =
452          RM_PREFIX + "state-store.max-completed-applications";
453      public static final int DEFAULT_RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS =
454          DEFAULT_RM_MAX_COMPLETED_APPLICATIONS;
455    
456      /** Default application name */
457      public static final String DEFAULT_APPLICATION_NAME = "N/A";
458    
459      /** Default application type */
460      public static final String DEFAULT_APPLICATION_TYPE = "YARN";
461    
462      /** Default application type length */
463      public static final int APPLICATION_TYPE_LENGTH = 20;
464      
465      /** Default queue name */
466      public static final String DEFAULT_QUEUE_NAME = "default";
467    
468      /**
469       * Buckets (in minutes) for the number of apps running in each queue.
470       */
471      public static final String RM_METRICS_RUNTIME_BUCKETS =
472        RM_PREFIX + "metrics.runtime.buckets";
473    
474      /**
475       * Default sizes of the runtime metric buckets in minutes.
476       */
477      public static final String DEFAULT_RM_METRICS_RUNTIME_BUCKETS = 
478        "60,300,1440";
479    
480      public static final String RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = RM_PREFIX
481          + "am-rm-tokens.master-key-rolling-interval-secs";
482    
483      public static final long DEFAULT_RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS =
484          24 * 60 * 60;
485    
486      public static final String RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS =
487          RM_PREFIX + "container-tokens.master-key-rolling-interval-secs";
488    
489      public static final long DEFAULT_RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS =
490          24 * 60 * 60;
491    
492      public static final String RM_NMTOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS =
493          RM_PREFIX + "nm-tokens.master-key-rolling-interval-secs";
494      
495      public static final long DEFAULT_RM_NMTOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS =
496          24 * 60 * 60;
497    
498      public static final String RM_NODEMANAGER_MINIMUM_VERSION =
499          RM_PREFIX + "nodemanager.minimum.version";
500    
501      public static final String DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION =
502          "NONE";
503    
504      ////////////////////////////////
505      // Node Manager Configs
506      ////////////////////////////////
507      
508      /** Prefix for all node manager configs.*/
509      public static final String NM_PREFIX = "yarn.nodemanager.";
510    
511      /** Environment variables that will be sent to containers.*/
512      public static final String NM_ADMIN_USER_ENV = NM_PREFIX + "admin-env";
513      public static final String DEFAULT_NM_ADMIN_USER_ENV = "MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX";
514    
515      /** Environment variables that containers may override rather than use NodeManager's default.*/
516      public static final String NM_ENV_WHITELIST = NM_PREFIX + "env-whitelist";
517      public static final String DEFAULT_NM_ENV_WHITELIST = StringUtils.join(",",
518        Arrays.asList(ApplicationConstants.Environment.JAVA_HOME.key(),
519          ApplicationConstants.Environment.HADOOP_COMMON_HOME.key(),
520          ApplicationConstants.Environment.HADOOP_HDFS_HOME.key(),
521          ApplicationConstants.Environment.HADOOP_CONF_DIR.key(),
522          ApplicationConstants.Environment.HADOOP_YARN_HOME.key()));
523      
524      /** address of node manager IPC.*/
525      public static final String NM_ADDRESS = NM_PREFIX + "address";
526      public static final int DEFAULT_NM_PORT = 0;
527      public static final String DEFAULT_NM_ADDRESS = "0.0.0.0:"
528          + DEFAULT_NM_PORT;
529      
530      /** who will execute(launch) the containers.*/
531      public static final String NM_CONTAINER_EXECUTOR = 
532        NM_PREFIX + "container-executor.class";
533    
534      /**  
535       * Adjustment to make to the container os scheduling priority.
536       * The valid values for this could vary depending on the platform.
537       * On Linux, higher values mean run the containers at a less 
538       * favorable priority than the NM. 
539       * The value specified is an int.
540       */
541      public static final String NM_CONTAINER_EXECUTOR_SCHED_PRIORITY = 
542        NM_PREFIX + "container-executor.os.sched.priority.adjustment";
543      public static final int DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY = 0;
544      
545      /** Number of threads container manager uses.*/
546      public static final String NM_CONTAINER_MGR_THREAD_COUNT =
547        NM_PREFIX + "container-manager.thread-count";
548      public static final int DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT = 20;
549      
550      /** Number of threads used in cleanup.*/
551      public static final String NM_DELETE_THREAD_COUNT = 
552        NM_PREFIX +  "delete.thread-count";
553      public static final int DEFAULT_NM_DELETE_THREAD_COUNT = 4;
554      
555      /** Keytab for NM.*/
556      public static final String NM_KEYTAB = NM_PREFIX + "keytab";
557      
558      /**List of directories to store localized files in.*/
559      public static final String NM_LOCAL_DIRS = NM_PREFIX + "local-dirs";
560      public static final String DEFAULT_NM_LOCAL_DIRS = "/tmp/nm-local-dir";
561    
562      /**
563       * Number of files in each localized directories
564       * Avoid tuning this too low. 
565       */
566      public static final String NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY =
567        NM_PREFIX + "local-cache.max-files-per-directory";
568      public static final int DEFAULT_NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY = 8192;
569    
570      /** Address where the localizer IPC is.*/
571      public static final String NM_LOCALIZER_ADDRESS =
572        NM_PREFIX + "localizer.address";
573      public static final int DEFAULT_NM_LOCALIZER_PORT = 8040;
574      public static final String DEFAULT_NM_LOCALIZER_ADDRESS = "0.0.0.0:" +
575        DEFAULT_NM_LOCALIZER_PORT;
576      
577      /** Interval in between cache cleanups.*/
578      public static final String NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS =
579        NM_PREFIX + "localizer.cache.cleanup.interval-ms";
580      public static final long DEFAULT_NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS = 
581        10 * 60 * 1000;
582      
583      /** Target size of localizer cache in MB, per local directory.*/
584      public static final String NM_LOCALIZER_CACHE_TARGET_SIZE_MB =
585        NM_PREFIX + "localizer.cache.target-size-mb";
586      public static final long DEFAULT_NM_LOCALIZER_CACHE_TARGET_SIZE_MB = 10 * 1024;
587      
588      /** Number of threads to handle localization requests.*/
589      public static final String NM_LOCALIZER_CLIENT_THREAD_COUNT =
590        NM_PREFIX + "localizer.client.thread-count";
591      public static final int DEFAULT_NM_LOCALIZER_CLIENT_THREAD_COUNT = 5;
592      
593      /** Number of threads to use for localization fetching.*/
594      public static final String NM_LOCALIZER_FETCH_THREAD_COUNT = 
595        NM_PREFIX + "localizer.fetch.thread-count";
596      public static final int DEFAULT_NM_LOCALIZER_FETCH_THREAD_COUNT = 4;
597    
598      /** Where to store container logs.*/
599      public static final String NM_LOG_DIRS = NM_PREFIX + "log-dirs";
600      public static final String DEFAULT_NM_LOG_DIRS = "/tmp/logs";
601    
602      public static final String NM_RESOURCEMANAGER_MINIMUM_VERSION =
603          NM_PREFIX + "resourcemanager.minimum.version";
604      public static final String DEFAULT_NM_RESOURCEMANAGER_MINIMUM_VERSION = "NONE";
605    
606      /** Interval at which the delayed token removal thread runs */
607      public static final String RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS =
608          RM_PREFIX + "delayed.delegation-token.removal-interval-ms";
609      public static final long DEFAULT_RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS =
610          30000l;
611      
612      /** Delegation Token renewer thread count */
613      public static final String RM_DELEGATION_TOKEN_RENEWER_THREAD_COUNT =
614          RM_PREFIX + "delegation-token-renewer.thread-count";
615      public static final int DEFAULT_RM_DELEGATION_TOKEN_RENEWER_THREAD_COUNT = 50;
616    
617      /** Whether to enable log aggregation */
618      public static final String LOG_AGGREGATION_ENABLED = YARN_PREFIX
619          + "log-aggregation-enable";
620      public static final boolean DEFAULT_LOG_AGGREGATION_ENABLED = false;
621      
622      /** 
623       * How long to wait before deleting aggregated logs, -1 disables.
624       * Be careful set this too small and you will spam the name node.
625       */
626      public static final String LOG_AGGREGATION_RETAIN_SECONDS = YARN_PREFIX
627          + "log-aggregation.retain-seconds";
628      public static final long DEFAULT_LOG_AGGREGATION_RETAIN_SECONDS = -1;
629      
630      /**
631       * How long to wait between aggregated log retention checks. If set to
632       * a value <= 0 then the value is computed as one-tenth of the log retention
633       * setting. Be careful set this too small and you will spam the name node.
634       */
635      public static final String LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS =
636          YARN_PREFIX + "log-aggregation.retain-check-interval-seconds";
637      public static final long DEFAULT_LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS = -1;
638    
639      /**
640       * Number of seconds to retain logs on the NodeManager. Only applicable if Log
641       * aggregation is disabled
642       */
643      public static final String NM_LOG_RETAIN_SECONDS = NM_PREFIX
644          + "log.retain-seconds";
645      public static final long DEFAULT_NM_LOG_RETAIN_SECONDS = 3 * 60 * 60;
646    
647      /**
648       * Number of threads used in log cleanup. Only applicable if Log aggregation
649       * is disabled
650       */
651      public static final String NM_LOG_DELETION_THREADS_COUNT = 
652        NM_PREFIX +  "log.deletion-threads-count";
653      public static final int DEFAULT_NM_LOG_DELETE_THREAD_COUNT = 4;
654    
655      /** Where to aggregate logs to.*/
656      public static final String NM_REMOTE_APP_LOG_DIR = 
657        NM_PREFIX + "remote-app-log-dir";
658      public static final String DEFAULT_NM_REMOTE_APP_LOG_DIR = "/tmp/logs";
659    
660      /**
661       * The remote log dir will be created at
662       * NM_REMOTE_APP_LOG_DIR/${user}/NM_REMOTE_APP_LOG_DIR_SUFFIX/${appId}
663       */
664      public static final String NM_REMOTE_APP_LOG_DIR_SUFFIX = 
665        NM_PREFIX + "remote-app-log-dir-suffix";
666      public static final String DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX="logs";
667    
668      public static final String YARN_LOG_SERVER_URL =
669        YARN_PREFIX + "log.server.url";
670      
671      public static final String YARN_TRACKING_URL_GENERATOR = 
672          YARN_PREFIX + "tracking.url.generator";
673    
674      /** Amount of memory in GB that can be allocated for containers.*/
675      public static final String NM_PMEM_MB = NM_PREFIX + "resource.memory-mb";
676      public static final int DEFAULT_NM_PMEM_MB = 8 * 1024;
677    
678      /** Specifies whether physical memory check is enabled. */
679      public static final String NM_PMEM_CHECK_ENABLED = NM_PREFIX
680          + "pmem-check-enabled";
681      public static final boolean DEFAULT_NM_PMEM_CHECK_ENABLED = true;
682    
683      /** Specifies whether physical memory check is enabled. */
684      public static final String NM_VMEM_CHECK_ENABLED = NM_PREFIX
685          + "vmem-check-enabled";
686      public static final boolean DEFAULT_NM_VMEM_CHECK_ENABLED = true;
687    
688      /** Conversion ratio for physical memory to virtual memory. */
689      public static final String NM_VMEM_PMEM_RATIO =
690        NM_PREFIX + "vmem-pmem-ratio";
691      public static final float DEFAULT_NM_VMEM_PMEM_RATIO = 2.1f;
692      
693      /** Number of Virtual CPU Cores which can be allocated for containers.*/
694      public static final String NM_VCORES = NM_PREFIX + "resource.cpu-vcores";
695      public static final int DEFAULT_NM_VCORES = 8;
696      
697      /** NM Webapp address.**/
698      public static final String NM_WEBAPP_ADDRESS = NM_PREFIX + "webapp.address";
699      public static final int DEFAULT_NM_WEBAPP_PORT = 8042;
700      public static final String DEFAULT_NM_WEBAPP_ADDRESS = "0.0.0.0:" +
701        DEFAULT_NM_WEBAPP_PORT;
702      
703      /** NM Webapp https address.**/
704      public static final String NM_WEBAPP_HTTPS_ADDRESS = NM_PREFIX
705          + "webapp.https.address";
706      public static final int DEFAULT_NM_WEBAPP_HTTPS_PORT = 8044;
707      public static final String DEFAULT_NM_WEBAPP_HTTPS_ADDRESS = "0.0.0.0:"
708          + DEFAULT_NM_WEBAPP_HTTPS_PORT; 
709      
710      /** How often to monitor containers.*/
711      public final static String NM_CONTAINER_MON_INTERVAL_MS =
712        NM_PREFIX + "container-monitor.interval-ms";
713      public final static int DEFAULT_NM_CONTAINER_MON_INTERVAL_MS = 3000;
714    
715      /** Class that calculates containers current resource utilization.*/
716      public static final String NM_CONTAINER_MON_RESOURCE_CALCULATOR =
717        NM_PREFIX + "container-monitor.resource-calculator.class";
718      /** Class that calculates process tree resource utilization.*/
719      public static final String NM_CONTAINER_MON_PROCESS_TREE =
720        NM_PREFIX + "container-monitor.process-tree.class";
721      public static final String PROCFS_USE_SMAPS_BASED_RSS_ENABLED = NM_PREFIX +
722          ".container-monitor.procfs-tree.smaps-based-rss.enabled";
723      public static final boolean DEFAULT_PROCFS_USE_SMAPS_BASED_RSS_ENABLED =
724          false;
725      
726      /** Prefix for all node manager disk health checker configs. */
727      private static final String NM_DISK_HEALTH_CHECK_PREFIX =
728          "yarn.nodemanager.disk-health-checker.";
729      /**
730       * Enable/Disable disks' health checker. Default is true. An expert level
731       * configuration property.
732       */
733      public static final String NM_DISK_HEALTH_CHECK_ENABLE =
734          NM_DISK_HEALTH_CHECK_PREFIX + "enable";
735      /** Frequency of running disks' health checker. */
736      public static final String NM_DISK_HEALTH_CHECK_INTERVAL_MS =
737          NM_DISK_HEALTH_CHECK_PREFIX + "interval-ms";
738      /** By default, disks' health is checked every 2 minutes. */
739      public static final long DEFAULT_NM_DISK_HEALTH_CHECK_INTERVAL_MS =
740          2 * 60 * 1000;
741    
742      /**
743       * The minimum fraction of number of disks to be healthy for the nodemanager
744       * to launch new containers. This applies to nm-local-dirs and nm-log-dirs.
745       */
746      public static final String NM_MIN_HEALTHY_DISKS_FRACTION =
747          NM_DISK_HEALTH_CHECK_PREFIX + "min-healthy-disks";
748      /**
749       * By default, at least 25% of disks are to be healthy to say that the node is
750       * healthy in terms of disks.
751       */
752      public static final float DEFAULT_NM_MIN_HEALTHY_DISKS_FRACTION = 0.25F;
753    
754      /**
755       * The maximum percentage of disk space that can be used after which a disk is
756       * marked as offline. Values can range from 0.0 to 100.0. If the value is
757       * greater than or equal to 100, NM will check for full disk. This applies to
758       * nm-local-dirs and nm-log-dirs.
759       */
760      public static final String NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE =
761          NM_DISK_HEALTH_CHECK_PREFIX + "max-disk-utilization-per-disk-percentage";
762      /**
763       * By default, 100% of the disk can be used before it is marked as offline.
764       */
765      public static final float DEFAULT_NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE =
766          100.0F;
767    
768      /**
769       * The minimum space that must be available on a local dir for it to be used.
770       * This applies to nm-local-dirs and nm-log-dirs.
771       */
772      public static final String NM_MIN_PER_DISK_FREE_SPACE_MB =
773          NM_DISK_HEALTH_CHECK_PREFIX + "min-free-space-per-disk-mb";
774      /**
775       * By default, all of the disk can be used before it is marked as offline.
776       */
777      public static final long DEFAULT_NM_MIN_PER_DISK_FREE_SPACE_MB = 0;
778    
779      /** Frequency of running node health script.*/
780      public static final String NM_HEALTH_CHECK_INTERVAL_MS = 
781        NM_PREFIX + "health-checker.interval-ms";
782      public static final long DEFAULT_NM_HEALTH_CHECK_INTERVAL_MS = 10 * 60 * 1000;
783    
784      /** Health check script time out period.*/  
785      public static final String NM_HEALTH_CHECK_SCRIPT_TIMEOUT_MS = 
786        NM_PREFIX + "health-checker.script.timeout-ms";
787      public static final long DEFAULT_NM_HEALTH_CHECK_SCRIPT_TIMEOUT_MS = 
788        2 * DEFAULT_NM_HEALTH_CHECK_INTERVAL_MS;
789      
790      /** The health check script to run.*/
791      public static final String NM_HEALTH_CHECK_SCRIPT_PATH = 
792        NM_PREFIX + "health-checker.script.path";
793      
794      /** The arguments to pass to the health check script.*/
795      public static final String NM_HEALTH_CHECK_SCRIPT_OPTS = 
796        NM_PREFIX + "health-checker.script.opts";
797      
798      /** The path to the Linux container executor.*/
799      public static final String NM_LINUX_CONTAINER_EXECUTOR_PATH =
800        NM_PREFIX + "linux-container-executor.path";
801      
802      /** 
803       * The UNIX group that the linux-container-executor should run as.
804       * This is intended to be set as part of container-executor.cfg. 
805       */
806      public static final String NM_LINUX_CONTAINER_GROUP =
807        NM_PREFIX + "linux-container-executor.group";
808    
809      /**
810       * The UNIX user that containers will run as when Linux-container-executor
811       * is used in nonsecure mode (a use case for this is using cgroups).
812       */
813      public static final String NM_NONSECURE_MODE_LOCAL_USER_KEY = NM_PREFIX +
814          "linux-container-executor.nonsecure-mode.local-user";
815    
816      public static final String DEFAULT_NM_NONSECURE_MODE_LOCAL_USER = "nobody";
817    
818      /**
819       * The allowed pattern for UNIX user names enforced by 
820       * Linux-container-executor when used in nonsecure mode (use case for this 
821       * is using cgroups). The default value is taken from /usr/sbin/adduser
822       */
823      public static final String NM_NONSECURE_MODE_USER_PATTERN_KEY = NM_PREFIX +
824          "linux-container-executor.nonsecure-mode.user-pattern";
825    
826      public static final String DEFAULT_NM_NONSECURE_MODE_USER_PATTERN = 
827          "^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$";
828    
829      /** The type of resource enforcement to use with the
830       *  linux container executor.
831       */
832      public static final String NM_LINUX_CONTAINER_RESOURCES_HANDLER = 
833      NM_PREFIX + "linux-container-executor.resources-handler.class";
834      
835      /** The path the linux container executor should use for cgroups */
836      public static final String NM_LINUX_CONTAINER_CGROUPS_HIERARCHY =
837        NM_PREFIX + "linux-container-executor.cgroups.hierarchy";
838      
839      /** Whether the linux container executor should mount cgroups if not found */
840      public static final String NM_LINUX_CONTAINER_CGROUPS_MOUNT =
841        NM_PREFIX + "linux-container-executor.cgroups.mount";
842      
843      /** Where the linux container executor should mount cgroups if not found */
844      public static final String NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH =
845        NM_PREFIX + "linux-container-executor.cgroups.mount-path";
846    
847    
848      /**
849       * Interval of time the linux container executor should try cleaning up
850       * cgroups entry when cleaning up a container. This is required due to what 
851       * it seems a race condition because the SIGTERM/SIGKILL is asynch.
852       */
853      public static final String NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT =
854       NM_PREFIX + "linux-container-executor.cgroups.delete-timeout-ms";
855    
856      public static final long DEFAULT_NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT =
857          1000;
858    
859      /** T-file compression types used to compress aggregated logs.*/
860      public static final String NM_LOG_AGG_COMPRESSION_TYPE = 
861        NM_PREFIX + "log-aggregation.compression-type";
862      public static final String DEFAULT_NM_LOG_AGG_COMPRESSION_TYPE = "none";
863      
864      /** The kerberos principal for the node manager.*/
865      public static final String NM_PRINCIPAL =
866        NM_PREFIX + "principal";
867      
868      public static final String NM_AUX_SERVICES = 
869        NM_PREFIX + "aux-services";
870      
871      public static final String NM_AUX_SERVICE_FMT =
872        NM_PREFIX + "aux-services.%s.class";
873    
874      public static final String NM_USER_HOME_DIR =
875          NM_PREFIX + "user-home-dir";
876      
877      /**The kerberos principal to be used for spnego filter for NM.*/
878      public static final String NM_WEBAPP_SPNEGO_USER_NAME_KEY =
879          NM_PREFIX + "webapp.spnego-principal";
880      
881      /**The kerberos keytab to be used for spnego filter for NM.*/
882      public static final String NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY =
883          NM_PREFIX + "webapp.spnego-keytab-file";
884      
885      public static final String DEFAULT_NM_USER_HOME_DIR= "/home/";
886    
887      ////////////////////////////////
888      // Web Proxy Configs
889      ////////////////////////////////
890      public static final String PROXY_PREFIX = "yarn.web-proxy.";
891      
892      /** The kerberos principal for the proxy.*/
893      public static final String PROXY_PRINCIPAL =
894        PROXY_PREFIX + "principal";
895      
896      /** Keytab for Proxy.*/
897      public static final String PROXY_KEYTAB = PROXY_PREFIX + "keytab";
898      
899      /** The address for the web proxy.*/
900      public static final String PROXY_ADDRESS =
901        PROXY_PREFIX + "address";
902      public static final int DEFAULT_PROXY_PORT = 9099;
903      public static final String DEFAULT_PROXY_ADDRESS =
904        "0.0.0.0:" + DEFAULT_RM_PORT;
905      
906      /**
907       * YARN Service Level Authorization
908       */
909      public static final String 
910      YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCETRACKER_PROTOCOL =
911          "security.resourcetracker.protocol.acl";
912      public static final String 
913      YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONCLIENT_PROTOCOL =
914          "security.applicationclient.protocol.acl";
915      public static final String 
916      YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCEMANAGER_ADMINISTRATION_PROTOCOL =
917          "security.resourcemanager-administration.protocol.acl";
918      public static final String 
919      YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONMASTER_PROTOCOL =
920          "security.applicationmaster.protocol.acl";
921    
922      public static final String 
923      YARN_SECURITY_SERVICE_AUTHORIZATION_CONTAINER_MANAGEMENT_PROTOCOL =
924          "security.containermanagement.protocol.acl";
925      public static final String 
926      YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCE_LOCALIZER =
927          "security.resourcelocalizer.protocol.acl";
928    
929      /** No. of milliseconds to wait between sending a SIGTERM and SIGKILL
930       * to a running container */
931      public static final String NM_SLEEP_DELAY_BEFORE_SIGKILL_MS =
932          NM_PREFIX + "sleep-delay-before-sigkill.ms";
933      public static final long DEFAULT_NM_SLEEP_DELAY_BEFORE_SIGKILL_MS =
934          250;
935    
936      /** Max time to wait for a process to come up when trying to cleanup
937       * container resources */
938      public static final String NM_PROCESS_KILL_WAIT_MS =
939          NM_PREFIX + "process-kill-wait.ms";
940      public static final long DEFAULT_NM_PROCESS_KILL_WAIT_MS =
941          2000;
942    
943      /** Max time to wait to establish a connection to RM */
944      public static final String RESOURCEMANAGER_CONNECT_MAX_WAIT_MS =
945          RM_PREFIX + "connect.max-wait.ms";
946      public static final int DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS =
947          15 * 60 * 1000;
948    
949      /** Time interval between each attempt to connect to RM */
950      public static final String RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS =
951          RM_PREFIX + "connect.retry-interval.ms";
952      public static final long DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS
953          = 30 * 1000;
954    
955      /**
956       * CLASSPATH for YARN applications. A comma-separated list of CLASSPATH
957       * entries
958       */
959      public static final String YARN_APPLICATION_CLASSPATH = YARN_PREFIX
960          + "application.classpath";
961    
962      /**
963       * Default platform-agnostic CLASSPATH for YARN applications. A
964       * comma-separated list of CLASSPATH entries. The parameter expansion marker
965       * will be replaced with real parameter expansion marker ('%' for Windows and
966       * '$' for Linux) by NodeManager on container launch. For example: {{VAR}}
967       * will be replaced as $VAR on Linux, and %VAR% on Windows.
968       */
969      @Public
970      @Unstable
971      public static final String[] DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH= {
972          ApplicationConstants.Environment.HADOOP_CONF_DIR.$$(),
973          ApplicationConstants.Environment.HADOOP_COMMON_HOME.$$()
974              + "/share/hadoop/common/*",
975          ApplicationConstants.Environment.HADOOP_COMMON_HOME.$$()
976              + "/share/hadoop/common/lib/*",
977          ApplicationConstants.Environment.HADOOP_HDFS_HOME.$$()
978              + "/share/hadoop/hdfs/*",
979          ApplicationConstants.Environment.HADOOP_HDFS_HOME.$$()
980              + "/share/hadoop/hdfs/lib/*",
981          ApplicationConstants.Environment.HADOOP_YARN_HOME.$$()
982              + "/share/hadoop/yarn/*",
983          ApplicationConstants.Environment.HADOOP_YARN_HOME.$$()
984              + "/share/hadoop/yarn/lib/*" };
985      /**
986       * <p>
987       * Default platform-specific CLASSPATH for YARN applications. A
988       * comma-separated list of CLASSPATH entries constructed based on the client
989       * OS environment expansion syntax.
990       * </p>
991       * <p>
992       * Note: Use {@link DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH} for
993       * cross-platform practice i.e. submit an application from a Windows client to
994       * a Linux/Unix server or vice versa.
995       * </p>
996       */
997      public static final String[] DEFAULT_YARN_APPLICATION_CLASSPATH = {
998          ApplicationConstants.Environment.HADOOP_CONF_DIR.$(),
999          ApplicationConstants.Environment.HADOOP_COMMON_HOME.$()
1000              + "/share/hadoop/common/*",
1001          ApplicationConstants.Environment.HADOOP_COMMON_HOME.$()
1002              + "/share/hadoop/common/lib/*",
1003          ApplicationConstants.Environment.HADOOP_HDFS_HOME.$()
1004              + "/share/hadoop/hdfs/*",
1005          ApplicationConstants.Environment.HADOOP_HDFS_HOME.$()
1006              + "/share/hadoop/hdfs/lib/*",
1007          ApplicationConstants.Environment.HADOOP_YARN_HOME.$()
1008              + "/share/hadoop/yarn/*",
1009          ApplicationConstants.Environment.HADOOP_YARN_HOME.$()
1010              + "/share/hadoop/yarn/lib/*" };
1011    
1012      /** Container temp directory */
1013      public static final String DEFAULT_CONTAINER_TEMP_DIR = "./tmp";
1014    
1015      public static final String IS_MINI_YARN_CLUSTER = YARN_PREFIX
1016          + "is.minicluster";
1017    
1018      public static final String YARN_MC_PREFIX = YARN_PREFIX + "minicluster.";
1019    
1020      /** Whether to use fixed ports with the minicluster. */
1021      public static final String YARN_MINICLUSTER_FIXED_PORTS =
1022          YARN_MC_PREFIX + "fixed.ports";
1023    
1024      /**
1025       * Default is false to be able to run tests concurrently without port
1026       * conflicts.
1027       */
1028      public static final boolean DEFAULT_YARN_MINICLUSTER_FIXED_PORTS = false;
1029    
1030      /**
1031       * Whether the NM should use RPC to connect to the RM. Default is false.
1032       * Can be set to true only when using fixed ports.
1033       */
1034      public static final String YARN_MINICLUSTER_USE_RPC = YARN_MC_PREFIX + "use-rpc";
1035      public static final boolean DEFAULT_YARN_MINICLUSTER_USE_RPC = false;
1036    
1037      /**
1038       * Whether users are explicitly trying to control resource monitoring
1039       * configuration for the MiniYARNCluster. Disabled by default.
1040       */
1041      public static final String YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING =
1042          YARN_MC_PREFIX + "control-resource-monitoring";
1043      public static final boolean
1044          DEFAULT_YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING = false;
1045    
1046      /** The log directory for the containers */
1047      public static final String YARN_APP_CONTAINER_LOG_DIR =
1048          YARN_PREFIX + "app.container.log.dir";
1049    
1050      public static final String YARN_APP_CONTAINER_LOG_SIZE =
1051          YARN_PREFIX + "app.container.log.filesize";
1052    
1053      public static final String YARN_APP_CONTAINER_LOG_BACKUPS =
1054          YARN_PREFIX + "app.container.log.backups";
1055    
1056      ////////////////////////////////
1057      // Timeline Service Configs
1058      ////////////////////////////////
1059    
1060      public static final String TIMELINE_SERVICE_PREFIX =
1061          YARN_PREFIX + "timeline-service.";
1062    
1063    
1064      // mark app-history related configs @Private as application history is going
1065      // to be integrated into the timeline service
1066      @Private
1067      public static final String APPLICATION_HISTORY_PREFIX =
1068          TIMELINE_SERVICE_PREFIX + "generic-application-history.";
1069    
1070      /**
1071       *  The setting that controls whether application history service is
1072       *  enabled or not.
1073       */
1074      @Private
1075      public static final String APPLICATION_HISTORY_ENABLED =
1076          APPLICATION_HISTORY_PREFIX + "enabled";
1077      @Private
1078      public static final boolean DEFAULT_APPLICATION_HISTORY_ENABLED = false;
1079    
1080      /** Application history store class */
1081      @Private
1082      public static final String APPLICATION_HISTORY_STORE =
1083          APPLICATION_HISTORY_PREFIX + "store-class";
1084    
1085      /** URI for FileSystemApplicationHistoryStore */
1086      @Private
1087      public static final String FS_APPLICATION_HISTORY_STORE_URI =
1088          APPLICATION_HISTORY_PREFIX + "fs-history-store.uri";
1089    
1090      /** T-file compression types used to compress history data.*/
1091      @Private
1092      public static final String FS_APPLICATION_HISTORY_STORE_COMPRESSION_TYPE =
1093          APPLICATION_HISTORY_PREFIX + "fs-history-store.compression-type";
1094      @Private
1095      public static final String DEFAULT_FS_APPLICATION_HISTORY_STORE_COMPRESSION_TYPE =
1096          "none";
1097    
1098      /** The setting that controls whether timeline service is enabled or not. */
1099      public static final String TIMELINE_SERVICE_ENABLED =
1100          TIMELINE_SERVICE_PREFIX + "enabled";
1101      public static final boolean DEFAULT_TIMELINE_SERVICE_ENABLED = false;
1102    
1103      /** host:port address for timeline service RPC APIs. */
1104      public static final String TIMELINE_SERVICE_ADDRESS =
1105          TIMELINE_SERVICE_PREFIX + "address";
1106      public static final int DEFAULT_TIMELINE_SERVICE_PORT = 10200;
1107      public static final String DEFAULT_TIMELINE_SERVICE_ADDRESS = "0.0.0.0:"
1108          + DEFAULT_TIMELINE_SERVICE_PORT;
1109    
1110      /** The number of threads to handle client RPC API requests. */
1111      public static final String TIMELINE_SERVICE_HANDLER_THREAD_COUNT =
1112          TIMELINE_SERVICE_PREFIX + "handler-thread-count";
1113      public static final int DEFAULT_TIMELINE_SERVICE_CLIENT_THREAD_COUNT = 10;
1114      
1115    
1116      /** The address of the timeline service web application.*/
1117      public static final String TIMELINE_SERVICE_WEBAPP_ADDRESS =
1118          TIMELINE_SERVICE_PREFIX  + "webapp.address";
1119    
1120      public static final int DEFAULT_TIMELINE_SERVICE_WEBAPP_PORT = 8188;
1121      public static final String DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS =
1122          "0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_WEBAPP_PORT;
1123    
1124      /** The https address of the timeline service web application.*/
1125      public static final String TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS =
1126          TIMELINE_SERVICE_PREFIX + "webapp.https.address";
1127    
1128      public static final int DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT = 8190;
1129      public static final String DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS =
1130          "0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT;
1131    
1132      /**The kerberos principal to be used for spnego filter for timeline service.*/
1133      public static final String TIMELINE_SERVICE_WEBAPP_SPNEGO_USER_NAME_KEY =
1134          TIMELINE_SERVICE_PREFIX + "webapp.spnego-principal";
1135    
1136      /**The kerberos keytab to be used for spnego filter for timeline service.*/
1137      public static final String TIMELINE_SERVICE_WEBAPP_SPNEGO_KEYTAB_FILE_KEY =
1138          TIMELINE_SERVICE_PREFIX + "webapp.spnego-keytab-file";
1139    
1140      /** Timeline service store class */
1141      public static final String TIMELINE_SERVICE_STORE =
1142          TIMELINE_SERVICE_PREFIX + "store-class";
1143    
1144      /** Timeline service enable data age off */
1145      public static final String TIMELINE_SERVICE_TTL_ENABLE =
1146          TIMELINE_SERVICE_PREFIX + "ttl-enable";
1147    
1148      /** Timeline service length of time to retain data */
1149      public static final String TIMELINE_SERVICE_TTL_MS =
1150          TIMELINE_SERVICE_PREFIX + "ttl-ms";
1151    
1152      public static final long DEFAULT_TIMELINE_SERVICE_TTL_MS =
1153          1000 * 60 * 60 * 24 * 7;
1154    
1155      public static final String TIMELINE_SERVICE_LEVELDB_PREFIX =
1156          TIMELINE_SERVICE_PREFIX + "leveldb-timeline-store.";
1157    
1158      /** Timeline service leveldb path */
1159      public static final String TIMELINE_SERVICE_LEVELDB_PATH =
1160          TIMELINE_SERVICE_LEVELDB_PREFIX + "path";
1161    
1162      /** Timeline service leveldb read cache (uncompressed blocks) */
1163      public static final String TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE =
1164          TIMELINE_SERVICE_LEVELDB_PREFIX + "read-cache-size";
1165    
1166      public static final long DEFAULT_TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE =
1167          100 * 1024 * 1024;
1168    
1169      /** Timeline service leveldb start time read cache (number of entities) */
1170      public static final String
1171          TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE =
1172          TIMELINE_SERVICE_LEVELDB_PREFIX + "start-time-read-cache-size";
1173    
1174      public static final int
1175          DEFAULT_TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE = 10000;
1176    
1177      /** Timeline service leveldb start time write cache (number of entities) */
1178      public static final String
1179          TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE =
1180          TIMELINE_SERVICE_LEVELDB_PREFIX + "start-time-write-cache-size";
1181    
1182      public static final int
1183          DEFAULT_TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE = 10000;
1184    
1185      /** Timeline service leveldb interval to wait between deletion rounds */
1186      public static final String TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS =
1187          TIMELINE_SERVICE_LEVELDB_PREFIX + "ttl-interval-ms";
1188    
1189      public static final long DEFAULT_TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS =
1190          1000 * 60 * 5;
1191    
1192      ////////////////////////////////
1193      // Other Configs
1194      ////////////////////////////////
1195    
1196      /**
1197       * Use YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS instead.
1198       * The interval of the yarn client's querying application state after
1199       * application submission. The unit is millisecond.
1200       */
1201      @Deprecated
1202      public static final String YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS =
1203          YARN_PREFIX + "client.app-submission.poll-interval";
1204    
1205      /**
1206       * The interval that the yarn client library uses to poll the completion
1207       * status of the asynchronous API of application client protocol.
1208       */
1209      public static final String YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS =
1210          YARN_PREFIX + "client.application-client-protocol.poll-interval-ms";
1211      public static final long DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS =
1212          200;
1213    
1214      /**
1215       * The duration that the yarn client library waits, cumulatively across polls,
1216       * for an expected state change to occur. Defaults to -1, which indicates no
1217       * limit.
1218       */
1219      public static final String YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS =
1220          YARN_PREFIX + "client.application-client-protocol.poll-timeout-ms";
1221      public static final long DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS =
1222          -1;
1223    
1224      /**
1225       * Max number of threads in NMClientAsync to process container management
1226       * events
1227       */
1228      public static final String NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE =
1229          YARN_PREFIX + "client.nodemanager-client-async.thread-pool-max-size";
1230      public static final int DEFAULT_NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE = 500;
1231    
1232      /**
1233       * Maximum number of proxy connections for node manager. It should always be
1234       * more than 1. NMClient and MRAppMaster will use this to cache connection
1235       * with node manager. There will be at max one connection per node manager.
1236       * Ex. configuring it to a value of 5 will make sure that client will at
1237       * max have 5 connections cached with 5 different node managers. These
1238       * connections will be timed out if idle for more than system wide idle
1239       * timeout period. The token if used for authentication then it will be used
1240       * only at connection creation time. If new token is received then earlier
1241       * connection should be closed in order to use newer token.
1242       * Note: {@link YarnConfiguration#NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE}
1243       * are related to each other.
1244       */
1245      public static final String NM_CLIENT_MAX_NM_PROXIES =
1246          YARN_PREFIX + "client.max-nodemanagers-proxies";
1247      public static final int DEFAULT_NM_CLIENT_MAX_NM_PROXIES = 500;
1248    
1249      public static final String YARN_HTTP_POLICY_KEY = YARN_PREFIX + "http.policy";
1250      public static final String YARN_HTTP_POLICY_DEFAULT = HttpConfig.Policy.HTTP_ONLY
1251          .name();
1252    
1253      public YarnConfiguration() {
1254        super();
1255      }
1256      
1257      public YarnConfiguration(Configuration conf) {
1258        super(conf);
1259        if (! (conf instanceof YarnConfiguration)) {
1260          this.reloadConfiguration();
1261        }
1262      }
1263    
1264      @Private
1265      public static List<String> getServiceAddressConfKeys(Configuration conf) {
1266        return useHttps(conf) ? RM_SERVICES_ADDRESS_CONF_KEYS_HTTPS
1267            : RM_SERVICES_ADDRESS_CONF_KEYS_HTTP;
1268      }
1269    
1270      /**
1271       * Get the socket address for <code>name</code> property as a
1272       * <code>InetSocketAddress</code>. On a HA cluster,
1273       * this fetches the address corresponding to the RM identified by
1274       * {@link #RM_HA_ID}.
1275       * @param name property name.
1276       * @param defaultAddress the default value
1277       * @param defaultPort the default port
1278       * @return InetSocketAddress
1279       */
1280      @Override
1281      public InetSocketAddress getSocketAddr(
1282          String name, String defaultAddress, int defaultPort) {
1283        String address;
1284        if (HAUtil.isHAEnabled(this) && getServiceAddressConfKeys(this).contains(name)) {
1285          address = HAUtil.getConfValueForRMInstance(name, defaultAddress, this);
1286        } else {
1287          address = get(name, defaultAddress);
1288        }
1289        return NetUtils.createSocketAddr(address, defaultPort, name);
1290      }
1291    
1292      @Override
1293      public InetSocketAddress updateConnectAddr(String name,
1294                                                 InetSocketAddress addr) {
1295        String prefix = name;
1296        if (HAUtil.isHAEnabled(this)) {
1297          prefix = HAUtil.addSuffix(prefix, HAUtil.getRMHAId(this));
1298        }
1299        return super.updateConnectAddr(prefix, addr);
1300      }
1301    
1302      @Private
1303      public static int getRMDefaultPortNumber(String addressPrefix,
1304          Configuration conf) {
1305        if (addressPrefix.equals(YarnConfiguration.RM_ADDRESS)) {
1306          return YarnConfiguration.DEFAULT_RM_PORT;
1307        } else if (addressPrefix.equals(YarnConfiguration.RM_SCHEDULER_ADDRESS)) {
1308          return YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT;
1309        } else if (addressPrefix.equals(YarnConfiguration.RM_WEBAPP_ADDRESS)) {
1310          return YarnConfiguration.DEFAULT_RM_WEBAPP_PORT;
1311        } else if (addressPrefix.equals(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS)) {
1312          return YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT;
1313        } else if (addressPrefix
1314            .equals(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS)) {
1315          return YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT;
1316        } else if (addressPrefix.equals(YarnConfiguration.RM_ADMIN_ADDRESS)) {
1317          return YarnConfiguration.DEFAULT_RM_ADMIN_PORT;
1318        } else {
1319          throw new HadoopIllegalArgumentException(
1320              "Invalid RM RPC address Prefix: " + addressPrefix
1321                  + ". The valid value should be one of "
1322                  + getServiceAddressConfKeys(conf));
1323        }
1324      }
1325    
1326      public static boolean useHttps(Configuration conf) {
1327        return HttpConfig.Policy.HTTPS_ONLY == HttpConfig.Policy.fromString(conf
1328            .get(YARN_HTTP_POLICY_KEY,
1329                YARN_HTTP_POLICY_DEFAULT));
1330      }
1331    
1332      @Private
1333      public static String getClusterId(Configuration conf) {
1334        String clusterId = conf.get(YarnConfiguration.RM_CLUSTER_ID);
1335        if (clusterId == null) {
1336          throw new HadoopIllegalArgumentException("Configuration doesn't specify" +
1337              YarnConfiguration.RM_CLUSTER_ID);
1338        }
1339        return clusterId;
1340      }
1341    }