001/** 002* Licensed to the Apache Software Foundation (ASF) under one 003* or more contributor license agreements. See the NOTICE file 004* distributed with this work for additional information 005* regarding copyright ownership. The ASF licenses this file 006* to you under the Apache License, Version 2.0 (the 007* "License"); you may not use this file except in compliance 008* with the License. You may obtain a copy of the License at 009* 010* http://www.apache.org/licenses/LICENSE-2.0 011* 012* Unless required by applicable law or agreed to in writing, software 013* distributed under the License is distributed on an "AS IS" BASIS, 014* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015* See the License for the specific language governing permissions and 016* limitations under the License. 017*/ 018 019package org.apache.hadoop.yarn.conf; 020 021import java.net.InetSocketAddress; 022import java.util.Arrays; 023import java.util.Collections; 024import java.util.List; 025 026import org.apache.hadoop.HadoopIllegalArgumentException; 027import org.apache.hadoop.classification.InterfaceAudience.Private; 028import org.apache.hadoop.classification.InterfaceAudience.Public; 029import org.apache.hadoop.classification.InterfaceStability.Evolving; 030import org.apache.hadoop.classification.InterfaceStability.Unstable; 031import org.apache.hadoop.conf.Configuration; 032import org.apache.hadoop.http.HttpConfig; 033import org.apache.hadoop.net.NetUtils; 034import org.apache.hadoop.util.StringUtils; 035import org.apache.hadoop.yarn.api.ApplicationConstants; 036 037@Public 038@Evolving 039public class YarnConfiguration extends Configuration { 040 041 @Private 042 public static final String CS_CONFIGURATION_FILE= "capacity-scheduler.xml"; 043 044 @Private 045 public static final String HADOOP_POLICY_CONFIGURATION_FILE = 046 "hadoop-policy.xml"; 047 048 @Private 049 public static final String YARN_SITE_CONFIGURATION_FILE = "yarn-site.xml"; 050 051 private static final String YARN_DEFAULT_CONFIGURATION_FILE = 052 "yarn-default.xml"; 053 054 @Private 055 public static final String CORE_SITE_CONFIGURATION_FILE = "core-site.xml"; 056 057 @Private 058 public static final List<String> RM_CONFIGURATION_FILES = 059 Collections.unmodifiableList(Arrays.asList( 060 CS_CONFIGURATION_FILE, 061 HADOOP_POLICY_CONFIGURATION_FILE, 062 YARN_SITE_CONFIGURATION_FILE, 063 CORE_SITE_CONFIGURATION_FILE)); 064 065 @Evolving 066 public static final int APPLICATION_MAX_TAGS = 10; 067 068 @Evolving 069 public static final int APPLICATION_MAX_TAG_LENGTH = 100; 070 071 static { 072 addDeprecatedKeys(); 073 Configuration.addDefaultResource(YARN_DEFAULT_CONFIGURATION_FILE); 074 Configuration.addDefaultResource(YARN_SITE_CONFIGURATION_FILE); 075 } 076 077 private static void addDeprecatedKeys() { 078 Configuration.addDeprecations(new DeprecationDelta[] { 079 new DeprecationDelta("yarn.client.max-nodemanagers-proxies", 080 NM_CLIENT_MAX_NM_PROXIES) 081 }); 082 } 083 084 //Configurations 085 086 public static final String YARN_PREFIX = "yarn."; 087 088 /** Delay before deleting resource to ease debugging of NM issues */ 089 public static final String DEBUG_NM_DELETE_DELAY_SEC = 090 YarnConfiguration.NM_PREFIX + "delete.debug-delay-sec"; 091 092 //////////////////////////////// 093 // IPC Configs 094 //////////////////////////////// 095 public static final String IPC_PREFIX = YARN_PREFIX + "ipc."; 096 097 /** Factory to create client IPC classes.*/ 098 public static final String IPC_CLIENT_FACTORY_CLASS = 099 IPC_PREFIX + "client.factory.class"; 100 public static final String DEFAULT_IPC_CLIENT_FACTORY_CLASS = 101 "org.apache.hadoop.yarn.factories.impl.pb.RpcClientFactoryPBImpl"; 102 103 /** Factory to create server IPC classes.*/ 104 public static final String IPC_SERVER_FACTORY_CLASS = 105 IPC_PREFIX + "server.factory.class"; 106 public static final String DEFAULT_IPC_SERVER_FACTORY_CLASS = 107 "org.apache.hadoop.yarn.factories.impl.pb.RpcServerFactoryPBImpl"; 108 109 /** Factory to create serializeable records.*/ 110 public static final String IPC_RECORD_FACTORY_CLASS = 111 IPC_PREFIX + "record.factory.class"; 112 public static final String DEFAULT_IPC_RECORD_FACTORY_CLASS = 113 "org.apache.hadoop.yarn.factories.impl.pb.RecordFactoryPBImpl"; 114 115 /** RPC class implementation*/ 116 public static final String IPC_RPC_IMPL = 117 IPC_PREFIX + "rpc.class"; 118 public static final String DEFAULT_IPC_RPC_IMPL = 119 "org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC"; 120 121 //////////////////////////////// 122 // Resource Manager Configs 123 //////////////////////////////// 124 public static final String RM_PREFIX = "yarn.resourcemanager."; 125 126 public static final String RM_CLUSTER_ID = RM_PREFIX + "cluster-id"; 127 128 public static final String RM_HOSTNAME = RM_PREFIX + "hostname"; 129 130 /** The address of the applications manager interface in the RM.*/ 131 public static final String RM_ADDRESS = 132 RM_PREFIX + "address"; 133 public static final int DEFAULT_RM_PORT = 8032; 134 public static final String DEFAULT_RM_ADDRESS = 135 "0.0.0.0:" + DEFAULT_RM_PORT; 136 137 /** The actual bind address for the RM.*/ 138 public static final String RM_BIND_HOST = 139 RM_PREFIX + "bind-host"; 140 141 /** The number of threads used to handle applications manager requests.*/ 142 public static final String RM_CLIENT_THREAD_COUNT = 143 RM_PREFIX + "client.thread-count"; 144 public static final int DEFAULT_RM_CLIENT_THREAD_COUNT = 50; 145 146 /** The Kerberos principal for the resource manager.*/ 147 public static final String RM_PRINCIPAL = 148 RM_PREFIX + "principal"; 149 150 /** The address of the scheduler interface.*/ 151 public static final String RM_SCHEDULER_ADDRESS = 152 RM_PREFIX + "scheduler.address"; 153 public static final int DEFAULT_RM_SCHEDULER_PORT = 8030; 154 public static final String DEFAULT_RM_SCHEDULER_ADDRESS = "0.0.0.0:" + 155 DEFAULT_RM_SCHEDULER_PORT; 156 157 /** Miniumum request grant-able by the RM scheduler. */ 158 public static final String RM_SCHEDULER_MINIMUM_ALLOCATION_MB = 159 YARN_PREFIX + "scheduler.minimum-allocation-mb"; 160 public static final int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB = 1024; 161 public static final String RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES = 162 YARN_PREFIX + "scheduler.minimum-allocation-vcores"; 163 public static final int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES = 1; 164 165 /** Maximum request grant-able by the RM scheduler. */ 166 public static final String RM_SCHEDULER_MAXIMUM_ALLOCATION_MB = 167 YARN_PREFIX + "scheduler.maximum-allocation-mb"; 168 public static final int DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB = 8192; 169 public static final String RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES = 170 YARN_PREFIX + "scheduler.maximum-allocation-vcores"; 171 public static final int DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES = 4; 172 173 /** Number of threads to handle scheduler interface.*/ 174 public static final String RM_SCHEDULER_CLIENT_THREAD_COUNT = 175 RM_PREFIX + "scheduler.client.thread-count"; 176 public static final int DEFAULT_RM_SCHEDULER_CLIENT_THREAD_COUNT = 50; 177 178 /** If the port should be included or not in the node name. The node name 179 * is used by the scheduler for resource requests allocation location 180 * matching. Typically this is just the hostname, using the port is needed 181 * when using minicluster and specific NM are required.*/ 182 public static final String RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME = 183 YARN_PREFIX + "scheduler.include-port-in-node-name"; 184 public static final boolean DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME = 185 false; 186 187 /** Enable Resource Manager webapp ui actions */ 188 public static final String RM_WEBAPP_UI_ACTIONS_ENABLED = 189 RM_PREFIX + "webapp.ui-actions.enabled"; 190 public static final boolean DEFAULT_RM_WEBAPP_UI_ACTIONS_ENABLED = 191 true; 192 193 /** Whether the RM should enable Reservation System */ 194 public static final String RM_RESERVATION_SYSTEM_ENABLE = RM_PREFIX 195 + "reservation-system.enable"; 196 public static final boolean DEFAULT_RM_RESERVATION_SYSTEM_ENABLE = false; 197 198 /** The class to use as the Reservation System. */ 199 public static final String RM_RESERVATION_SYSTEM_CLASS = RM_PREFIX 200 + "reservation-system.class"; 201 202 /** The PlanFollower for the Reservation System. */ 203 public static final String RM_RESERVATION_SYSTEM_PLAN_FOLLOWER = RM_PREFIX 204 + "reservation-system.plan.follower"; 205 206 /** The step size of the Reservation System. */ 207 public static final String RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP = 208 RM_PREFIX + "reservation-system.planfollower.time-step"; 209 public static final long DEFAULT_RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP = 210 1000L; 211 212 /** 213 * Enable periodic monitor threads. 214 * @see #RM_SCHEDULER_MONITOR_POLICIES 215 */ 216 public static final String RM_SCHEDULER_ENABLE_MONITORS = 217 RM_PREFIX + "scheduler.monitor.enable"; 218 public static final boolean DEFAULT_RM_SCHEDULER_ENABLE_MONITORS = false; 219 220 /** List of SchedulingEditPolicy classes affecting the scheduler. */ 221 public static final String RM_SCHEDULER_MONITOR_POLICIES = 222 RM_PREFIX + "scheduler.monitor.policies"; 223 224 /** The address of the RM web application.*/ 225 public static final String RM_WEBAPP_ADDRESS = 226 RM_PREFIX + "webapp.address"; 227 228 public static final int DEFAULT_RM_WEBAPP_PORT = 8088; 229 public static final String DEFAULT_RM_WEBAPP_ADDRESS = "0.0.0.0:" + 230 DEFAULT_RM_WEBAPP_PORT; 231 232 /** The https address of the RM web application.*/ 233 public static final String RM_WEBAPP_HTTPS_ADDRESS = 234 RM_PREFIX + "webapp.https.address"; 235 public static final boolean YARN_SSL_CLIENT_HTTPS_NEED_AUTH_DEFAULT = false; 236 public static final String YARN_SSL_SERVER_RESOURCE_DEFAULT = "ssl-server.xml"; 237 238 public static final int DEFAULT_RM_WEBAPP_HTTPS_PORT = 8090; 239 public static final String DEFAULT_RM_WEBAPP_HTTPS_ADDRESS = "0.0.0.0:" 240 + DEFAULT_RM_WEBAPP_HTTPS_PORT; 241 242 public static final String RM_RESOURCE_TRACKER_ADDRESS = 243 RM_PREFIX + "resource-tracker.address"; 244 public static final int DEFAULT_RM_RESOURCE_TRACKER_PORT = 8031; 245 public static final String DEFAULT_RM_RESOURCE_TRACKER_ADDRESS = 246 "0.0.0.0:" + DEFAULT_RM_RESOURCE_TRACKER_PORT; 247 248 /** The expiry interval for application master reporting.*/ 249 public static final String RM_AM_EXPIRY_INTERVAL_MS = 250 YARN_PREFIX + "am.liveness-monitor.expiry-interval-ms"; 251 public static final int DEFAULT_RM_AM_EXPIRY_INTERVAL_MS = 600000; 252 253 /** How long to wait until a node manager is considered dead.*/ 254 public static final String RM_NM_EXPIRY_INTERVAL_MS = 255 YARN_PREFIX + "nm.liveness-monitor.expiry-interval-ms"; 256 public static final int DEFAULT_RM_NM_EXPIRY_INTERVAL_MS = 600000; 257 258 /** Are acls enabled.*/ 259 public static final String YARN_ACL_ENABLE = 260 YARN_PREFIX + "acl.enable"; 261 public static final boolean DEFAULT_YARN_ACL_ENABLE = false; 262 263 /** ACL of who can be admin of YARN cluster.*/ 264 public static final String YARN_ADMIN_ACL = 265 YARN_PREFIX + "admin.acl"; 266 public static final String DEFAULT_YARN_ADMIN_ACL = "*"; 267 268 /** ACL used in case none is found. Allows nothing. */ 269 public static final String DEFAULT_YARN_APP_ACL = " "; 270 271 /** The address of the RM admin interface.*/ 272 public static final String RM_ADMIN_ADDRESS = 273 RM_PREFIX + "admin.address"; 274 public static final int DEFAULT_RM_ADMIN_PORT = 8033; 275 public static final String DEFAULT_RM_ADMIN_ADDRESS = "0.0.0.0:" + 276 DEFAULT_RM_ADMIN_PORT; 277 278 /**Number of threads used to handle RM admin interface.*/ 279 public static final String RM_ADMIN_CLIENT_THREAD_COUNT = 280 RM_PREFIX + "admin.client.thread-count"; 281 public static final int DEFAULT_RM_ADMIN_CLIENT_THREAD_COUNT = 1; 282 283 /** 284 * The maximum number of application attempts. 285 * It's a global setting for all application masters. 286 */ 287 public static final String RM_AM_MAX_ATTEMPTS = 288 RM_PREFIX + "am.max-attempts"; 289 public static final int DEFAULT_RM_AM_MAX_ATTEMPTS = 2; 290 291 /** The keytab for the resource manager.*/ 292 public static final String RM_KEYTAB = 293 RM_PREFIX + "keytab"; 294 295 /**The kerberos principal to be used for spnego filter for RM.*/ 296 public static final String RM_WEBAPP_SPNEGO_USER_NAME_KEY = 297 RM_PREFIX + "webapp.spnego-principal"; 298 299 /**The kerberos keytab to be used for spnego filter for RM.*/ 300 public static final String RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY = 301 RM_PREFIX + "webapp.spnego-keytab-file"; 302 303 /** 304 * Flag to enable override of the default kerberos authentication filter with 305 * the RM authentication filter to allow authentication using delegation 306 * tokens(fallback to kerberos if the tokens are missing). Only applicable 307 * when the http authentication type is kerberos. 308 */ 309 public static final String RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER = RM_PREFIX 310 + "webapp.delegation-token-auth-filter.enabled"; 311 public static final boolean DEFAULT_RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER = 312 true; 313 314 /** How long to wait until a container is considered dead.*/ 315 public static final String RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS = 316 RM_PREFIX + "rm.container-allocation.expiry-interval-ms"; 317 public static final int DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS = 600000; 318 319 /** Path to file with nodes to include.*/ 320 public static final String RM_NODES_INCLUDE_FILE_PATH = 321 RM_PREFIX + "nodes.include-path"; 322 public static final String DEFAULT_RM_NODES_INCLUDE_FILE_PATH = ""; 323 324 /** Path to file with nodes to exclude.*/ 325 public static final String RM_NODES_EXCLUDE_FILE_PATH = 326 RM_PREFIX + "nodes.exclude-path"; 327 public static final String DEFAULT_RM_NODES_EXCLUDE_FILE_PATH = ""; 328 329 /** Number of threads to handle resource tracker calls.*/ 330 public static final String RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT = 331 RM_PREFIX + "resource-tracker.client.thread-count"; 332 public static final int DEFAULT_RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT = 50; 333 334 /** The class to use as the resource scheduler.*/ 335 public static final String RM_SCHEDULER = 336 RM_PREFIX + "scheduler.class"; 337 338 public static final String DEFAULT_RM_SCHEDULER = 339 "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler"; 340 341 /** RM set next Heartbeat interval for NM */ 342 public static final String RM_NM_HEARTBEAT_INTERVAL_MS = 343 RM_PREFIX + "nodemanagers.heartbeat-interval-ms"; 344 public static final long DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS = 1000; 345 346 /** Number of worker threads that write the history data. */ 347 public static final String RM_HISTORY_WRITER_MULTI_THREADED_DISPATCHER_POOL_SIZE = 348 RM_PREFIX + "history-writer.multi-threaded-dispatcher.pool-size"; 349 public static final int DEFAULT_RM_HISTORY_WRITER_MULTI_THREADED_DISPATCHER_POOL_SIZE = 350 10; 351 352 /** 353 * The setting that controls whether yarn system metrics is published on the 354 * timeline server or not by RM. 355 */ 356 public static final String RM_SYSTEM_METRICS_PUBLISHER_ENABLED = 357 RM_PREFIX + "system-metrics-publisher.enabled"; 358 public static final boolean DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_ENABLED = false; 359 360 public static final String RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE = 361 RM_PREFIX + "system-metrics-publisher.dispatcher.pool-size"; 362 public static final int DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE = 363 10; 364 365 //RM delegation token related keys 366 public static final String RM_DELEGATION_KEY_UPDATE_INTERVAL_KEY = 367 RM_PREFIX + "delegation.key.update-interval"; 368 public static final long RM_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT = 369 24*60*60*1000; // 1 day 370 public static final String RM_DELEGATION_TOKEN_RENEW_INTERVAL_KEY = 371 RM_PREFIX + "delegation.token.renew-interval"; 372 public static final long RM_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT = 373 24*60*60*1000; // 1 day 374 public static final String RM_DELEGATION_TOKEN_MAX_LIFETIME_KEY = 375 RM_PREFIX + "delegation.token.max-lifetime"; 376 public static final long RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = 377 7*24*60*60*1000; // 7 days 378 379 public static final String RECOVERY_ENABLED = RM_PREFIX + "recovery.enabled"; 380 public static final boolean DEFAULT_RM_RECOVERY_ENABLED = false; 381 382 @Private 383 public static final String RM_WORK_PRESERVING_RECOVERY_ENABLED = RM_PREFIX 384 + "work-preserving-recovery.enabled"; 385 @Private 386 public static final boolean DEFAULT_RM_WORK_PRESERVING_RECOVERY_ENABLED = 387 true; 388 389 public static final String RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS = 390 RM_PREFIX + "work-preserving-recovery.scheduling-wait-ms"; 391 public static final long DEFAULT_RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS = 392 10000; 393 394 /** Zookeeper interaction configs */ 395 public static final String RM_ZK_PREFIX = RM_PREFIX + "zk-"; 396 397 public static final String RM_ZK_ADDRESS = RM_ZK_PREFIX + "address"; 398 399 public static final String RM_ZK_NUM_RETRIES = RM_ZK_PREFIX + "num-retries"; 400 public static final int DEFAULT_ZK_RM_NUM_RETRIES = 1000; 401 402 public static final String RM_ZK_RETRY_INTERVAL_MS = 403 RM_ZK_PREFIX + "retry-interval-ms"; 404 public static final long DEFAULT_RM_ZK_RETRY_INTERVAL_MS = 1000; 405 406 public static final String RM_ZK_TIMEOUT_MS = RM_ZK_PREFIX + "timeout-ms"; 407 public static final int DEFAULT_RM_ZK_TIMEOUT_MS = 10000; 408 409 public static final String RM_ZK_ACL = RM_ZK_PREFIX + "acl"; 410 public static final String DEFAULT_RM_ZK_ACL = "world:anyone:rwcda"; 411 412 public static final String RM_ZK_AUTH = RM_ZK_PREFIX + "auth"; 413 414 public static final String ZK_STATE_STORE_PREFIX = 415 RM_PREFIX + "zk-state-store."; 416 417 /** Parent znode path under which ZKRMStateStore will create znodes */ 418 public static final String ZK_RM_STATE_STORE_PARENT_PATH = 419 ZK_STATE_STORE_PREFIX + "parent-path"; 420 public static final String DEFAULT_ZK_RM_STATE_STORE_PARENT_PATH = "/rmstore"; 421 422 /** Root node ACLs for fencing */ 423 public static final String ZK_RM_STATE_STORE_ROOT_NODE_ACL = 424 ZK_STATE_STORE_PREFIX + "root-node.acl"; 425 426 /** HA related configs */ 427 public static final String RM_HA_PREFIX = RM_PREFIX + "ha."; 428 public static final String RM_HA_ENABLED = RM_HA_PREFIX + "enabled"; 429 public static final boolean DEFAULT_RM_HA_ENABLED = false; 430 431 public static final String RM_HA_IDS = RM_HA_PREFIX + "rm-ids"; 432 public static final String RM_HA_ID = RM_HA_PREFIX + "id"; 433 434 /** Store the related configuration files in File System */ 435 public static final String FS_BASED_RM_CONF_STORE = RM_PREFIX 436 + "configuration.file-system-based-store"; 437 public static final String DEFAULT_FS_BASED_RM_CONF_STORE = "/yarn/conf"; 438 439 public static final String RM_CONFIGURATION_PROVIDER_CLASS = RM_PREFIX 440 + "configuration.provider-class"; 441 public static final String DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS = 442 "org.apache.hadoop.yarn.LocalConfigurationProvider"; 443 444 public static final String YARN_AUTHORIZATION_PROVIDER = YARN_PREFIX 445 + "authorization-provider"; 446 private static final List<String> RM_SERVICES_ADDRESS_CONF_KEYS_HTTP = 447 Collections.unmodifiableList(Arrays.asList( 448 RM_ADDRESS, 449 RM_SCHEDULER_ADDRESS, 450 RM_ADMIN_ADDRESS, 451 RM_RESOURCE_TRACKER_ADDRESS, 452 RM_WEBAPP_ADDRESS)); 453 454 private static final List<String> RM_SERVICES_ADDRESS_CONF_KEYS_HTTPS = 455 Collections.unmodifiableList(Arrays.asList( 456 RM_ADDRESS, 457 RM_SCHEDULER_ADDRESS, 458 RM_ADMIN_ADDRESS, 459 RM_RESOURCE_TRACKER_ADDRESS, 460 RM_WEBAPP_HTTPS_ADDRESS)); 461 462 public static final String AUTO_FAILOVER_PREFIX = 463 RM_HA_PREFIX + "automatic-failover."; 464 465 public static final String AUTO_FAILOVER_ENABLED = 466 AUTO_FAILOVER_PREFIX + "enabled"; 467 public static final boolean DEFAULT_AUTO_FAILOVER_ENABLED = true; 468 469 public static final String AUTO_FAILOVER_EMBEDDED = 470 AUTO_FAILOVER_PREFIX + "embedded"; 471 public static final boolean DEFAULT_AUTO_FAILOVER_EMBEDDED = true; 472 473 public static final String AUTO_FAILOVER_ZK_BASE_PATH = 474 AUTO_FAILOVER_PREFIX + "zk-base-path"; 475 public static final String DEFAULT_AUTO_FAILOVER_ZK_BASE_PATH = 476 "/yarn-leader-election"; 477 478 public static final String CLIENT_FAILOVER_PREFIX = 479 YARN_PREFIX + "client.failover-"; 480 public static final String CLIENT_FAILOVER_PROXY_PROVIDER = 481 CLIENT_FAILOVER_PREFIX + "proxy-provider"; 482 public static final String DEFAULT_CLIENT_FAILOVER_PROXY_PROVIDER = 483 "org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider"; 484 485 public static final String CLIENT_FAILOVER_MAX_ATTEMPTS = 486 CLIENT_FAILOVER_PREFIX + "max-attempts"; 487 488 public static final String CLIENT_FAILOVER_SLEEPTIME_BASE_MS = 489 CLIENT_FAILOVER_PREFIX + "sleep-base-ms"; 490 491 public static final String CLIENT_FAILOVER_SLEEPTIME_MAX_MS = 492 CLIENT_FAILOVER_PREFIX + "sleep-max-ms"; 493 494 public static final String CLIENT_FAILOVER_RETRIES = 495 CLIENT_FAILOVER_PREFIX + "retries"; 496 public static final int DEFAULT_CLIENT_FAILOVER_RETRIES = 0; 497 498 public static final String CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS = 499 CLIENT_FAILOVER_PREFIX + "retries-on-socket-timeouts"; 500 public static final int 501 DEFAULT_CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS = 0; 502 503 //////////////////////////////// 504 // RM state store configs 505 //////////////////////////////// 506 /** The class to use as the persistent store.*/ 507 public static final String RM_STORE = RM_PREFIX + "store.class"; 508 509 /** URI for FileSystemRMStateStore */ 510 public static final String FS_RM_STATE_STORE_URI = RM_PREFIX 511 + "fs.state-store.uri"; 512 public static final String FS_RM_STATE_STORE_RETRY_POLICY_SPEC = RM_PREFIX 513 + "fs.state-store.retry-policy-spec"; 514 public static final String DEFAULT_FS_RM_STATE_STORE_RETRY_POLICY_SPEC = 515 "2000, 500"; 516 517 public static final String FS_RM_STATE_STORE_NUM_RETRIES = 518 RM_PREFIX + "fs.state-store.num-retries"; 519 public static final int DEFAULT_FS_RM_STATE_STORE_NUM_RETRIES = 0; 520 521 public static final String FS_RM_STATE_STORE_RETRY_INTERVAL_MS = 522 RM_PREFIX + "fs.state-store.retry-interval-ms"; 523 public static final long DEFAULT_FS_RM_STATE_STORE_RETRY_INTERVAL_MS = 524 1000L; 525 526 public static final String RM_LEVELDB_STORE_PATH = RM_PREFIX 527 + "leveldb-state-store.path"; 528 529 /** The maximum number of completed applications RM keeps. */ 530 public static final String RM_MAX_COMPLETED_APPLICATIONS = 531 RM_PREFIX + "max-completed-applications"; 532 public static final int DEFAULT_RM_MAX_COMPLETED_APPLICATIONS = 10000; 533 534 /** 535 * The maximum number of completed applications RM state store keeps, by 536 * default equals to DEFAULT_RM_MAX_COMPLETED_APPLICATIONS 537 */ 538 public static final String RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS = 539 RM_PREFIX + "state-store.max-completed-applications"; 540 public static final int DEFAULT_RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS = 541 DEFAULT_RM_MAX_COMPLETED_APPLICATIONS; 542 543 /** Default application name */ 544 public static final String DEFAULT_APPLICATION_NAME = "N/A"; 545 546 /** Default application type */ 547 public static final String DEFAULT_APPLICATION_TYPE = "YARN"; 548 549 /** Default application type length */ 550 public static final int APPLICATION_TYPE_LENGTH = 20; 551 552 /** Default queue name */ 553 public static final String DEFAULT_QUEUE_NAME = "default"; 554 555 /** 556 * Buckets (in minutes) for the number of apps running in each queue. 557 */ 558 public static final String RM_METRICS_RUNTIME_BUCKETS = 559 RM_PREFIX + "metrics.runtime.buckets"; 560 561 /** 562 * Default sizes of the runtime metric buckets in minutes. 563 */ 564 public static final String DEFAULT_RM_METRICS_RUNTIME_BUCKETS = 565 "60,300,1440"; 566 567 public static final String RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = RM_PREFIX 568 + "am-rm-tokens.master-key-rolling-interval-secs"; 569 570 public static final long DEFAULT_RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = 571 24 * 60 * 60; 572 573 public static final String RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = 574 RM_PREFIX + "container-tokens.master-key-rolling-interval-secs"; 575 576 public static final long DEFAULT_RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = 577 24 * 60 * 60; 578 579 public static final String RM_NMTOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = 580 RM_PREFIX + "nm-tokens.master-key-rolling-interval-secs"; 581 582 public static final long DEFAULT_RM_NMTOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = 583 24 * 60 * 60; 584 585 public static final String RM_NODEMANAGER_MINIMUM_VERSION = 586 RM_PREFIX + "nodemanager.minimum.version"; 587 588 public static final String DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION = 589 "NONE"; 590 591 /** 592 * RM proxy users' prefix 593 */ 594 public static final String RM_PROXY_USER_PREFIX = RM_PREFIX + "proxyuser."; 595 596 //////////////////////////////// 597 // Node Manager Configs 598 //////////////////////////////// 599 600 /** Prefix for all node manager configs.*/ 601 public static final String NM_PREFIX = "yarn.nodemanager."; 602 603 /** Environment variables that will be sent to containers.*/ 604 public static final String NM_ADMIN_USER_ENV = NM_PREFIX + "admin-env"; 605 public static final String DEFAULT_NM_ADMIN_USER_ENV = "MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX"; 606 607 /** Environment variables that containers may override rather than use NodeManager's default.*/ 608 public static final String NM_ENV_WHITELIST = NM_PREFIX + "env-whitelist"; 609 public static final String DEFAULT_NM_ENV_WHITELIST = StringUtils.join(",", 610 Arrays.asList(ApplicationConstants.Environment.JAVA_HOME.key(), 611 ApplicationConstants.Environment.HADOOP_COMMON_HOME.key(), 612 ApplicationConstants.Environment.HADOOP_HDFS_HOME.key(), 613 ApplicationConstants.Environment.HADOOP_CONF_DIR.key(), 614 ApplicationConstants.Environment.HADOOP_YARN_HOME.key())); 615 616 /** address of node manager IPC.*/ 617 public static final String NM_ADDRESS = NM_PREFIX + "address"; 618 public static final int DEFAULT_NM_PORT = 0; 619 public static final String DEFAULT_NM_ADDRESS = "0.0.0.0:" 620 + DEFAULT_NM_PORT; 621 622 /** The actual bind address or the NM.*/ 623 public static final String NM_BIND_HOST = 624 NM_PREFIX + "bind-host"; 625 626 /** who will execute(launch) the containers.*/ 627 public static final String NM_CONTAINER_EXECUTOR = 628 NM_PREFIX + "container-executor.class"; 629 630 /** 631 * Adjustment to make to the container os scheduling priority. 632 * The valid values for this could vary depending on the platform. 633 * On Linux, higher values mean run the containers at a less 634 * favorable priority than the NM. 635 * The value specified is an int. 636 */ 637 public static final String NM_CONTAINER_EXECUTOR_SCHED_PRIORITY = 638 NM_PREFIX + "container-executor.os.sched.priority.adjustment"; 639 public static final int DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY = 0; 640 641 /** Number of threads container manager uses.*/ 642 public static final String NM_CONTAINER_MGR_THREAD_COUNT = 643 NM_PREFIX + "container-manager.thread-count"; 644 public static final int DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT = 20; 645 646 /** Number of threads used in cleanup.*/ 647 public static final String NM_DELETE_THREAD_COUNT = 648 NM_PREFIX + "delete.thread-count"; 649 public static final int DEFAULT_NM_DELETE_THREAD_COUNT = 4; 650 651 /** Keytab for NM.*/ 652 public static final String NM_KEYTAB = NM_PREFIX + "keytab"; 653 654 /**List of directories to store localized files in.*/ 655 public static final String NM_LOCAL_DIRS = NM_PREFIX + "local-dirs"; 656 public static final String DEFAULT_NM_LOCAL_DIRS = "/tmp/nm-local-dir"; 657 658 /** 659 * Number of files in each localized directories 660 * Avoid tuning this too low. 661 */ 662 public static final String NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY = 663 NM_PREFIX + "local-cache.max-files-per-directory"; 664 public static final int DEFAULT_NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY = 8192; 665 666 /** Address where the localizer IPC is.*/ 667 public static final String NM_LOCALIZER_ADDRESS = 668 NM_PREFIX + "localizer.address"; 669 public static final int DEFAULT_NM_LOCALIZER_PORT = 8040; 670 public static final String DEFAULT_NM_LOCALIZER_ADDRESS = "0.0.0.0:" + 671 DEFAULT_NM_LOCALIZER_PORT; 672 673 /** Interval in between cache cleanups.*/ 674 public static final String NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS = 675 NM_PREFIX + "localizer.cache.cleanup.interval-ms"; 676 public static final long DEFAULT_NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS = 677 10 * 60 * 1000; 678 679 /** 680 * Target size of localizer cache in MB, per nodemanager. It is a target 681 * retention size that only includes resources with PUBLIC and PRIVATE 682 * visibility and excludes resources with APPLICATION visibility 683 */ 684 public static final String NM_LOCALIZER_CACHE_TARGET_SIZE_MB = 685 NM_PREFIX + "localizer.cache.target-size-mb"; 686 public static final long DEFAULT_NM_LOCALIZER_CACHE_TARGET_SIZE_MB = 10 * 1024; 687 688 /** Number of threads to handle localization requests.*/ 689 public static final String NM_LOCALIZER_CLIENT_THREAD_COUNT = 690 NM_PREFIX + "localizer.client.thread-count"; 691 public static final int DEFAULT_NM_LOCALIZER_CLIENT_THREAD_COUNT = 5; 692 693 /** Number of threads to use for localization fetching.*/ 694 public static final String NM_LOCALIZER_FETCH_THREAD_COUNT = 695 NM_PREFIX + "localizer.fetch.thread-count"; 696 public static final int DEFAULT_NM_LOCALIZER_FETCH_THREAD_COUNT = 4; 697 698 /** Where to store container logs.*/ 699 public static final String NM_LOG_DIRS = NM_PREFIX + "log-dirs"; 700 public static final String DEFAULT_NM_LOG_DIRS = "/tmp/logs"; 701 702 public static final String NM_RESOURCEMANAGER_MINIMUM_VERSION = 703 NM_PREFIX + "resourcemanager.minimum.version"; 704 public static final String DEFAULT_NM_RESOURCEMANAGER_MINIMUM_VERSION = "NONE"; 705 706 /** Interval at which the delayed token removal thread runs */ 707 public static final String RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS = 708 RM_PREFIX + "delayed.delegation-token.removal-interval-ms"; 709 public static final long DEFAULT_RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS = 710 30000l; 711 712 /** Delegation Token renewer thread count */ 713 public static final String RM_DELEGATION_TOKEN_RENEWER_THREAD_COUNT = 714 RM_PREFIX + "delegation-token-renewer.thread-count"; 715 public static final int DEFAULT_RM_DELEGATION_TOKEN_RENEWER_THREAD_COUNT = 50; 716 717 public static final String RM_PROXY_USER_PRIVILEGES_ENABLED = RM_PREFIX 718 + "proxy-user-privileges.enabled"; 719 public static boolean DEFAULT_RM_PROXY_USER_PRIVILEGES_ENABLED = false; 720 721 /** Whether to enable log aggregation */ 722 public static final String LOG_AGGREGATION_ENABLED = YARN_PREFIX 723 + "log-aggregation-enable"; 724 public static final boolean DEFAULT_LOG_AGGREGATION_ENABLED = false; 725 726 /** 727 * How long to wait before deleting aggregated logs, -1 disables. 728 * Be careful set this too small and you will spam the name node. 729 */ 730 public static final String LOG_AGGREGATION_RETAIN_SECONDS = YARN_PREFIX 731 + "log-aggregation.retain-seconds"; 732 public static final long DEFAULT_LOG_AGGREGATION_RETAIN_SECONDS = -1; 733 734 /** 735 * How long to wait between aggregated log retention checks. If set to 736 * a value {@literal <=} 0 then the value is computed as one-tenth of the 737 * log retention setting. Be careful set this too small and you will spam 738 * the name node. 739 */ 740 public static final String LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS = 741 YARN_PREFIX + "log-aggregation.retain-check-interval-seconds"; 742 public static final long DEFAULT_LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS = -1; 743 744 /** 745 * Number of seconds to retain logs on the NodeManager. Only applicable if Log 746 * aggregation is disabled 747 */ 748 public static final String NM_LOG_RETAIN_SECONDS = NM_PREFIX 749 + "log.retain-seconds"; 750 public static final long DEFAULT_NM_LOG_RETAIN_SECONDS = 3 * 60 * 60; 751 752 /** 753 * Define how often NMs wake up and upload log files 754 */ 755 public static final String NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS = 756 NM_PREFIX + "log-aggregation.roll-monitoring-interval-seconds"; 757 public static final long 758 DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS = -1; 759 /** 760 * Number of threads used in log cleanup. Only applicable if Log aggregation 761 * is disabled 762 */ 763 public static final String NM_LOG_DELETION_THREADS_COUNT = 764 NM_PREFIX + "log.deletion-threads-count"; 765 public static final int DEFAULT_NM_LOG_DELETE_THREAD_COUNT = 4; 766 767 /** Where to aggregate logs to.*/ 768 public static final String NM_REMOTE_APP_LOG_DIR = 769 NM_PREFIX + "remote-app-log-dir"; 770 public static final String DEFAULT_NM_REMOTE_APP_LOG_DIR = "/tmp/logs"; 771 772 /** 773 * The remote log dir will be created at 774 * NM_REMOTE_APP_LOG_DIR/${user}/NM_REMOTE_APP_LOG_DIR_SUFFIX/${appId} 775 */ 776 public static final String NM_REMOTE_APP_LOG_DIR_SUFFIX = 777 NM_PREFIX + "remote-app-log-dir-suffix"; 778 public static final String DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX="logs"; 779 780 public static final String YARN_LOG_SERVER_URL = 781 YARN_PREFIX + "log.server.url"; 782 783 public static final String YARN_TRACKING_URL_GENERATOR = 784 YARN_PREFIX + "tracking.url.generator"; 785 786 /** Amount of memory in GB that can be allocated for containers.*/ 787 public static final String NM_PMEM_MB = NM_PREFIX + "resource.memory-mb"; 788 public static final int DEFAULT_NM_PMEM_MB = 8 * 1024; 789 790 /** Specifies whether physical memory check is enabled. */ 791 public static final String NM_PMEM_CHECK_ENABLED = NM_PREFIX 792 + "pmem-check-enabled"; 793 public static final boolean DEFAULT_NM_PMEM_CHECK_ENABLED = true; 794 795 /** Specifies whether physical memory check is enabled. */ 796 public static final String NM_VMEM_CHECK_ENABLED = NM_PREFIX 797 + "vmem-check-enabled"; 798 public static final boolean DEFAULT_NM_VMEM_CHECK_ENABLED = true; 799 800 /** Conversion ratio for physical memory to virtual memory. */ 801 public static final String NM_VMEM_PMEM_RATIO = 802 NM_PREFIX + "vmem-pmem-ratio"; 803 public static final float DEFAULT_NM_VMEM_PMEM_RATIO = 2.1f; 804 805 /** Number of Virtual CPU Cores which can be allocated for containers.*/ 806 public static final String NM_VCORES = NM_PREFIX + "resource.cpu-vcores"; 807 public static final int DEFAULT_NM_VCORES = 8; 808 809 /** Percentage of overall CPU which can be allocated for containers. */ 810 public static final String NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT = 811 NM_PREFIX + "resource.percentage-physical-cpu-limit"; 812 public static final int DEFAULT_NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT = 813 100; 814 815 /** NM Webapp address.**/ 816 public static final String NM_WEBAPP_ADDRESS = NM_PREFIX + "webapp.address"; 817 public static final int DEFAULT_NM_WEBAPP_PORT = 8042; 818 public static final String DEFAULT_NM_WEBAPP_ADDRESS = "0.0.0.0:" + 819 DEFAULT_NM_WEBAPP_PORT; 820 821 /** NM Webapp https address.**/ 822 public static final String NM_WEBAPP_HTTPS_ADDRESS = NM_PREFIX 823 + "webapp.https.address"; 824 public static final int DEFAULT_NM_WEBAPP_HTTPS_PORT = 8044; 825 public static final String DEFAULT_NM_WEBAPP_HTTPS_ADDRESS = "0.0.0.0:" 826 + DEFAULT_NM_WEBAPP_HTTPS_PORT; 827 828 /** How often to monitor containers.*/ 829 public final static String NM_CONTAINER_MON_INTERVAL_MS = 830 NM_PREFIX + "container-monitor.interval-ms"; 831 public final static int DEFAULT_NM_CONTAINER_MON_INTERVAL_MS = 3000; 832 833 /** Class that calculates containers current resource utilization.*/ 834 public static final String NM_CONTAINER_MON_RESOURCE_CALCULATOR = 835 NM_PREFIX + "container-monitor.resource-calculator.class"; 836 /** Class that calculates process tree resource utilization.*/ 837 public static final String NM_CONTAINER_MON_PROCESS_TREE = 838 NM_PREFIX + "container-monitor.process-tree.class"; 839 public static final String PROCFS_USE_SMAPS_BASED_RSS_ENABLED = NM_PREFIX + 840 "container-monitor.procfs-tree.smaps-based-rss.enabled"; 841 public static final boolean DEFAULT_PROCFS_USE_SMAPS_BASED_RSS_ENABLED = 842 false; 843 844 /** Enable/disable container metrics. */ 845 @Private 846 public static final String NM_CONTAINER_METRICS_ENABLE = 847 NM_PREFIX + "container-metrics.enable"; 848 @Private 849 public static final boolean DEFAULT_NM_CONTAINER_METRICS_ENABLE = true; 850 851 /** Container metrics flush period. -1 for flush on completion. */ 852 @Private 853 public static final String NM_CONTAINER_METRICS_PERIOD_MS = 854 NM_PREFIX + "container-metrics.period-ms"; 855 @Private 856 public static final int DEFAULT_NM_CONTAINER_METRICS_PERIOD_MS = -1; 857 858 /** Prefix for all node manager disk health checker configs. */ 859 private static final String NM_DISK_HEALTH_CHECK_PREFIX = 860 "yarn.nodemanager.disk-health-checker."; 861 /** 862 * Enable/Disable disks' health checker. Default is true. An expert level 863 * configuration property. 864 */ 865 public static final String NM_DISK_HEALTH_CHECK_ENABLE = 866 NM_DISK_HEALTH_CHECK_PREFIX + "enable"; 867 /** Frequency of running disks' health checker. */ 868 public static final String NM_DISK_HEALTH_CHECK_INTERVAL_MS = 869 NM_DISK_HEALTH_CHECK_PREFIX + "interval-ms"; 870 /** By default, disks' health is checked every 2 minutes. */ 871 public static final long DEFAULT_NM_DISK_HEALTH_CHECK_INTERVAL_MS = 872 2 * 60 * 1000; 873 874 /** 875 * The minimum fraction of number of disks to be healthy for the nodemanager 876 * to launch new containers. This applies to nm-local-dirs and nm-log-dirs. 877 */ 878 public static final String NM_MIN_HEALTHY_DISKS_FRACTION = 879 NM_DISK_HEALTH_CHECK_PREFIX + "min-healthy-disks"; 880 /** 881 * By default, at least 25% of disks are to be healthy to say that the node is 882 * healthy in terms of disks. 883 */ 884 public static final float DEFAULT_NM_MIN_HEALTHY_DISKS_FRACTION = 0.25F; 885 886 /** 887 * The maximum percentage of disk space that can be used after which a disk is 888 * marked as offline. Values can range from 0.0 to 100.0. If the value is 889 * greater than or equal to 100, NM will check for full disk. This applies to 890 * nm-local-dirs and nm-log-dirs. 891 */ 892 public static final String NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE = 893 NM_DISK_HEALTH_CHECK_PREFIX + "max-disk-utilization-per-disk-percentage"; 894 /** 895 * By default, 90% of the disk can be used before it is marked as offline. 896 */ 897 public static final float DEFAULT_NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE = 898 90.0F; 899 900 /** 901 * The minimum space that must be available on a local dir for it to be used. 902 * This applies to nm-local-dirs and nm-log-dirs. 903 */ 904 public static final String NM_MIN_PER_DISK_FREE_SPACE_MB = 905 NM_DISK_HEALTH_CHECK_PREFIX + "min-free-space-per-disk-mb"; 906 /** 907 * By default, all of the disk can be used before it is marked as offline. 908 */ 909 public static final long DEFAULT_NM_MIN_PER_DISK_FREE_SPACE_MB = 0; 910 911 /** Frequency of running node health script.*/ 912 public static final String NM_HEALTH_CHECK_INTERVAL_MS = 913 NM_PREFIX + "health-checker.interval-ms"; 914 public static final long DEFAULT_NM_HEALTH_CHECK_INTERVAL_MS = 10 * 60 * 1000; 915 916 /** Health check script time out period.*/ 917 public static final String NM_HEALTH_CHECK_SCRIPT_TIMEOUT_MS = 918 NM_PREFIX + "health-checker.script.timeout-ms"; 919 public static final long DEFAULT_NM_HEALTH_CHECK_SCRIPT_TIMEOUT_MS = 920 2 * DEFAULT_NM_HEALTH_CHECK_INTERVAL_MS; 921 922 /** The health check script to run.*/ 923 public static final String NM_HEALTH_CHECK_SCRIPT_PATH = 924 NM_PREFIX + "health-checker.script.path"; 925 926 /** The arguments to pass to the health check script.*/ 927 public static final String NM_HEALTH_CHECK_SCRIPT_OPTS = 928 NM_PREFIX + "health-checker.script.opts"; 929 930 /** The Docker image name(For DockerContainerExecutor).*/ 931 public static final String NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME = 932 NM_PREFIX + "docker-container-executor.image-name"; 933 934 /** The name of the docker executor (For DockerContainerExecutor).*/ 935 public static final String NM_DOCKER_CONTAINER_EXECUTOR_EXEC_NAME = 936 NM_PREFIX + "docker-container-executor.exec-name"; 937 938 /** The default docker executor (For DockerContainerExecutor).*/ 939 public static final String NM_DEFAULT_DOCKER_CONTAINER_EXECUTOR_EXEC_NAME = 940 "/usr/bin/docker"; 941 942 /** The path to the Linux container executor.*/ 943 public static final String NM_LINUX_CONTAINER_EXECUTOR_PATH = 944 NM_PREFIX + "linux-container-executor.path"; 945 946 /** 947 * The UNIX group that the linux-container-executor should run as. 948 * This is intended to be set as part of container-executor.cfg. 949 */ 950 public static final String NM_LINUX_CONTAINER_GROUP = 951 NM_PREFIX + "linux-container-executor.group"; 952 953 /** 954 * If linux-container-executor should limit itself to one user 955 * when running in non-secure mode. 956 */ 957 public static final String NM_NONSECURE_MODE_LIMIT_USERS= NM_PREFIX + 958 "linux-container-executor.nonsecure-mode.limit-users"; 959 960 public static final boolean DEFAULT_NM_NONSECURE_MODE_LIMIT_USERS = true; 961 962 /** 963 * The UNIX user that containers will run as when Linux-container-executor 964 * is used in nonsecure mode (a use case for this is using cgroups). 965 */ 966 public static final String NM_NONSECURE_MODE_LOCAL_USER_KEY = NM_PREFIX + 967 "linux-container-executor.nonsecure-mode.local-user"; 968 969 public static final String DEFAULT_NM_NONSECURE_MODE_LOCAL_USER = "nobody"; 970 971 /** 972 * The allowed pattern for UNIX user names enforced by 973 * Linux-container-executor when used in nonsecure mode (use case for this 974 * is using cgroups). The default value is taken from /usr/sbin/adduser 975 */ 976 public static final String NM_NONSECURE_MODE_USER_PATTERN_KEY = NM_PREFIX + 977 "linux-container-executor.nonsecure-mode.user-pattern"; 978 979 public static final String DEFAULT_NM_NONSECURE_MODE_USER_PATTERN = 980 "^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$"; 981 982 /** The type of resource enforcement to use with the 983 * linux container executor. 984 */ 985 public static final String NM_LINUX_CONTAINER_RESOURCES_HANDLER = 986 NM_PREFIX + "linux-container-executor.resources-handler.class"; 987 988 /** The path the linux container executor should use for cgroups */ 989 public static final String NM_LINUX_CONTAINER_CGROUPS_HIERARCHY = 990 NM_PREFIX + "linux-container-executor.cgroups.hierarchy"; 991 992 /** Whether the linux container executor should mount cgroups if not found */ 993 public static final String NM_LINUX_CONTAINER_CGROUPS_MOUNT = 994 NM_PREFIX + "linux-container-executor.cgroups.mount"; 995 996 /** Where the linux container executor should mount cgroups if not found */ 997 public static final String NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH = 998 NM_PREFIX + "linux-container-executor.cgroups.mount-path"; 999 1000 /** 1001 * Whether the apps should run in strict resource usage mode(not allowed to 1002 * use spare CPU) 1003 */ 1004 public static final String NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE = 1005 NM_PREFIX + "linux-container-executor.cgroups.strict-resource-usage"; 1006 public static final boolean DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE = 1007 false; 1008 1009 1010 1011 /** 1012 * Interval of time the linux container executor should try cleaning up 1013 * cgroups entry when cleaning up a container. This is required due to what 1014 * it seems a race condition because the SIGTERM/SIGKILL is asynch. 1015 */ 1016 public static final String NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT = 1017 NM_PREFIX + "linux-container-executor.cgroups.delete-timeout-ms"; 1018 1019 public static final long DEFAULT_NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT = 1020 1000; 1021 1022 /** 1023 * Delay between attempts to remove linux cgroup. 1024 */ 1025 public static final String NM_LINUX_CONTAINER_CGROUPS_DELETE_DELAY = 1026 NM_PREFIX + "linux-container-executor.cgroups.delete-delay-ms"; 1027 1028 public static final long DEFAULT_NM_LINUX_CONTAINER_CGROUPS_DELETE_DELAY = 1029 20; 1030 1031 /** 1032 * Indicates if memory and CPU limits will be set for the Windows Job 1033 * Object for the containers launched by the default container executor. 1034 */ 1035 public static final String NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED = 1036 NM_PREFIX + "windows-container.memory-limit.enabled"; 1037 public static final boolean DEFAULT_NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED = false; 1038 1039 public static final String NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED = 1040 NM_PREFIX + "windows-container.cpu-limit.enabled"; 1041 public static final boolean DEFAULT_NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED = false; 1042 1043 /** 1044 /* The Windows group that the windows-secure-container-executor should run as. 1045 */ 1046 public static final String NM_WINDOWS_SECURE_CONTAINER_GROUP = 1047 NM_PREFIX + "windows-secure-container-executor.group"; 1048 1049 /** T-file compression types used to compress aggregated logs.*/ 1050 public static final String NM_LOG_AGG_COMPRESSION_TYPE = 1051 NM_PREFIX + "log-aggregation.compression-type"; 1052 public static final String DEFAULT_NM_LOG_AGG_COMPRESSION_TYPE = "none"; 1053 1054 /** The kerberos principal for the node manager.*/ 1055 public static final String NM_PRINCIPAL = 1056 NM_PREFIX + "principal"; 1057 1058 public static final String NM_AUX_SERVICES = 1059 NM_PREFIX + "aux-services"; 1060 1061 public static final String NM_AUX_SERVICE_FMT = 1062 NM_PREFIX + "aux-services.%s.class"; 1063 1064 public static final String NM_USER_HOME_DIR = 1065 NM_PREFIX + "user-home-dir"; 1066 1067 /**The kerberos principal to be used for spnego filter for NM.*/ 1068 public static final String NM_WEBAPP_SPNEGO_USER_NAME_KEY = 1069 NM_PREFIX + "webapp.spnego-principal"; 1070 1071 /**The kerberos keytab to be used for spnego filter for NM.*/ 1072 public static final String NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY = 1073 NM_PREFIX + "webapp.spnego-keytab-file"; 1074 1075 public static final String DEFAULT_NM_USER_HOME_DIR= "/home/"; 1076 1077 public static final String NM_RECOVERY_PREFIX = NM_PREFIX + "recovery."; 1078 public static final String NM_RECOVERY_ENABLED = 1079 NM_RECOVERY_PREFIX + "enabled"; 1080 public static final boolean DEFAULT_NM_RECOVERY_ENABLED = false; 1081 1082 public static final String NM_RECOVERY_DIR = NM_RECOVERY_PREFIX + "dir"; 1083 1084 //////////////////////////////// 1085 // Web Proxy Configs 1086 //////////////////////////////// 1087 public static final String PROXY_PREFIX = "yarn.web-proxy."; 1088 1089 /** The kerberos principal for the proxy.*/ 1090 public static final String PROXY_PRINCIPAL = 1091 PROXY_PREFIX + "principal"; 1092 1093 /** Keytab for Proxy.*/ 1094 public static final String PROXY_KEYTAB = PROXY_PREFIX + "keytab"; 1095 1096 /** The address for the web proxy.*/ 1097 public static final String PROXY_ADDRESS = 1098 PROXY_PREFIX + "address"; 1099 public static final int DEFAULT_PROXY_PORT = 9099; 1100 public static final String DEFAULT_PROXY_ADDRESS = 1101 "0.0.0.0:" + DEFAULT_PROXY_PORT; 1102 1103 /** 1104 * YARN Service Level Authorization 1105 */ 1106 public static final String 1107 YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCETRACKER_PROTOCOL = 1108 "security.resourcetracker.protocol.acl"; 1109 public static final String 1110 YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONCLIENT_PROTOCOL = 1111 "security.applicationclient.protocol.acl"; 1112 public static final String 1113 YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCEMANAGER_ADMINISTRATION_PROTOCOL = 1114 "security.resourcemanager-administration.protocol.acl"; 1115 public static final String 1116 YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONMASTER_PROTOCOL = 1117 "security.applicationmaster.protocol.acl"; 1118 1119 public static final String 1120 YARN_SECURITY_SERVICE_AUTHORIZATION_CONTAINER_MANAGEMENT_PROTOCOL = 1121 "security.containermanagement.protocol.acl"; 1122 public static final String 1123 YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCE_LOCALIZER = 1124 "security.resourcelocalizer.protocol.acl"; 1125 1126 public static final String 1127 YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONHISTORY_PROTOCOL = 1128 "security.applicationhistory.protocol.acl"; 1129 1130 /** No. of milliseconds to wait between sending a SIGTERM and SIGKILL 1131 * to a running container */ 1132 public static final String NM_SLEEP_DELAY_BEFORE_SIGKILL_MS = 1133 NM_PREFIX + "sleep-delay-before-sigkill.ms"; 1134 public static final long DEFAULT_NM_SLEEP_DELAY_BEFORE_SIGKILL_MS = 1135 250; 1136 1137 /** Max time to wait for a process to come up when trying to cleanup 1138 * container resources */ 1139 public static final String NM_PROCESS_KILL_WAIT_MS = 1140 NM_PREFIX + "process-kill-wait.ms"; 1141 public static final long DEFAULT_NM_PROCESS_KILL_WAIT_MS = 1142 2000; 1143 1144 /** Max time to wait to establish a connection to RM */ 1145 public static final String RESOURCEMANAGER_CONNECT_MAX_WAIT_MS = 1146 RM_PREFIX + "connect.max-wait.ms"; 1147 public static final long DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS = 1148 15 * 60 * 1000; 1149 1150 /** Time interval between each attempt to connect to RM */ 1151 public static final String RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS = 1152 RM_PREFIX + "connect.retry-interval.ms"; 1153 public static final long DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS 1154 = 30 * 1000; 1155 1156 /** 1157 * CLASSPATH for YARN applications. A comma-separated list of CLASSPATH 1158 * entries 1159 */ 1160 public static final String YARN_APPLICATION_CLASSPATH = YARN_PREFIX 1161 + "application.classpath"; 1162 1163 /** 1164 * Default platform-agnostic CLASSPATH for YARN applications. A 1165 * comma-separated list of CLASSPATH entries. The parameter expansion marker 1166 * will be replaced with real parameter expansion marker ('%' for Windows and 1167 * '$' for Linux) by NodeManager on container launch. For example: {{VAR}} 1168 * will be replaced as $VAR on Linux, and %VAR% on Windows. 1169 */ 1170 @Public 1171 @Unstable 1172 public static final String[] DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH= { 1173 ApplicationConstants.Environment.HADOOP_CONF_DIR.$$(), 1174 ApplicationConstants.Environment.HADOOP_COMMON_HOME.$$() 1175 + "/share/hadoop/common/*", 1176 ApplicationConstants.Environment.HADOOP_COMMON_HOME.$$() 1177 + "/share/hadoop/common/lib/*", 1178 ApplicationConstants.Environment.HADOOP_HDFS_HOME.$$() 1179 + "/share/hadoop/hdfs/*", 1180 ApplicationConstants.Environment.HADOOP_HDFS_HOME.$$() 1181 + "/share/hadoop/hdfs/lib/*", 1182 ApplicationConstants.Environment.HADOOP_YARN_HOME.$$() 1183 + "/share/hadoop/yarn/*", 1184 ApplicationConstants.Environment.HADOOP_YARN_HOME.$$() 1185 + "/share/hadoop/yarn/lib/*" }; 1186 /** 1187 * <p> 1188 * Default platform-specific CLASSPATH for YARN applications. A 1189 * comma-separated list of CLASSPATH entries constructed based on the client 1190 * OS environment expansion syntax. 1191 * </p> 1192 * <p> 1193 * Note: Use {@link #DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH} for 1194 * cross-platform practice i.e. submit an application from a Windows client to 1195 * a Linux/Unix server or vice versa. 1196 * </p> 1197 */ 1198 public static final String[] DEFAULT_YARN_APPLICATION_CLASSPATH = { 1199 ApplicationConstants.Environment.HADOOP_CONF_DIR.$(), 1200 ApplicationConstants.Environment.HADOOP_COMMON_HOME.$() 1201 + "/share/hadoop/common/*", 1202 ApplicationConstants.Environment.HADOOP_COMMON_HOME.$() 1203 + "/share/hadoop/common/lib/*", 1204 ApplicationConstants.Environment.HADOOP_HDFS_HOME.$() 1205 + "/share/hadoop/hdfs/*", 1206 ApplicationConstants.Environment.HADOOP_HDFS_HOME.$() 1207 + "/share/hadoop/hdfs/lib/*", 1208 ApplicationConstants.Environment.HADOOP_YARN_HOME.$() 1209 + "/share/hadoop/yarn/*", 1210 ApplicationConstants.Environment.HADOOP_YARN_HOME.$() 1211 + "/share/hadoop/yarn/lib/*" }; 1212 1213 /** Container temp directory */ 1214 public static final String DEFAULT_CONTAINER_TEMP_DIR = "./tmp"; 1215 1216 public static final String IS_MINI_YARN_CLUSTER = YARN_PREFIX 1217 + "is.minicluster"; 1218 1219 public static final String YARN_MC_PREFIX = YARN_PREFIX + "minicluster."; 1220 1221 /** Whether to use fixed ports with the minicluster. */ 1222 public static final String YARN_MINICLUSTER_FIXED_PORTS = 1223 YARN_MC_PREFIX + "fixed.ports"; 1224 1225 /** 1226 * Default is false to be able to run tests concurrently without port 1227 * conflicts. 1228 */ 1229 public static final boolean DEFAULT_YARN_MINICLUSTER_FIXED_PORTS = false; 1230 1231 /** 1232 * Whether the NM should use RPC to connect to the RM. Default is false. 1233 * Can be set to true only when using fixed ports. 1234 */ 1235 public static final String YARN_MINICLUSTER_USE_RPC = YARN_MC_PREFIX + "use-rpc"; 1236 public static final boolean DEFAULT_YARN_MINICLUSTER_USE_RPC = false; 1237 1238 /** 1239 * Whether users are explicitly trying to control resource monitoring 1240 * configuration for the MiniYARNCluster. Disabled by default. 1241 */ 1242 public static final String YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING = 1243 YARN_MC_PREFIX + "control-resource-monitoring"; 1244 public static final boolean 1245 DEFAULT_YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING = false; 1246 1247 /** Allow changing the memory for the NodeManager in the MiniYARNCluster */ 1248 public static final String YARN_MINICLUSTER_NM_PMEM_MB = 1249 YARN_MC_PREFIX + YarnConfiguration.NM_PMEM_MB; 1250 public static final int DEFAULT_YARN_MINICLUSTER_NM_PMEM_MB = 4 * 1024; 1251 1252 /** The log directory for the containers */ 1253 public static final String YARN_APP_CONTAINER_LOG_DIR = 1254 YARN_PREFIX + "app.container.log.dir"; 1255 1256 public static final String YARN_APP_CONTAINER_LOG_SIZE = 1257 YARN_PREFIX + "app.container.log.filesize"; 1258 1259 public static final String YARN_APP_CONTAINER_LOG_BACKUPS = 1260 YARN_PREFIX + "app.container.log.backups"; 1261 1262 //////////////////////////////// 1263 // Timeline Service Configs 1264 //////////////////////////////// 1265 1266 public static final String TIMELINE_SERVICE_PREFIX = 1267 YARN_PREFIX + "timeline-service."; 1268 1269 1270 // mark app-history related configs @Private as application history is going 1271 // to be integrated into the timeline service 1272 @Private 1273 public static final String APPLICATION_HISTORY_PREFIX = 1274 TIMELINE_SERVICE_PREFIX + "generic-application-history."; 1275 1276 /** 1277 * The setting that controls whether application history service is 1278 * enabled or not. 1279 */ 1280 @Private 1281 public static final String APPLICATION_HISTORY_ENABLED = 1282 APPLICATION_HISTORY_PREFIX + "enabled"; 1283 @Private 1284 public static final boolean DEFAULT_APPLICATION_HISTORY_ENABLED = false; 1285 1286 /** Application history store class */ 1287 @Private 1288 public static final String APPLICATION_HISTORY_STORE = 1289 APPLICATION_HISTORY_PREFIX + "store-class"; 1290 1291 /** URI for FileSystemApplicationHistoryStore */ 1292 @Private 1293 public static final String FS_APPLICATION_HISTORY_STORE_URI = 1294 APPLICATION_HISTORY_PREFIX + "fs-history-store.uri"; 1295 1296 /** T-file compression types used to compress history data.*/ 1297 @Private 1298 public static final String FS_APPLICATION_HISTORY_STORE_COMPRESSION_TYPE = 1299 APPLICATION_HISTORY_PREFIX + "fs-history-store.compression-type"; 1300 @Private 1301 public static final String DEFAULT_FS_APPLICATION_HISTORY_STORE_COMPRESSION_TYPE = 1302 "none"; 1303 1304 /** The setting that controls whether timeline service is enabled or not. */ 1305 public static final String TIMELINE_SERVICE_ENABLED = 1306 TIMELINE_SERVICE_PREFIX + "enabled"; 1307 public static final boolean DEFAULT_TIMELINE_SERVICE_ENABLED = false; 1308 1309 /** host:port address for timeline service RPC APIs. */ 1310 public static final String TIMELINE_SERVICE_ADDRESS = 1311 TIMELINE_SERVICE_PREFIX + "address"; 1312 public static final int DEFAULT_TIMELINE_SERVICE_PORT = 10200; 1313 public static final String DEFAULT_TIMELINE_SERVICE_ADDRESS = "0.0.0.0:" 1314 + DEFAULT_TIMELINE_SERVICE_PORT; 1315 1316 /** The listening endpoint for the timeline service application.*/ 1317 public static final String TIMELINE_SERVICE_BIND_HOST = 1318 TIMELINE_SERVICE_PREFIX + "bind-host"; 1319 1320 /** The number of threads to handle client RPC API requests. */ 1321 public static final String TIMELINE_SERVICE_HANDLER_THREAD_COUNT = 1322 TIMELINE_SERVICE_PREFIX + "handler-thread-count"; 1323 public static final int DEFAULT_TIMELINE_SERVICE_CLIENT_THREAD_COUNT = 10; 1324 1325 1326 /** The address of the timeline service web application.*/ 1327 public static final String TIMELINE_SERVICE_WEBAPP_ADDRESS = 1328 TIMELINE_SERVICE_PREFIX + "webapp.address"; 1329 1330 public static final int DEFAULT_TIMELINE_SERVICE_WEBAPP_PORT = 8188; 1331 public static final String DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS = 1332 "0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_WEBAPP_PORT; 1333 1334 /** The https address of the timeline service web application.*/ 1335 public static final String TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS = 1336 TIMELINE_SERVICE_PREFIX + "webapp.https.address"; 1337 1338 public static final int DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT = 8190; 1339 public static final String DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS = 1340 "0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT; 1341 1342 /** Timeline service store class */ 1343 public static final String TIMELINE_SERVICE_STORE = 1344 TIMELINE_SERVICE_PREFIX + "store-class"; 1345 1346 /** Timeline service enable data age off */ 1347 public static final String TIMELINE_SERVICE_TTL_ENABLE = 1348 TIMELINE_SERVICE_PREFIX + "ttl-enable"; 1349 1350 /** Timeline service length of time to retain data */ 1351 public static final String TIMELINE_SERVICE_TTL_MS = 1352 TIMELINE_SERVICE_PREFIX + "ttl-ms"; 1353 1354 public static final long DEFAULT_TIMELINE_SERVICE_TTL_MS = 1355 1000 * 60 * 60 * 24 * 7; 1356 1357 public static final String TIMELINE_SERVICE_LEVELDB_PREFIX = 1358 TIMELINE_SERVICE_PREFIX + "leveldb-timeline-store."; 1359 1360 /** Timeline service leveldb path */ 1361 public static final String TIMELINE_SERVICE_LEVELDB_PATH = 1362 TIMELINE_SERVICE_LEVELDB_PREFIX + "path"; 1363 1364 /** Timeline service leveldb read cache (uncompressed blocks) */ 1365 public static final String TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE = 1366 TIMELINE_SERVICE_LEVELDB_PREFIX + "read-cache-size"; 1367 1368 public static final long DEFAULT_TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE = 1369 100 * 1024 * 1024; 1370 1371 /** Timeline service leveldb start time read cache (number of entities) */ 1372 public static final String 1373 TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE = 1374 TIMELINE_SERVICE_LEVELDB_PREFIX + "start-time-read-cache-size"; 1375 1376 public static final int 1377 DEFAULT_TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE = 10000; 1378 1379 /** Timeline service leveldb start time write cache (number of entities) */ 1380 public static final String 1381 TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE = 1382 TIMELINE_SERVICE_LEVELDB_PREFIX + "start-time-write-cache-size"; 1383 1384 public static final int 1385 DEFAULT_TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE = 10000; 1386 1387 /** Timeline service leveldb interval to wait between deletion rounds */ 1388 public static final String TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS = 1389 TIMELINE_SERVICE_LEVELDB_PREFIX + "ttl-interval-ms"; 1390 1391 public static final long DEFAULT_TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS = 1392 1000 * 60 * 5; 1393 1394 /** The Kerberos principal for the timeline server.*/ 1395 public static final String TIMELINE_SERVICE_PRINCIPAL = 1396 TIMELINE_SERVICE_PREFIX + "principal"; 1397 1398 /** The Kerberos keytab for the timeline server.*/ 1399 public static final String TIMELINE_SERVICE_KEYTAB = 1400 TIMELINE_SERVICE_PREFIX + "keytab"; 1401 1402 /** Enables cross origin support for timeline server.*/ 1403 public static final String TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED = 1404 TIMELINE_SERVICE_PREFIX + "http-cross-origin.enabled"; 1405 1406 /** Default value for cross origin support for timeline server.*/ 1407 public static final boolean 1408 TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED_DEFAULT = false; 1409 1410 /** Timeline client settings */ 1411 public static final String TIMELINE_SERVICE_CLIENT_PREFIX = 1412 TIMELINE_SERVICE_PREFIX + "client."; 1413 1414 /** Timeline client call, max retries (-1 means no limit) */ 1415 public static final String TIMELINE_SERVICE_CLIENT_MAX_RETRIES = 1416 TIMELINE_SERVICE_CLIENT_PREFIX + "max-retries"; 1417 1418 public static final int DEFAULT_TIMELINE_SERVICE_CLIENT_MAX_RETRIES = 30; 1419 1420 /** Timeline client call, retry interval */ 1421 public static final String TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS = 1422 TIMELINE_SERVICE_CLIENT_PREFIX + "retry-interval-ms"; 1423 1424 public static final long 1425 DEFAULT_TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS = 1000; 1426 1427 /** Timeline client policy for whether connections are fatal */ 1428 public static final String TIMELINE_SERVICE_CLIENT_BEST_EFFORT = 1429 TIMELINE_SERVICE_CLIENT_PREFIX + "best-effort"; 1430 1431 public static final boolean 1432 DEFAULT_TIMELINE_SERVICE_CLIENT_BEST_EFFORT = false; 1433 1434 /** Flag to enable recovery of timeline service */ 1435 public static final String TIMELINE_SERVICE_RECOVERY_ENABLED = 1436 TIMELINE_SERVICE_PREFIX + "recovery.enabled"; 1437 public static final boolean DEFAULT_TIMELINE_SERVICE_RECOVERY_ENABLED = false; 1438 1439 /** Timeline service state store class */ 1440 public static final String TIMELINE_SERVICE_STATE_STORE_CLASS = 1441 TIMELINE_SERVICE_PREFIX + "state-store-class"; 1442 1443 public static final String TIMELINE_SERVICE_LEVELDB_STATE_STORE_PREFIX = 1444 TIMELINE_SERVICE_PREFIX + "leveldb-state-store."; 1445 1446 /** Timeline service state store leveldb path */ 1447 public static final String TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH = 1448 TIMELINE_SERVICE_LEVELDB_STATE_STORE_PREFIX + "path"; 1449 1450 // Timeline delegation token related keys 1451 public static final String TIMELINE_DELEGATION_KEY_UPDATE_INTERVAL = 1452 TIMELINE_SERVICE_PREFIX + "delegation.key.update-interval"; 1453 public static final long DEFAULT_TIMELINE_DELEGATION_KEY_UPDATE_INTERVAL = 1454 24*60*60*1000; // 1 day 1455 public static final String TIMELINE_DELEGATION_TOKEN_RENEW_INTERVAL = 1456 TIMELINE_SERVICE_PREFIX + "delegation.token.renew-interval"; 1457 public static final long DEFAULT_TIMELINE_DELEGATION_TOKEN_RENEW_INTERVAL = 1458 24*60*60*1000; // 1 day 1459 public static final String TIMELINE_DELEGATION_TOKEN_MAX_LIFETIME = 1460 TIMELINE_SERVICE_PREFIX + "delegation.token.max-lifetime"; 1461 public static final long DEFAULT_TIMELINE_DELEGATION_TOKEN_MAX_LIFETIME = 1462 7*24*60*60*1000; // 7 days 1463 1464 // /////////////////////////////// 1465 // Shared Cache Configs 1466 // /////////////////////////////// 1467 public static final String SHARED_CACHE_PREFIX = "yarn.sharedcache."; 1468 1469 // common configs 1470 /** whether the shared cache is enabled/disabled */ 1471 public static final String SHARED_CACHE_ENABLED = 1472 SHARED_CACHE_PREFIX + "enabled"; 1473 public static final boolean DEFAULT_SHARED_CACHE_ENABLED = false; 1474 1475 /** The config key for the shared cache root directory. */ 1476 public static final String SHARED_CACHE_ROOT = 1477 SHARED_CACHE_PREFIX + "root-dir"; 1478 public static final String DEFAULT_SHARED_CACHE_ROOT = "/sharedcache"; 1479 1480 /** The config key for the level of nested directories before getting to the 1481 * checksum directory. */ 1482 public static final String SHARED_CACHE_NESTED_LEVEL = 1483 SHARED_CACHE_PREFIX + "nested-level"; 1484 public static final int DEFAULT_SHARED_CACHE_NESTED_LEVEL = 3; 1485 1486 // Shared Cache Manager Configs 1487 1488 public static final String SCM_STORE_PREFIX = SHARED_CACHE_PREFIX + "store."; 1489 1490 public static final String SCM_STORE_CLASS = SCM_STORE_PREFIX + "class"; 1491 public static final String DEFAULT_SCM_STORE_CLASS = 1492 "org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore"; 1493 1494 public static final String SCM_APP_CHECKER_CLASS = SHARED_CACHE_PREFIX 1495 + "app-checker.class"; 1496 public static final String DEFAULT_SCM_APP_CHECKER_CLASS = 1497 "org.apache.hadoop.yarn.server.sharedcachemanager.RemoteAppChecker"; 1498 1499 /** The address of the SCM admin interface. */ 1500 public static final String SCM_ADMIN_ADDRESS = 1501 SHARED_CACHE_PREFIX + "admin.address"; 1502 public static final int DEFAULT_SCM_ADMIN_PORT = 8047; 1503 public static final String DEFAULT_SCM_ADMIN_ADDRESS = 1504 "0.0.0.0:" + DEFAULT_SCM_ADMIN_PORT; 1505 1506 /** Number of threads used to handle SCM admin interface. */ 1507 public static final String SCM_ADMIN_CLIENT_THREAD_COUNT = 1508 SHARED_CACHE_PREFIX + "admin.thread-count"; 1509 public static final int DEFAULT_SCM_ADMIN_CLIENT_THREAD_COUNT = 1; 1510 1511 /** The address of the SCM web application. */ 1512 public static final String SCM_WEBAPP_ADDRESS = 1513 SHARED_CACHE_PREFIX + "webapp.address"; 1514 public static final int DEFAULT_SCM_WEBAPP_PORT = 8788; 1515 public static final String DEFAULT_SCM_WEBAPP_ADDRESS = 1516 "0.0.0.0:" + DEFAULT_SCM_WEBAPP_PORT; 1517 1518 // In-memory SCM store configuration 1519 1520 public static final String IN_MEMORY_STORE_PREFIX = 1521 SCM_STORE_PREFIX + "in-memory."; 1522 1523 /** 1524 * A resource in the InMemorySCMStore is considered stale if the time since 1525 * the last reference exceeds the staleness period. This value is specified in 1526 * minutes. 1527 */ 1528 public static final String IN_MEMORY_STALENESS_PERIOD_MINS = 1529 IN_MEMORY_STORE_PREFIX + "staleness-period-mins"; 1530 public static final int DEFAULT_IN_MEMORY_STALENESS_PERIOD_MINS = 1531 7 * 24 * 60; 1532 1533 /** 1534 * Initial delay before the in-memory store runs its first check to remove 1535 * dead initial applications. Specified in minutes. 1536 */ 1537 public static final String IN_MEMORY_INITIAL_DELAY_MINS = 1538 IN_MEMORY_STORE_PREFIX + "initial-delay-mins"; 1539 public static final int DEFAULT_IN_MEMORY_INITIAL_DELAY_MINS = 10; 1540 1541 /** 1542 * The frequency at which the in-memory store checks to remove dead initial 1543 * applications. Specified in minutes. 1544 */ 1545 public static final String IN_MEMORY_CHECK_PERIOD_MINS = 1546 IN_MEMORY_STORE_PREFIX + "check-period-mins"; 1547 public static final int DEFAULT_IN_MEMORY_CHECK_PERIOD_MINS = 12 * 60; 1548 1549 // SCM Cleaner service configuration 1550 1551 private static final String SCM_CLEANER_PREFIX = SHARED_CACHE_PREFIX 1552 + "cleaner."; 1553 1554 /** 1555 * The frequency at which a cleaner task runs. Specified in minutes. 1556 */ 1557 public static final String SCM_CLEANER_PERIOD_MINS = 1558 SCM_CLEANER_PREFIX + "period-mins"; 1559 public static final int DEFAULT_SCM_CLEANER_PERIOD_MINS = 24 * 60; 1560 1561 /** 1562 * Initial delay before the first cleaner task is scheduled. Specified in 1563 * minutes. 1564 */ 1565 public static final String SCM_CLEANER_INITIAL_DELAY_MINS = 1566 SCM_CLEANER_PREFIX + "initial-delay-mins"; 1567 public static final int DEFAULT_SCM_CLEANER_INITIAL_DELAY_MINS = 10; 1568 1569 /** 1570 * The time to sleep between processing each shared cache resource. Specified 1571 * in milliseconds. 1572 */ 1573 public static final String SCM_CLEANER_RESOURCE_SLEEP_MS = 1574 SCM_CLEANER_PREFIX + "resource-sleep-ms"; 1575 public static final long DEFAULT_SCM_CLEANER_RESOURCE_SLEEP_MS = 0L; 1576 1577 /** The address of the node manager interface in the SCM. */ 1578 public static final String SCM_UPLOADER_SERVER_ADDRESS = SHARED_CACHE_PREFIX 1579 + "uploader.server.address"; 1580 public static final int DEFAULT_SCM_UPLOADER_SERVER_PORT = 8046; 1581 public static final String DEFAULT_SCM_UPLOADER_SERVER_ADDRESS = "0.0.0.0:" 1582 + DEFAULT_SCM_UPLOADER_SERVER_PORT; 1583 1584 /** 1585 * The number of SCM threads used to handle notify requests from the node 1586 * manager. 1587 */ 1588 public static final String SCM_UPLOADER_SERVER_THREAD_COUNT = 1589 SHARED_CACHE_PREFIX + "uploader.server.thread-count"; 1590 public static final int DEFAULT_SCM_UPLOADER_SERVER_THREAD_COUNT = 50; 1591 1592 /** The address of the client interface in the SCM. */ 1593 public static final String SCM_CLIENT_SERVER_ADDRESS = 1594 SHARED_CACHE_PREFIX + "client-server.address"; 1595 public static final int DEFAULT_SCM_CLIENT_SERVER_PORT = 8045; 1596 public static final String DEFAULT_SCM_CLIENT_SERVER_ADDRESS = "0.0.0.0:" 1597 + DEFAULT_SCM_CLIENT_SERVER_PORT; 1598 1599 /** The number of threads used to handle shared cache manager requests. */ 1600 public static final String SCM_CLIENT_SERVER_THREAD_COUNT = 1601 SHARED_CACHE_PREFIX + "client-server.thread-count"; 1602 public static final int DEFAULT_SCM_CLIENT_SERVER_THREAD_COUNT = 50; 1603 1604 /** the checksum algorithm implementation **/ 1605 public static final String SHARED_CACHE_CHECKSUM_ALGO_IMPL = 1606 SHARED_CACHE_PREFIX + "checksum.algo.impl"; 1607 public static final String DEFAULT_SHARED_CACHE_CHECKSUM_ALGO_IMPL = 1608 "org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl"; 1609 1610 // node manager (uploader) configs 1611 /** 1612 * The replication factor for the node manager uploader for the shared cache. 1613 */ 1614 public static final String SHARED_CACHE_NM_UPLOADER_REPLICATION_FACTOR = 1615 SHARED_CACHE_PREFIX + "nm.uploader.replication.factor"; 1616 public static final int DEFAULT_SHARED_CACHE_NM_UPLOADER_REPLICATION_FACTOR = 1617 10; 1618 1619 public static final String SHARED_CACHE_NM_UPLOADER_THREAD_COUNT = 1620 SHARED_CACHE_PREFIX + "nm.uploader.thread-count"; 1621 public static final int DEFAULT_SHARED_CACHE_NM_UPLOADER_THREAD_COUNT = 20; 1622 1623 //////////////////////////////// 1624 // Other Configs 1625 //////////////////////////////// 1626 1627 /** 1628 * Use YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS instead. 1629 * The interval of the yarn client's querying application state after 1630 * application submission. The unit is millisecond. 1631 */ 1632 @Deprecated 1633 public static final String YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS = 1634 YARN_PREFIX + "client.app-submission.poll-interval"; 1635 1636 /** 1637 * The interval that the yarn client library uses to poll the completion 1638 * status of the asynchronous API of application client protocol. 1639 */ 1640 public static final String YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS = 1641 YARN_PREFIX + "client.application-client-protocol.poll-interval-ms"; 1642 public static final long DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS = 1643 200; 1644 1645 /** 1646 * The duration that the yarn client library waits, cumulatively across polls, 1647 * for an expected state change to occur. Defaults to -1, which indicates no 1648 * limit. 1649 */ 1650 public static final String YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS = 1651 YARN_PREFIX + "client.application-client-protocol.poll-timeout-ms"; 1652 public static final long DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS = 1653 -1; 1654 1655 /** 1656 * Max number of threads in NMClientAsync to process container management 1657 * events 1658 */ 1659 public static final String NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE = 1660 YARN_PREFIX + "client.nodemanager-client-async.thread-pool-max-size"; 1661 public static final int DEFAULT_NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE = 500; 1662 1663 /** 1664 * Maximum number of proxy connections to cache for node managers. If set 1665 * to a value greater than zero then the cache is enabled and the NMClient 1666 * and MRAppMaster will cache the specified number of node manager proxies. 1667 * There will be at max one proxy per node manager. Ex. configuring it to a 1668 * value of 5 will make sure that client will at max have 5 proxies cached 1669 * with 5 different node managers. These connections for these proxies will 1670 * be timed out if idle for more than the system wide idle timeout period. 1671 * Note that this could cause issues on large clusters as many connections 1672 * could linger simultaneously and lead to a large number of connection 1673 * threads. The token used for authentication will be used only at 1674 * connection creation time. If a new token is received then the earlier 1675 * connection should be closed in order to use the new token. This and 1676 * {@link YarnConfiguration#NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE} are related 1677 * and should be in sync (no need for them to be equal). 1678 * If the value of this property is zero then the connection cache is 1679 * disabled and connections will use a zero idle timeout to prevent too 1680 * many connection threads on large clusters. 1681 */ 1682 public static final String NM_CLIENT_MAX_NM_PROXIES = 1683 YARN_PREFIX + "client.max-cached-nodemanagers-proxies"; 1684 public static final int DEFAULT_NM_CLIENT_MAX_NM_PROXIES = 0; 1685 1686 /** Max time to wait to establish a connection to NM */ 1687 public static final String CLIENT_NM_CONNECT_MAX_WAIT_MS = 1688 YARN_PREFIX + "client.nodemanager-connect.max-wait-ms"; 1689 public static final long DEFAULT_CLIENT_NM_CONNECT_MAX_WAIT_MS = 1690 15 * 60 * 1000; 1691 1692 /** Time interval between each attempt to connect to NM */ 1693 public static final String CLIENT_NM_CONNECT_RETRY_INTERVAL_MS = 1694 YARN_PREFIX + "client.nodemanager-connect.retry-interval-ms"; 1695 public static final long DEFAULT_CLIENT_NM_CONNECT_RETRY_INTERVAL_MS 1696 = 10 * 1000; 1697 1698 public static final String YARN_HTTP_POLICY_KEY = YARN_PREFIX + "http.policy"; 1699 public static final String YARN_HTTP_POLICY_DEFAULT = HttpConfig.Policy.HTTP_ONLY 1700 .name(); 1701 1702 /** 1703 * Node-labels configurations 1704 */ 1705 public static final String NODE_LABELS_PREFIX = YARN_PREFIX + "node-labels."; 1706 1707 /** URI for NodeLabelManager */ 1708 public static final String FS_NODE_LABELS_STORE_ROOT_DIR = NODE_LABELS_PREFIX 1709 + "fs-store.root-dir"; 1710 public static final String FS_NODE_LABELS_STORE_RETRY_POLICY_SPEC = 1711 NODE_LABELS_PREFIX + "fs-store.retry-policy-spec"; 1712 public static final String DEFAULT_FS_NODE_LABELS_STORE_RETRY_POLICY_SPEC = 1713 "2000, 500"; 1714 1715 /** 1716 * Flag to indicate if the node labels feature enabled, by default it's 1717 * disabled 1718 */ 1719 public static final String NODE_LABELS_ENABLED = NODE_LABELS_PREFIX 1720 + "enabled"; 1721 public static final boolean DEFAULT_NODE_LABELS_ENABLED = false; 1722 1723 public YarnConfiguration() { 1724 super(); 1725 } 1726 1727 public YarnConfiguration(Configuration conf) { 1728 super(conf); 1729 if (! (conf instanceof YarnConfiguration)) { 1730 this.reloadConfiguration(); 1731 } 1732 } 1733 1734 @Private 1735 public static List<String> getServiceAddressConfKeys(Configuration conf) { 1736 return useHttps(conf) ? RM_SERVICES_ADDRESS_CONF_KEYS_HTTPS 1737 : RM_SERVICES_ADDRESS_CONF_KEYS_HTTP; 1738 } 1739 1740 /** 1741 * Get the socket address for <code>name</code> property as a 1742 * <code>InetSocketAddress</code>. On a HA cluster, 1743 * this fetches the address corresponding to the RM identified by 1744 * {@link #RM_HA_ID}. 1745 * @param name property name. 1746 * @param defaultAddress the default value 1747 * @param defaultPort the default port 1748 * @return InetSocketAddress 1749 */ 1750 @Override 1751 public InetSocketAddress getSocketAddr( 1752 String name, String defaultAddress, int defaultPort) { 1753 String address; 1754 if (HAUtil.isHAEnabled(this) && getServiceAddressConfKeys(this).contains(name)) { 1755 address = HAUtil.getConfValueForRMInstance(name, defaultAddress, this); 1756 } else { 1757 address = get(name, defaultAddress); 1758 } 1759 return NetUtils.createSocketAddr(address, defaultPort, name); 1760 } 1761 1762 @Override 1763 public InetSocketAddress updateConnectAddr(String name, 1764 InetSocketAddress addr) { 1765 String prefix = name; 1766 if (HAUtil.isHAEnabled(this)) { 1767 prefix = HAUtil.addSuffix(prefix, HAUtil.getRMHAId(this)); 1768 } 1769 return super.updateConnectAddr(prefix, addr); 1770 } 1771 1772 @Private 1773 public static int getRMDefaultPortNumber(String addressPrefix, 1774 Configuration conf) { 1775 if (addressPrefix.equals(YarnConfiguration.RM_ADDRESS)) { 1776 return YarnConfiguration.DEFAULT_RM_PORT; 1777 } else if (addressPrefix.equals(YarnConfiguration.RM_SCHEDULER_ADDRESS)) { 1778 return YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT; 1779 } else if (addressPrefix.equals(YarnConfiguration.RM_WEBAPP_ADDRESS)) { 1780 return YarnConfiguration.DEFAULT_RM_WEBAPP_PORT; 1781 } else if (addressPrefix.equals(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS)) { 1782 return YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT; 1783 } else if (addressPrefix 1784 .equals(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS)) { 1785 return YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT; 1786 } else if (addressPrefix.equals(YarnConfiguration.RM_ADMIN_ADDRESS)) { 1787 return YarnConfiguration.DEFAULT_RM_ADMIN_PORT; 1788 } else { 1789 throw new HadoopIllegalArgumentException( 1790 "Invalid RM RPC address Prefix: " + addressPrefix 1791 + ". The valid value should be one of " 1792 + getServiceAddressConfKeys(conf)); 1793 } 1794 } 1795 1796 public static boolean useHttps(Configuration conf) { 1797 return HttpConfig.Policy.HTTPS_ONLY == HttpConfig.Policy.fromString(conf 1798 .get(YARN_HTTP_POLICY_KEY, 1799 YARN_HTTP_POLICY_DEFAULT)); 1800 } 1801 1802 @Private 1803 public static String getClusterId(Configuration conf) { 1804 String clusterId = conf.get(YarnConfiguration.RM_CLUSTER_ID); 1805 if (clusterId == null) { 1806 throw new HadoopIllegalArgumentException("Configuration doesn't specify " + 1807 YarnConfiguration.RM_CLUSTER_ID); 1808 } 1809 return clusterId; 1810 } 1811}