@InterfaceAudience.LimitedPrivate(value="yarn") @InterfaceStability.Unstable public class FairScheduler extends AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
Modifier and Type | Field and Description |
---|---|
protected boolean |
assignMultiple |
static org.apache.hadoop.yarn.api.records.Resource |
CONTAINER_RESERVED |
protected boolean |
continuousSchedulingEnabled |
protected int |
continuousSchedulingSleepMs |
protected long |
lastPreemptionUpdateTime |
protected int |
maxAssign |
protected long |
nodeLocalityDelayMs |
protected double |
nodeLocalityThreshold |
protected boolean |
preemptionEnabled |
protected long |
preemptionInterval |
protected float |
preemptionUtilizationThreshold |
protected long |
rackLocalityDelayMs |
protected double |
rackLocalityThreshold |
protected boolean |
sizeBasedWeight |
protected long |
THREAD_JOIN_TIMEOUT_MS |
protected long |
updateInterval |
protected long |
waitTimeBeforeKill |
protected WeightAdjuster |
weightAdjuster |
applications, clusterResource, EMPTY_ALLOCATION, EMPTY_CONTAINER_LIST, maximumAllocation, minimumAllocation, nmExpireInterval, nodes, rmContext
Constructor and Description |
---|
FairScheduler() |
Modifier and Type | Method and Description |
---|---|
protected void |
addApplication(org.apache.hadoop.yarn.api.records.ApplicationId applicationId,
String queueName,
String user,
boolean isAppRecovering)
Add a new application to the scheduler, with a given id, queue name, and
user.
|
protected void |
addApplicationAttempt(org.apache.hadoop.yarn.api.records.ApplicationAttemptId applicationAttemptId,
boolean transferStateFromPreviousAttempt,
boolean isAttemptRecovering)
Add a new application attempt to the scheduler.
|
Allocation |
allocate(org.apache.hadoop.yarn.api.records.ApplicationAttemptId appAttemptId,
List<org.apache.hadoop.yarn.api.records.ResourceRequest> ask,
List<org.apache.hadoop.yarn.api.records.ContainerId> release,
List<String> blacklistAdditions,
List<String> blacklistRemovals,
List<org.apache.hadoop.yarn.api.records.UpdateContainerRequest> increaseRequests,
List<org.apache.hadoop.yarn.api.records.UpdateContainerRequest> decreaseRequests)
The main api between the ApplicationMaster and the Scheduler.
|
boolean |
checkAccess(org.apache.hadoop.security.UserGroupInformation callerUGI,
org.apache.hadoop.yarn.api.records.QueueACL acl,
String queueName)
Check if the user has permission to perform the operation.
|
protected void |
completedContainerInternal(RMContainer rmContainer,
org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus,
RMContainerEventType event)
Clean up a completed container.
|
protected void |
decreaseContainer(SchedContainerChangeRequest decreaseRequest,
SchedulerApplicationAttempt attempt) |
AllocationConfiguration |
getAllocationConfiguration() |
List<org.apache.hadoop.yarn.api.records.ApplicationAttemptId> |
getAppsInQueue(String queueName)
Gets the apps under a given queue
|
ResourceWeights |
getAppWeight(FSAppAttempt app) |
org.apache.hadoop.yarn.util.Clock |
getClock() |
FairSchedulerConfiguration |
getConf() |
RMContainerTokenSecretManager |
getContainerTokenSecretManager() |
int |
getContinuousSchedulingSleepMs() |
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerEventLog |
getEventLog() |
org.apache.hadoop.yarn.api.records.Resource |
getIncrementResourceCapability() |
long |
getNodeLocalityDelayMs() |
double |
getNodeLocalityThreshold() |
int |
getNumClusterNodes()
Get the number of nodes available in the cluster.
|
int |
getNumNodesInRack(String rackName) |
Set<String> |
getPlanQueues()
Gets the list of names for queues managed by the Reservation System
|
org.apache.hadoop.yarn.api.records.QueueInfo |
getQueueInfo(String queueName,
boolean includeChildQueues,
boolean recursive)
Get queue information
|
QueueManager |
getQueueManager() |
List<org.apache.hadoop.yarn.api.records.QueueUserACLInfo> |
getQueueUserAclInfo()
Get acls for queues for current user.
|
long |
getRackLocalityDelayMs() |
double |
getRackLocalityThreshold() |
float |
getReservableNodesRatio() |
org.apache.hadoop.yarn.util.resource.ResourceCalculator |
getResourceCalculator() |
QueueMetrics |
getRootQueueMetrics()
Get the root queue for the scheduler.
|
FSAppAttempt |
getSchedulerApp(org.apache.hadoop.yarn.api.records.ApplicationAttemptId appAttemptId) |
EnumSet<org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes> |
getSchedulingResourceTypes()
Return a collection of the resource types that are considered when
scheduling
|
void |
handle(SchedulerEvent event) |
boolean |
isAtLeastReservationThreshold(org.apache.hadoop.yarn.util.resource.ResourceCalculator resourceCalculator,
org.apache.hadoop.yarn.api.records.Resource resource) |
boolean |
isContinuousSchedulingEnabled() |
String |
moveApplication(org.apache.hadoop.yarn.api.records.ApplicationId appId,
String queueName)
Moves the given application to the given queue
|
protected void |
preemptResources(org.apache.hadoop.yarn.api.records.Resource toPreempt)
Preempt a quantity of resources.
|
protected void |
preemptTasksIfNecessary()
Check for queues that need tasks preempted, either because they have been
below their guaranteed share for minSharePreemptionTimeout or they have
been below their fair share threshold for the fairSharePreemptionTimeout.
|
void |
recover(RMStateStore.RMState state) |
void |
reinitialize(org.apache.hadoop.conf.Configuration conf,
RMContext rmContext)
Re-initialize the
ResourceScheduler . |
void |
removeQueue(String queueName)
Only supports removing empty leaf queues
|
protected org.apache.hadoop.yarn.api.records.Resource |
resourceDeficit(FSLeafQueue sched,
long curTime)
Return the resource amount that this queue is allowed to preempt, if any.
|
void |
serviceInit(org.apache.hadoop.conf.Configuration conf) |
void |
serviceStart() |
void |
serviceStop() |
void |
setEntitlement(String queueName,
QueueEntitlement entitlement)
This method increase the entitlement for current queue (must respect
invariants, e.g., no overcommit of parents, non negative, etc.).
|
void |
setRMContext(RMContext rmContext)
Set RMContext for
ResourceScheduler . |
protected void |
update()
Recompute the internal variables used by the scheduler - per-job weights,
fair shares, deficits, minimum slot allocations, and amount of used and
required resources per job.
|
void |
updateNodeResource(RMNode nm,
org.apache.hadoop.yarn.api.records.ResourceOption resourceOption)
Process resource update on a node and update Queue.
|
protected void |
warnOrKillContainer(RMContainer container) |
addBlacklistedNodeIdsToList, addQueue, checkAndGetApplicationPriority, clearPendingContainerCache, completedContainer, containerIncreasedOnNode, containerLaunchedOnNode, createReleaseCache, createSchedContainerChangeRequests, decreaseContainers, getApplicationAttempt, getAppResourceUsageReport, getClusterResource, getCurrentAttemptForContainer, getMaxClusterLevelAppPriority, getMaximumResourceCapability, getMaximumResourceCapability, getMinimumResourceCapability, getNodeReport, getPendingResourceRequestsForAttempt, getRMContainer, getSchedulerAppInfo, getSchedulerApplications, getSchedulerNode, getTransferredContainers, initMaximumResourceCapability, killAllAppsInQueue, moveAllApps, recoverContainersOnNode, refreshMaximumAllocation, releaseContainers, setClusterMaxPriority, updateApplicationPriority, updateMaximumAllocation
close, getBlockers, getConfig, getFailureCause, getFailureState, getLifecycleHistory, getName, getServiceState, getStartTime, init, isInState, noteFailure, putBlocker, registerGlobalListener, registerServiceListener, removeBlocker, setConfig, start, stop, toString, unregisterGlobalListener, unregisterServiceListener, waitForServiceToStop
public static final org.apache.hadoop.yarn.api.records.Resource CONTAINER_RESERVED
protected long updateInterval
protected final long THREAD_JOIN_TIMEOUT_MS
protected long lastPreemptionUpdateTime
protected boolean preemptionEnabled
protected float preemptionUtilizationThreshold
protected long preemptionInterval
protected long waitTimeBeforeKill
protected boolean sizeBasedWeight
protected WeightAdjuster weightAdjuster
protected boolean continuousSchedulingEnabled
protected int continuousSchedulingSleepMs
protected double nodeLocalityThreshold
protected double rackLocalityThreshold
protected long nodeLocalityDelayMs
protected long rackLocalityDelayMs
protected boolean assignMultiple
protected int maxAssign
public boolean isAtLeastReservationThreshold(org.apache.hadoop.yarn.util.resource.ResourceCalculator resourceCalculator, org.apache.hadoop.yarn.api.records.Resource resource)
public FairSchedulerConfiguration getConf()
public int getNumNodesInRack(String rackName)
public QueueManager getQueueManager()
protected void update()
protected void preemptTasksIfNecessary()
protected void preemptResources(org.apache.hadoop.yarn.api.records.Resource toPreempt)
protected void warnOrKillContainer(RMContainer container)
protected org.apache.hadoop.yarn.api.records.Resource resourceDeficit(FSLeafQueue sched, long curTime)
public RMContainerTokenSecretManager getContainerTokenSecretManager()
public ResourceWeights getAppWeight(FSAppAttempt app)
public org.apache.hadoop.yarn.api.records.Resource getIncrementResourceCapability()
public double getNodeLocalityThreshold()
public double getRackLocalityThreshold()
public long getNodeLocalityDelayMs()
public long getRackLocalityDelayMs()
public boolean isContinuousSchedulingEnabled()
public int getContinuousSchedulingSleepMs()
public org.apache.hadoop.yarn.util.Clock getClock()
public org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerEventLog getEventLog()
protected void addApplication(org.apache.hadoop.yarn.api.records.ApplicationId applicationId, String queueName, String user, boolean isAppRecovering)
protected void addApplicationAttempt(org.apache.hadoop.yarn.api.records.ApplicationAttemptId applicationAttemptId, boolean transferStateFromPreviousAttempt, boolean isAttemptRecovering)
protected void completedContainerInternal(RMContainer rmContainer, org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus, RMContainerEventType event)
completedContainerInternal
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
public Allocation allocate(org.apache.hadoop.yarn.api.records.ApplicationAttemptId appAttemptId, List<org.apache.hadoop.yarn.api.records.ResourceRequest> ask, List<org.apache.hadoop.yarn.api.records.ContainerId> release, List<String> blacklistAdditions, List<String> blacklistRemovals, List<org.apache.hadoop.yarn.api.records.UpdateContainerRequest> increaseRequests, List<org.apache.hadoop.yarn.api.records.UpdateContainerRequest> decreaseRequests)
YarnScheduler
Allocation
for the applicationpublic FSAppAttempt getSchedulerApp(org.apache.hadoop.yarn.api.records.ApplicationAttemptId appAttemptId)
public org.apache.hadoop.yarn.util.resource.ResourceCalculator getResourceCalculator()
public QueueMetrics getRootQueueMetrics()
YarnScheduler
public void handle(SchedulerEvent event)
public void recover(RMStateStore.RMState state) throws Exception
Exception
public void setRMContext(RMContext rmContext)
ResourceScheduler
ResourceScheduler
.
This method should be called immediately after instantiating
a scheduler once.rmContext
- created by ResourceManagerpublic void serviceInit(org.apache.hadoop.conf.Configuration conf) throws Exception
serviceInit
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
Exception
public void serviceStart() throws Exception
serviceStart
in class org.apache.hadoop.service.AbstractService
Exception
public void serviceStop() throws Exception
serviceStop
in class org.apache.hadoop.service.AbstractService
Exception
public void reinitialize(org.apache.hadoop.conf.Configuration conf, RMContext rmContext) throws IOException
ResourceScheduler
ResourceScheduler
.conf
- configurationIOException
public org.apache.hadoop.yarn.api.records.QueueInfo getQueueInfo(String queueName, boolean includeChildQueues, boolean recursive) throws IOException
YarnScheduler
queueName
- queue nameincludeChildQueues
- include child queues?recursive
- get children queues?IOException
public List<org.apache.hadoop.yarn.api.records.QueueUserACLInfo> getQueueUserAclInfo()
YarnScheduler
public int getNumClusterNodes()
YarnScheduler
public boolean checkAccess(org.apache.hadoop.security.UserGroupInformation callerUGI, org.apache.hadoop.yarn.api.records.QueueACL acl, String queueName)
YarnScheduler
QueueACL.ADMINISTER_QUEUE
permission,
this user can view/modify the applications in this queuetrue
if the user has the permission,
false
otherwisepublic AllocationConfiguration getAllocationConfiguration()
public List<org.apache.hadoop.yarn.api.records.ApplicationAttemptId> getAppsInQueue(String queueName)
YarnScheduler
queueName
- the name of the queue.public String moveApplication(org.apache.hadoop.yarn.api.records.ApplicationId appId, String queueName) throws org.apache.hadoop.yarn.exceptions.YarnException
YarnScheduler
moveApplication
in interface YarnScheduler
moveApplication
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
org.apache.hadoop.yarn.exceptions.YarnException
- if the move cannot be carried outpublic void updateNodeResource(RMNode nm, org.apache.hadoop.yarn.api.records.ResourceOption resourceOption)
updateNodeResource
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
public EnumSet<org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes> getSchedulingResourceTypes()
getSchedulingResourceTypes
in interface YarnScheduler
getSchedulingResourceTypes
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
public Set<String> getPlanQueues() throws org.apache.hadoop.yarn.exceptions.YarnException
YarnScheduler
getPlanQueues
in interface YarnScheduler
getPlanQueues
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
org.apache.hadoop.yarn.exceptions.YarnException
public void setEntitlement(String queueName, QueueEntitlement entitlement) throws org.apache.hadoop.yarn.exceptions.YarnException
YarnScheduler
setEntitlement
in interface YarnScheduler
setEntitlement
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
queueName
- the queue for which we change entitlemententitlement
- the new entitlement for the queue (capacity,
maxCapacity, etc..)org.apache.hadoop.yarn.exceptions.YarnException
public void removeQueue(String queueName) throws org.apache.hadoop.yarn.exceptions.YarnException
removeQueue
in interface YarnScheduler
removeQueue
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
queueName
- name of queue to removeorg.apache.hadoop.yarn.exceptions.YarnException
- if queue to remove is either not a leaf or if its
not emptyprotected void decreaseContainer(SchedContainerChangeRequest decreaseRequest, SchedulerApplicationAttempt attempt)
decreaseContainer
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
public float getReservableNodesRatio()
Copyright © 2017 Apache Software Foundation. All Rights Reserved.