@InterfaceAudience.Private @InterfaceStability.Unstable public abstract class AbstractYarnScheduler<T extends SchedulerApplicationAttempt,N extends SchedulerNode> extends org.apache.hadoop.service.AbstractService implements ResourceScheduler
Modifier and Type | Field and Description |
---|---|
protected ConcurrentMap<org.apache.hadoop.yarn.api.records.ApplicationId,SchedulerApplication<T>> |
applications |
protected org.apache.hadoop.yarn.api.records.Resource |
clusterResource |
protected static Allocation |
EMPTY_ALLOCATION |
protected static List<org.apache.hadoop.yarn.api.records.Container> |
EMPTY_CONTAINER_LIST |
protected org.apache.hadoop.yarn.api.records.Resource |
maximumAllocation |
protected org.apache.hadoop.yarn.api.records.Resource |
minimumAllocation |
protected int |
nmExpireInterval |
protected Map<org.apache.hadoop.yarn.api.records.NodeId,N> |
nodes |
protected RMContext |
rmContext |
Constructor and Description |
---|
AbstractYarnScheduler(String name)
Construct the service.
|
Modifier and Type | Method and Description |
---|---|
void |
addBlacklistedNodeIdsToList(SchedulerApplicationAttempt app,
List<org.apache.hadoop.yarn.api.records.NodeId> blacklistNodeIdList)
Add blacklisted NodeIds to the list that is passed.
|
void |
addQueue(Queue newQueue)
Add to the scheduler a new Queue.
|
org.apache.hadoop.yarn.api.records.Priority |
checkAndGetApplicationPriority(org.apache.hadoop.yarn.api.records.Priority priorityFromContext,
String user,
String queueName,
org.apache.hadoop.yarn.api.records.ApplicationId applicationId)
Verify whether a submitted application priority is valid as per configured
Queue
|
void |
clearPendingContainerCache() |
void |
completedContainer(RMContainer rmContainer,
org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus,
RMContainerEventType event) |
protected abstract void |
completedContainerInternal(RMContainer rmContainer,
org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus,
RMContainerEventType event) |
protected void |
containerIncreasedOnNode(org.apache.hadoop.yarn.api.records.ContainerId containerId,
SchedulerNode node,
org.apache.hadoop.yarn.api.records.Container increasedContainerReportedByNM) |
protected void |
containerLaunchedOnNode(org.apache.hadoop.yarn.api.records.ContainerId containerId,
SchedulerNode node) |
protected void |
createReleaseCache() |
protected List<SchedContainerChangeRequest> |
createSchedContainerChangeRequests(List<org.apache.hadoop.yarn.api.records.UpdateContainerRequest> changeRequests,
boolean increase) |
protected abstract void |
decreaseContainer(SchedContainerChangeRequest decreaseRequest,
SchedulerApplicationAttempt attempt) |
protected void |
decreaseContainers(List<org.apache.hadoop.yarn.api.records.UpdateContainerRequest> decreaseRequests,
SchedulerApplicationAttempt attempt) |
T |
getApplicationAttempt(org.apache.hadoop.yarn.api.records.ApplicationAttemptId applicationAttemptId) |
org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport |
getAppResourceUsageReport(org.apache.hadoop.yarn.api.records.ApplicationAttemptId appAttemptId)
Get a resource usage report from a given app attempt ID.
|
org.apache.hadoop.yarn.api.records.Resource |
getClusterResource()
Get the whole resource capacity of the cluster.
|
T |
getCurrentAttemptForContainer(org.apache.hadoop.yarn.api.records.ContainerId containerId) |
org.apache.hadoop.yarn.api.records.Priority |
getMaxClusterLevelAppPriority()
Get cluster max priority.
|
org.apache.hadoop.yarn.api.records.Resource |
getMaximumResourceCapability()
Get maximum allocatable
Resource at the cluster level. |
org.apache.hadoop.yarn.api.records.Resource |
getMaximumResourceCapability(String queueName)
Get maximum allocatable
Resource for the queue specified. |
org.apache.hadoop.yarn.api.records.Resource |
getMinimumResourceCapability()
Get minimum allocatable
Resource . |
SchedulerNodeReport |
getNodeReport(org.apache.hadoop.yarn.api.records.NodeId nodeId)
Get node resource usage report.
|
List<org.apache.hadoop.yarn.api.records.ResourceRequest> |
getPendingResourceRequestsForAttempt(org.apache.hadoop.yarn.api.records.ApplicationAttemptId attemptId) |
Set<String> |
getPlanQueues()
Gets the list of names for queues managed by the Reservation System
|
RMContainer |
getRMContainer(org.apache.hadoop.yarn.api.records.ContainerId containerId)
Get the container for the given containerId.
|
SchedulerAppReport |
getSchedulerAppInfo(org.apache.hadoop.yarn.api.records.ApplicationAttemptId appAttemptId)
Get the Scheduler app for a given app attempt Id.
|
Map<org.apache.hadoop.yarn.api.records.ApplicationId,SchedulerApplication<T>> |
getSchedulerApplications() |
SchedulerNode |
getSchedulerNode(org.apache.hadoop.yarn.api.records.NodeId nodeId)
Get SchedulerNode corresponds to nodeId.
|
EnumSet<org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes> |
getSchedulingResourceTypes()
Return a collection of the resource types that are considered when
scheduling
|
List<org.apache.hadoop.yarn.api.records.Container> |
getTransferredContainers(org.apache.hadoop.yarn.api.records.ApplicationAttemptId currentAttempt)
Get previous attempts' live containers for work-preserving AM restart.
|
protected void |
initMaximumResourceCapability(org.apache.hadoop.yarn.api.records.Resource maximumAllocation) |
void |
killAllAppsInQueue(String queueName)
Terminate all applications in the specified queue.
|
void |
moveAllApps(String sourceQueue,
String destQueue)
Completely drain sourceQueue of applications, by moving all of them to
destQueue.
|
String |
moveApplication(org.apache.hadoop.yarn.api.records.ApplicationId appId,
String newQueue)
Moves the given application to the given queue
|
void |
recoverContainersOnNode(List<org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus> containerReports,
RMNode nm) |
protected void |
refreshMaximumAllocation(org.apache.hadoop.yarn.api.records.Resource newMaxAlloc) |
protected void |
releaseContainers(List<org.apache.hadoop.yarn.api.records.ContainerId> containers,
SchedulerApplicationAttempt attempt) |
void |
removeQueue(String queueName)
Remove an existing queue.
|
void |
serviceInit(org.apache.hadoop.conf.Configuration conf) |
void |
setClusterMaxPriority(org.apache.hadoop.conf.Configuration conf)
Set the cluster max priority
|
void |
setEntitlement(String queue,
QueueEntitlement entitlement)
This method increase the entitlement for current queue (must respect
invariants, e.g., no overcommit of parents, non negative, etc.).
|
void |
updateApplicationPriority(org.apache.hadoop.yarn.api.records.Priority newPriority,
org.apache.hadoop.yarn.api.records.ApplicationId applicationId)
Change application priority of a submitted application at runtime
|
protected void |
updateMaximumAllocation(SchedulerNode node,
boolean add) |
void |
updateNodeResource(RMNode nm,
org.apache.hadoop.yarn.api.records.ResourceOption resourceOption)
Process resource update on a node.
|
close, getBlockers, getConfig, getFailureCause, getFailureState, getLifecycleHistory, getName, getServiceState, getStartTime, init, isInState, noteFailure, putBlocker, registerGlobalListener, registerServiceListener, removeBlocker, serviceStart, serviceStop, setConfig, start, stop, toString, unregisterGlobalListener, unregisterServiceListener, waitForServiceToStop
clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait
reinitialize, setRMContext
allocate, checkAccess, getAppsInQueue, getNumClusterNodes, getQueueInfo, getQueueUserAclInfo, getResourceCalculator, getRootQueueMetrics
recover
protected Map<org.apache.hadoop.yarn.api.records.NodeId,N extends SchedulerNode> nodes
protected org.apache.hadoop.yarn.api.records.Resource clusterResource
protected org.apache.hadoop.yarn.api.records.Resource minimumAllocation
protected org.apache.hadoop.yarn.api.records.Resource maximumAllocation
protected RMContext rmContext
protected ConcurrentMap<org.apache.hadoop.yarn.api.records.ApplicationId,SchedulerApplication<T extends SchedulerApplicationAttempt>> applications
protected int nmExpireInterval
protected static final List<org.apache.hadoop.yarn.api.records.Container> EMPTY_CONTAINER_LIST
protected static final Allocation EMPTY_ALLOCATION
public AbstractYarnScheduler(String name)
name
- service namepublic void serviceInit(org.apache.hadoop.conf.Configuration conf) throws Exception
serviceInit
in class org.apache.hadoop.service.AbstractService
Exception
public List<org.apache.hadoop.yarn.api.records.Container> getTransferredContainers(org.apache.hadoop.yarn.api.records.ApplicationAttemptId currentAttempt)
YarnScheduler
getTransferredContainers
in interface YarnScheduler
currentAttempt
- the id of the application attemptpublic Map<org.apache.hadoop.yarn.api.records.ApplicationId,SchedulerApplication<T>> getSchedulerApplications()
public void addBlacklistedNodeIdsToList(SchedulerApplicationAttempt app, List<org.apache.hadoop.yarn.api.records.NodeId> blacklistNodeIdList)
app
- application attempt.blacklistNodeIdList
- the list to store blacklisted NodeIds.public org.apache.hadoop.yarn.api.records.Resource getClusterResource()
YarnScheduler
getClusterResource
in interface YarnScheduler
public org.apache.hadoop.yarn.api.records.Resource getMinimumResourceCapability()
YarnScheduler
Resource
.getMinimumResourceCapability
in interface YarnScheduler
public org.apache.hadoop.yarn.api.records.Resource getMaximumResourceCapability()
YarnScheduler
Resource
at the cluster level.getMaximumResourceCapability
in interface YarnScheduler
public org.apache.hadoop.yarn.api.records.Resource getMaximumResourceCapability(String queueName)
YarnScheduler
Resource
for the queue specified.getMaximumResourceCapability
in interface YarnScheduler
queueName
- queue nameprotected void initMaximumResourceCapability(org.apache.hadoop.yarn.api.records.Resource maximumAllocation)
protected void containerLaunchedOnNode(org.apache.hadoop.yarn.api.records.ContainerId containerId, SchedulerNode node)
protected void containerIncreasedOnNode(org.apache.hadoop.yarn.api.records.ContainerId containerId, SchedulerNode node, org.apache.hadoop.yarn.api.records.Container increasedContainerReportedByNM)
public T getApplicationAttempt(org.apache.hadoop.yarn.api.records.ApplicationAttemptId applicationAttemptId)
public SchedulerAppReport getSchedulerAppInfo(org.apache.hadoop.yarn.api.records.ApplicationAttemptId appAttemptId)
YarnScheduler
getSchedulerAppInfo
in interface YarnScheduler
appAttemptId
- the id of the application attemptpublic org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport getAppResourceUsageReport(org.apache.hadoop.yarn.api.records.ApplicationAttemptId appAttemptId)
YarnScheduler
getAppResourceUsageReport
in interface YarnScheduler
appAttemptId
- the id of the application attemptpublic T getCurrentAttemptForContainer(org.apache.hadoop.yarn.api.records.ContainerId containerId)
public RMContainer getRMContainer(org.apache.hadoop.yarn.api.records.ContainerId containerId)
YarnScheduler
getRMContainer
in interface YarnScheduler
public SchedulerNodeReport getNodeReport(org.apache.hadoop.yarn.api.records.NodeId nodeId)
YarnScheduler
getNodeReport
in interface YarnScheduler
SchedulerNodeReport
for the node or null
if nodeId does not point to a defined node.public String moveApplication(org.apache.hadoop.yarn.api.records.ApplicationId appId, String newQueue) throws org.apache.hadoop.yarn.exceptions.YarnException
YarnScheduler
moveApplication
in interface YarnScheduler
org.apache.hadoop.yarn.exceptions.YarnException
- if the move cannot be carried outpublic void removeQueue(String queueName) throws org.apache.hadoop.yarn.exceptions.YarnException
YarnScheduler
removeQueue
in interface YarnScheduler
queueName
- name of the queue to removeorg.apache.hadoop.yarn.exceptions.YarnException
public void addQueue(Queue newQueue) throws org.apache.hadoop.yarn.exceptions.YarnException
YarnScheduler
addQueue
in interface YarnScheduler
newQueue
- the queue being added.org.apache.hadoop.yarn.exceptions.YarnException
public void setEntitlement(String queue, QueueEntitlement entitlement) throws org.apache.hadoop.yarn.exceptions.YarnException
YarnScheduler
setEntitlement
in interface YarnScheduler
queue
- the queue for which we change entitlemententitlement
- the new entitlement for the queue (capacity,
maxCapacity, etc..)org.apache.hadoop.yarn.exceptions.YarnException
public void recoverContainersOnNode(List<org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus> containerReports, RMNode nm)
protected void createReleaseCache()
public void clearPendingContainerCache()
@InterfaceAudience.Private public void completedContainer(RMContainer rmContainer, org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus, RMContainerEventType event)
protected abstract void completedContainerInternal(RMContainer rmContainer, org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus, RMContainerEventType event)
protected void releaseContainers(List<org.apache.hadoop.yarn.api.records.ContainerId> containers, SchedulerApplicationAttempt attempt)
protected void decreaseContainers(List<org.apache.hadoop.yarn.api.records.UpdateContainerRequest> decreaseRequests, SchedulerApplicationAttempt attempt)
protected abstract void decreaseContainer(SchedContainerChangeRequest decreaseRequest, SchedulerApplicationAttempt attempt)
public SchedulerNode getSchedulerNode(org.apache.hadoop.yarn.api.records.NodeId nodeId)
YarnScheduler
getSchedulerNode
in interface YarnScheduler
nodeId
- the node id of RMNodepublic void moveAllApps(String sourceQueue, String destQueue) throws org.apache.hadoop.yarn.exceptions.YarnException
YarnScheduler
moveAllApps
in interface YarnScheduler
org.apache.hadoop.yarn.exceptions.YarnException
public void killAllAppsInQueue(String queueName) throws org.apache.hadoop.yarn.exceptions.YarnException
YarnScheduler
killAllAppsInQueue
in interface YarnScheduler
queueName
- the name of queue to be drainedorg.apache.hadoop.yarn.exceptions.YarnException
public void updateNodeResource(RMNode nm, org.apache.hadoop.yarn.api.records.ResourceOption resourceOption)
public EnumSet<org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes> getSchedulingResourceTypes()
getSchedulingResourceTypes
in interface YarnScheduler
public Set<String> getPlanQueues() throws org.apache.hadoop.yarn.exceptions.YarnException
YarnScheduler
getPlanQueues
in interface YarnScheduler
org.apache.hadoop.yarn.exceptions.YarnException
protected void updateMaximumAllocation(SchedulerNode node, boolean add)
protected void refreshMaximumAllocation(org.apache.hadoop.yarn.api.records.Resource newMaxAlloc)
public List<org.apache.hadoop.yarn.api.records.ResourceRequest> getPendingResourceRequestsForAttempt(org.apache.hadoop.yarn.api.records.ApplicationAttemptId attemptId)
getPendingResourceRequestsForAttempt
in interface YarnScheduler
public org.apache.hadoop.yarn.api.records.Priority checkAndGetApplicationPriority(org.apache.hadoop.yarn.api.records.Priority priorityFromContext, String user, String queueName, org.apache.hadoop.yarn.api.records.ApplicationId applicationId) throws org.apache.hadoop.yarn.exceptions.YarnException
YarnScheduler
checkAndGetApplicationPriority
in interface YarnScheduler
priorityFromContext
- Submitted Application priority.user
- User who submitted the ApplicationqueueName
- Name of the QueueapplicationId
- Application IDorg.apache.hadoop.yarn.exceptions.YarnException
public void updateApplicationPriority(org.apache.hadoop.yarn.api.records.Priority newPriority, org.apache.hadoop.yarn.api.records.ApplicationId applicationId) throws org.apache.hadoop.yarn.exceptions.YarnException
YarnScheduler
updateApplicationPriority
in interface YarnScheduler
newPriority
- Submitted Application priority.applicationId
- Application IDorg.apache.hadoop.yarn.exceptions.YarnException
public org.apache.hadoop.yarn.api.records.Priority getMaxClusterLevelAppPriority()
YarnScheduler
getMaxClusterLevelAppPriority
in interface YarnScheduler
public void setClusterMaxPriority(org.apache.hadoop.conf.Configuration conf) throws org.apache.hadoop.yarn.exceptions.YarnException
YarnScheduler
setClusterMaxPriority
in interface YarnScheduler
org.apache.hadoop.yarn.exceptions.YarnException
protected List<SchedContainerChangeRequest> createSchedContainerChangeRequests(List<org.apache.hadoop.yarn.api.records.UpdateContainerRequest> changeRequests, boolean increase)
Copyright © 2017 Apache Software Foundation. All Rights Reserved.