@InterfaceAudience.Private @InterfaceStability.Unstable public class LeafQueue extends AbstractCSQueue
Modifier and Type | Class and Description |
---|---|
static class |
LeafQueue.User |
Modifier and Type | Field and Description |
---|---|
protected int |
maxApplications |
protected int |
maxApplicationsPerUser |
authorizer, csContext, queueEntity
Constructor and Description |
---|
LeafQueue(CapacitySchedulerContext cs,
String queueName,
CSQueue parent,
CSQueue old) |
Modifier and Type | Method and Description |
---|---|
CSAssignment |
assignContainers(org.apache.hadoop.yarn.api.records.Resource clusterResource,
FiCaSchedulerNode node,
ResourceLimits currentResourceLimits,
SchedulingMode schedulingMode)
Assign containers to applications in the queue or it's children (if any).
|
void |
attachContainer(org.apache.hadoop.yarn.api.records.Resource clusterResource,
FiCaSchedulerApp application,
RMContainer rmContainer)
Attach a container to this queue
|
org.apache.hadoop.yarn.api.records.Resource |
calculateAndGetAMResourceLimit() |
org.apache.hadoop.yarn.api.records.Resource |
calculateAndGetAMResourceLimitPerPartition(String nodePartition) |
protected boolean |
canAssignToUser(org.apache.hadoop.yarn.api.records.Resource clusterResource,
String userName,
org.apache.hadoop.yarn.api.records.Resource limit,
org.apache.hadoop.yarn.api.records.Resource rsrv,
String nodePartition,
ResourceLimits currentResourceLimits) |
void |
collectSchedulerApplications(Collection<org.apache.hadoop.yarn.api.records.ApplicationAttemptId> apps)
Adds all applications in the queue and its subqueues to the given collection.
|
void |
completedContainer(org.apache.hadoop.yarn.api.records.Resource clusterResource,
FiCaSchedulerApp application,
FiCaSchedulerNode node,
RMContainer rmContainer,
org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus,
RMContainerEventType event,
CSQueue childQueue,
boolean sortQueues)
A container assigned to the queue has completed.
|
void |
decAMUsedResource(String nodeLabel,
org.apache.hadoop.yarn.api.records.Resource resourceToDec,
SchedulerApplicationAttempt application) |
void |
decreaseContainer(org.apache.hadoop.yarn.api.records.Resource clusterResource,
SchedContainerChangeRequest decreaseRequest,
FiCaSchedulerApp app)
Decrease container resource in the queue
|
void |
decUsedResource(String nodeLabel,
org.apache.hadoop.yarn.api.records.Resource resourceToDec,
SchedulerApplicationAttempt application)
When partition of node updated, we will update queue's resource usage if it
has container(s) running on that.
|
void |
detachContainer(org.apache.hadoop.yarn.api.records.Resource clusterResource,
FiCaSchedulerApp application,
RMContainer rmContainer)
Detach a container from this queue
|
void |
finishApplication(org.apache.hadoop.yarn.api.records.ApplicationId application,
String user)
An application submitted to this queue has finished.
|
void |
finishApplicationAttempt(FiCaSchedulerApp application,
String queue)
An application attempt submitted to this queue has finished.
|
ActiveUsersManager |
getActiveUsersManager()
Get the
ActiveUsersManager for the queue. |
Collection<FiCaSchedulerApp> |
getAllApplications()
Obtain (read-only) collection of all applications.
|
Set<String> |
getAllUsers()
Get all valid users in this queue.
|
org.apache.hadoop.yarn.api.records.Resource |
getAMResourceLimit() |
org.apache.hadoop.yarn.api.records.Resource |
getAMResourceLimitPerPartition(String nodePartition) |
Collection<FiCaSchedulerApp> |
getApplications()
Obtain (read-only) collection of active applications.
|
List<CSQueue> |
getChildQueues()
Get child queues
|
org.apache.hadoop.yarn.api.records.Priority |
getDefaultApplicationPriority()
Get the Default Application Priority for this queue
|
protected org.apache.hadoop.yarn.api.records.Resource |
getHeadroom(LeafQueue.User user,
org.apache.hadoop.yarn.api.records.Resource queueCurrentLimit,
org.apache.hadoop.yarn.api.records.Resource clusterResource,
FiCaSchedulerApp application) |
protected org.apache.hadoop.yarn.api.records.Resource |
getHeadroom(LeafQueue.User user,
org.apache.hadoop.yarn.api.records.Resource queueCurrentLimit,
org.apache.hadoop.yarn.api.records.Resource clusterResource,
FiCaSchedulerApp application,
String partition) |
Map<String,TreeSet<RMContainer>> |
getIgnoreExclusivityRMContainers() |
float |
getMaxAMResourcePerQueuePercent()
Used only by tests.
|
int |
getMaxApplications() |
int |
getMaxApplicationsPerUser() |
float |
getMinimumAllocationFactor()
Used only by tests.
|
int |
getNodeLocalityDelay() |
int |
getNumActiveApplications() |
int |
getNumActiveApplications(String user) |
int |
getNumApplications()
Get the number of applications in the queue.
|
int |
getNumApplications(String user) |
int |
getNumPendingApplications() |
int |
getNumPendingApplications(String user) |
OrderingPolicy<FiCaSchedulerApp> |
getOrderingPolicy() |
Collection<FiCaSchedulerApp> |
getPendingApplications()
Obtain (read-only) collection of pending applications.
|
OrderingPolicy<FiCaSchedulerApp> |
getPendingAppsOrderingPolicy() |
org.apache.hadoop.yarn.api.records.QueueInfo |
getQueueInfo(boolean includeChildQueues,
boolean recursive)
Get queue information
|
List<org.apache.hadoop.yarn.api.records.QueueUserACLInfo> |
getQueueUserAclInfo(org.apache.hadoop.security.UserGroupInformation user)
Get queue ACLs for given
user . |
boolean |
getRackLocalityFullReset() |
org.apache.hadoop.yarn.api.records.Resource |
getResourceLimitForAllUsers(String userName,
org.apache.hadoop.yarn.api.records.Resource clusterResource,
String partition,
SchedulingMode schedulingMode) |
org.apache.hadoop.yarn.api.records.QueueState |
getState()
Get the current run-state of the queue
|
org.apache.hadoop.yarn.api.records.Resource |
getTotalPendingResourcesConsideringUserLimit(org.apache.hadoop.yarn.api.records.Resource resources,
String partition) |
float |
getUsageRatio(String label) |
LeafQueue.User |
getUser(String userName) |
org.apache.hadoop.yarn.api.records.Resource |
getUserAMResourceLimit() |
org.apache.hadoop.yarn.api.records.Resource |
getUserAMResourceLimitPerPartition(String nodePartition,
String userName) |
int |
getUserLimit() |
float |
getUserLimitFactor() |
org.apache.hadoop.yarn.api.records.Resource |
getUserLimitPerUser(String userName,
org.apache.hadoop.yarn.api.records.Resource resources,
String partition) |
ArrayList<UserInfo> |
getUsers() |
void |
incAMUsedResource(String nodeLabel,
org.apache.hadoop.yarn.api.records.Resource resourceToInc,
SchedulerApplicationAttempt application) |
void |
incUsedResource(String nodeLabel,
org.apache.hadoop.yarn.api.records.Resource resourceToInc,
SchedulerApplicationAttempt application)
When partition of node updated, we will update queue's resource usage if it
has container(s) running on that.
|
void |
recoverContainer(org.apache.hadoop.yarn.api.records.Resource clusterResource,
SchedulerApplicationAttempt attempt,
RMContainer rmContainer)
Recover the state of the queue for a given container.
|
void |
reinitialize(CSQueue newlyParsedQueue,
org.apache.hadoop.yarn.api.records.Resource clusterResource)
Reinitialize the queue.
|
void |
removeApplicationAttempt(FiCaSchedulerApp application,
LeafQueue.User user) |
void |
setAbsoluteCapacity(float absoluteCapacity) |
void |
setCapacity(float capacity) |
void |
setMaxApplications(int maxApplications) |
void |
setNodeLabelManager(RMNodeLabelsManager mgr) |
void |
setOrderingPolicy(OrderingPolicy<FiCaSchedulerApp> orderingPolicy) |
protected void |
setupQueueConfigs(org.apache.hadoop.yarn.api.records.Resource clusterResource) |
void |
submitApplication(org.apache.hadoop.yarn.api.records.ApplicationId applicationId,
String userName,
String queue)
Submit a new application to the queue.
|
void |
submitApplicationAttempt(FiCaSchedulerApp application,
String userName)
Submit an application attempt to the queue.
|
String |
toString() |
void |
unreserveIncreasedContainer(org.apache.hadoop.yarn.api.records.Resource clusterResource,
FiCaSchedulerApp app,
FiCaSchedulerNode node,
RMContainer rmContainer)
We have a reserved increased container in the queue, we need to unreserve
it.
|
void |
updateClusterResource(org.apache.hadoop.yarn.api.records.Resource clusterResource,
ResourceLimits currentResourceLimits)
Update the cluster resource for queues as we add/remove nodes
|
accessibleToPartition, decPendingResource, decReservedResource, getAbsoluteCapacity, getAbsoluteMaximumCapacity, getAbsoluteUsedCapacity, getAccessibleNodeLabels, getACLs, getCapacity, getDefaultNodeLabelExpression, getKillableContainers, getMaximumAllocation, getMaximumCapacity, getMetrics, getMinimumAllocation, getNodeLabelsForQueue, getNumContainers, getParent, getPreemptionDisabled, getPrivilegedEntity, getQueueCapacities, getQueueInfo, getQueueName, getQueuePath, getQueueResourceUsage, getQueueStatistics, getReservationContinueLooking, getTotalKillableResource, getUsedCapacity, getUsedResources, getUserWeights, hasAccess, hasChildQueues, incPendingResource, incReservedResource, releaseResource, setAbsoluteUsedCapacity, setParent, setupConfigurableCapacities, setUsedCapacity
protected int maxApplications
protected int maxApplicationsPerUser
public LeafQueue(CapacitySchedulerContext cs, String queueName, CSQueue parent, CSQueue old) throws IOException
IOException
protected void setupQueueConfigs(org.apache.hadoop.yarn.api.records.Resource clusterResource) throws IOException
IOException
@InterfaceAudience.Private public float getMinimumAllocationFactor()
@InterfaceAudience.Private public float getMaxAMResourcePerQueuePercent()
public int getMaxApplications()
public int getMaxApplicationsPerUser()
public ActiveUsersManager getActiveUsersManager()
CSQueue
ActiveUsersManager
for the queue.ActiveUsersManager
for the queuepublic List<CSQueue> getChildQueues()
CSQueue
public int getNumApplications()
CSQueue
public int getNumPendingApplications()
public int getNumActiveApplications()
@InterfaceAudience.Private public int getNumApplications(String user)
@InterfaceAudience.Private public int getNumPendingApplications(String user)
@InterfaceAudience.Private public int getNumActiveApplications(String user)
public org.apache.hadoop.yarn.api.records.QueueState getState()
CSQueue
getState
in interface CSQueue
getState
in class AbstractCSQueue
@InterfaceAudience.Private public int getUserLimit()
@InterfaceAudience.Private public float getUserLimitFactor()
public org.apache.hadoop.yarn.api.records.QueueInfo getQueueInfo(boolean includeChildQueues, boolean recursive)
Queue
includeChildQueues
- include child queues?recursive
- recursively get child queue information?public List<org.apache.hadoop.yarn.api.records.QueueUserACLInfo> getQueueUserAclInfo(org.apache.hadoop.security.UserGroupInformation user)
Queue
user
.user
- usernamepublic void setNodeLabelManager(RMNodeLabelsManager mgr)
public LeafQueue.User getUser(String userName)
public ArrayList<UserInfo> getUsers()
public void reinitialize(CSQueue newlyParsedQueue, org.apache.hadoop.yarn.api.records.Resource clusterResource) throws IOException
CSQueue
newlyParsedQueue
- new queue to re-initalize fromclusterResource
- resources in the clusterIOException
public void submitApplicationAttempt(FiCaSchedulerApp application, String userName)
CSQueue
public void submitApplication(org.apache.hadoop.yarn.api.records.ApplicationId applicationId, String userName, String queue) throws org.apache.hadoop.security.AccessControlException
CSQueue
applicationId
- the applicationId of the application being submitteduserName
- user who submitted the applicationqueue
- queue to which the application is submittedorg.apache.hadoop.security.AccessControlException
public org.apache.hadoop.yarn.api.records.Resource getAMResourceLimit()
public org.apache.hadoop.yarn.api.records.Resource getAMResourceLimitPerPartition(String nodePartition)
public org.apache.hadoop.yarn.api.records.Resource calculateAndGetAMResourceLimit()
public org.apache.hadoop.yarn.api.records.Resource getUserAMResourceLimit()
public org.apache.hadoop.yarn.api.records.Resource getUserAMResourceLimitPerPartition(String nodePartition, String userName)
public org.apache.hadoop.yarn.api.records.Resource calculateAndGetAMResourceLimitPerPartition(String nodePartition)
public void finishApplication(org.apache.hadoop.yarn.api.records.ApplicationId application, String user)
CSQueue
user
- user who submitted the applicationpublic void finishApplicationAttempt(FiCaSchedulerApp application, String queue)
CSQueue
public void removeApplicationAttempt(FiCaSchedulerApp application, LeafQueue.User user)
public CSAssignment assignContainers(org.apache.hadoop.yarn.api.records.Resource clusterResource, FiCaSchedulerNode node, ResourceLimits currentResourceLimits, SchedulingMode schedulingMode)
CSQueue
clusterResource
- the resource of the cluster.node
- node on which resources are availablecurrentResourceLimits
- how much overall resource of this queue can use.schedulingMode
- Type of exclusive check when assign container on a
NodeManager, see SchedulingMode
.protected org.apache.hadoop.yarn.api.records.Resource getHeadroom(LeafQueue.User user, org.apache.hadoop.yarn.api.records.Resource queueCurrentLimit, org.apache.hadoop.yarn.api.records.Resource clusterResource, FiCaSchedulerApp application)
protected org.apache.hadoop.yarn.api.records.Resource getHeadroom(LeafQueue.User user, org.apache.hadoop.yarn.api.records.Resource queueCurrentLimit, org.apache.hadoop.yarn.api.records.Resource clusterResource, FiCaSchedulerApp application, String partition)
@Lock(value=org.apache.hadoop.yarn.server.utils.Lock.NoLock.class) public int getNodeLocalityDelay()
@Lock(value=org.apache.hadoop.yarn.server.utils.Lock.NoLock.class) public boolean getRackLocalityFullReset()
@InterfaceAudience.Private protected boolean canAssignToUser(org.apache.hadoop.yarn.api.records.Resource clusterResource, String userName, org.apache.hadoop.yarn.api.records.Resource limit, org.apache.hadoop.yarn.api.records.Resource rsrv, String nodePartition, ResourceLimits currentResourceLimits)
public void unreserveIncreasedContainer(org.apache.hadoop.yarn.api.records.Resource clusterResource, FiCaSchedulerApp app, FiCaSchedulerNode node, RMContainer rmContainer)
CSQueue
public void completedContainer(org.apache.hadoop.yarn.api.records.Resource clusterResource, FiCaSchedulerApp application, FiCaSchedulerNode node, RMContainer rmContainer, org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus, RMContainerEventType event, CSQueue childQueue, boolean sortQueues)
CSQueue
clusterResource
- the resource of the clusterapplication
- application to which the container was assignednode
- node on which the container completedrmContainer
- completed container,
null
if it was just a reservationcontainerStatus
- ContainerStatus
for the completed
containerevent
- event to be sent to the containerchildQueue
- CSQueue
to reinsert in childQueuessortQueues
- indicates whether it should re-sort the queuespublic void updateClusterResource(org.apache.hadoop.yarn.api.records.Resource clusterResource, ResourceLimits currentResourceLimits)
CSQueue
clusterResource
- the current cluster resourcecurrentResourceLimits
- the current ResourceLimitspublic void incUsedResource(String nodeLabel, org.apache.hadoop.yarn.api.records.Resource resourceToInc, SchedulerApplicationAttempt application)
CSQueue
incUsedResource
in interface CSQueue
incUsedResource
in class AbstractCSQueue
public void decUsedResource(String nodeLabel, org.apache.hadoop.yarn.api.records.Resource resourceToDec, SchedulerApplicationAttempt application)
CSQueue
decUsedResource
in interface CSQueue
decUsedResource
in class AbstractCSQueue
public void incAMUsedResource(String nodeLabel, org.apache.hadoop.yarn.api.records.Resource resourceToInc, SchedulerApplicationAttempt application)
public void decAMUsedResource(String nodeLabel, org.apache.hadoop.yarn.api.records.Resource resourceToDec, SchedulerApplicationAttempt application)
public float getUsageRatio(String label)
public void recoverContainer(org.apache.hadoop.yarn.api.records.Resource clusterResource, SchedulerApplicationAttempt attempt, RMContainer rmContainer)
Queue
clusterResource
- the resource of the clusterattempt
- the application for which the container was allocatedrmContainer
- the container that was recovered.public Collection<FiCaSchedulerApp> getPendingApplications()
public Collection<FiCaSchedulerApp> getApplications()
public Collection<FiCaSchedulerApp> getAllApplications()
public org.apache.hadoop.yarn.api.records.Resource getTotalPendingResourcesConsideringUserLimit(org.apache.hadoop.yarn.api.records.Resource resources, String partition)
public org.apache.hadoop.yarn.api.records.Resource getUserLimitPerUser(String userName, org.apache.hadoop.yarn.api.records.Resource resources, String partition)
public void collectSchedulerApplications(Collection<org.apache.hadoop.yarn.api.records.ApplicationAttemptId> apps)
CSQueue
apps
- the collection to add the applications topublic void attachContainer(org.apache.hadoop.yarn.api.records.Resource clusterResource, FiCaSchedulerApp application, RMContainer rmContainer)
CSQueue
clusterResource
- the current cluster resourceapplication
- application to which the container was assignedrmContainer
- the container to attachpublic void detachContainer(org.apache.hadoop.yarn.api.records.Resource clusterResource, FiCaSchedulerApp application, RMContainer rmContainer)
CSQueue
clusterResource
- the current cluster resourceapplication
- application to which the container was assignedrmContainer
- the container to detachpublic Map<String,TreeSet<RMContainer>> getIgnoreExclusivityRMContainers()
public void setCapacity(float capacity)
public void setAbsoluteCapacity(float absoluteCapacity)
public void setMaxApplications(int maxApplications)
public OrderingPolicy<FiCaSchedulerApp> getOrderingPolicy()
public void setOrderingPolicy(OrderingPolicy<FiCaSchedulerApp> orderingPolicy)
public org.apache.hadoop.yarn.api.records.Priority getDefaultApplicationPriority()
Queue
getDefaultApplicationPriority
in interface Queue
getDefaultApplicationPriority
in class AbstractCSQueue
public void decreaseContainer(org.apache.hadoop.yarn.api.records.Resource clusterResource, SchedContainerChangeRequest decreaseRequest, FiCaSchedulerApp app) throws org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException
CSQueue
clusterResource
- Total cluster resourcedecreaseRequest
- The decrease requestapp
- The application of interestorg.apache.hadoop.yarn.exceptions.InvalidResourceRequestException
public OrderingPolicy<FiCaSchedulerApp> getPendingAppsOrderingPolicy()
public org.apache.hadoop.yarn.api.records.Resource getResourceLimitForAllUsers(String userName, org.apache.hadoop.yarn.api.records.Resource clusterResource, String partition, SchedulingMode schedulingMode)
Copyright © 2017 Apache Software Foundation. All Rights Reserved.