org.apache.hadoop.yarn.nodelabels

Class CommonNodeLabelsManager

java.lang.Object

org.apache.hadoop.service.AbstractService

org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager

All Implemented Interfaces:

Closeable, AutoCloseable, org.apache.hadoop.service.Service

@InterfaceAudience.Private
public class CommonNodeLabelsManager
extends org.apache.hadoop.service.AbstractService

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	CommonNodeLabelsManager.Host A Host can have multiple Nodes
protected static class	CommonNodeLabelsManager.Node

Nested classes/interfaces inherited from interface org.apache.hadoop.service.Service

org.apache.hadoop.service.Service.STATE

Field Summary

Fields

Modifier and Type	Field and Description
static Set<String>	ACCESS_ANY_LABEL_SET
static String	ANY
protected Dispatcher	dispatcher
static Set<org.apache.hadoop.yarn.api.records.NodeLabel>	EMPTY_NODELABEL_SET
static Set<String>	EMPTY_STRING_SET
protected ConcurrentMap<String,RMNodeLabel>	labelCollections
protected static org.slf4j.Logger	LOG
static String	NO_LABEL If a user doesn't specify label of a queue or node, it belongs DEFAULT_LABEL
static String	NODE_LABELS_NOT_ENABLED_ERR Error messages
protected ConcurrentMap<String,CommonNodeLabelsManager.Host>	nodeCollections
protected RMNodeLabel	noNodeLabel
protected ReentrantReadWriteLock.ReadLock	readLock
protected NodeLabelsStore	store
static int	WILDCARD_PORT
protected ReentrantReadWriteLock.WriteLock	writeLock

Constructor Summary

Constructors

Constructor and Description
CommonNodeLabelsManager()

Method Summary

Modifier and Type	Method and Description
void	addLabelsToNode(Map<org.apache.hadoop.yarn.api.records.NodeId,Set<String>> addedLabelsToNode) add more labels to nodes.
void	addToCluserNodeLabels(Collection<org.apache.hadoop.yarn.api.records.NodeLabel> labels)
void	addToCluserNodeLabelsWithDefaultExclusivity(Set<String> labels) Add multiple node labels to repository.
protected void	checkAddLabelsToNode(Map<org.apache.hadoop.yarn.api.records.NodeId,Set<String>> addedLabelsToNode)
protected void	checkRemoveFromClusterNodeLabels(Collection<String> labelsToRemove)
protected void	checkRemoveLabelsFromNode(Map<org.apache.hadoop.yarn.api.records.NodeId,Set<String>> removeLabelsFromNode)
protected void	checkReplaceLabelsOnNode(Map<org.apache.hadoop.yarn.api.records.NodeId,Set<String>> replaceLabelsToNode)
protected void	createHostIfNonExisted(String hostName)
protected void	createNodeIfNonExisted(org.apache.hadoop.yarn.api.records.NodeId nodeId)
Set<String>	getClusterNodeLabelNames() Get existing valid labels in repository
List<org.apache.hadoop.yarn.api.records.NodeLabel>	getClusterNodeLabels()
protected Set<String>	getLabelsByNode(org.apache.hadoop.yarn.api.records.NodeId nodeId)
protected Set<String>	getLabelsByNode(org.apache.hadoop.yarn.api.records.NodeId nodeId, Map<String,CommonNodeLabelsManager.Host> map)
Set<org.apache.hadoop.yarn.api.records.NodeLabel>	getLabelsInfoByNode(org.apache.hadoop.yarn.api.records.NodeId nodeId)
Map<org.apache.hadoop.yarn.api.records.NodeLabel,Set<org.apache.hadoop.yarn.api.records.NodeId>>	getLabelsInfoToNodes() Get mapping of labels to nodes for all the labels.
Map<org.apache.hadoop.yarn.api.records.NodeLabel,Set<org.apache.hadoop.yarn.api.records.NodeId>>	getLabelsInfoToNodes(Set<String> labels) Get mapping of labels info to nodes for specified set of labels.
Map<String,Set<org.apache.hadoop.yarn.api.records.NodeId>>	getLabelsToNodes() Get mapping of labels to nodes for all the labels.
Map<String,Set<org.apache.hadoop.yarn.api.records.NodeId>>	getLabelsToNodes(Set<String> labels) Get mapping of labels to nodes for specified set of labels.
protected CommonNodeLabelsManager.Node	getNMInNodeSet(org.apache.hadoop.yarn.api.records.NodeId nodeId)
protected CommonNodeLabelsManager.Node	getNMInNodeSet(org.apache.hadoop.yarn.api.records.NodeId nodeId, Map<String,CommonNodeLabelsManager.Host> map)
protected CommonNodeLabelsManager.Node	getNMInNodeSet(org.apache.hadoop.yarn.api.records.NodeId nodeId, Map<String,CommonNodeLabelsManager.Host> map, boolean checkRunning)
Map<org.apache.hadoop.yarn.api.records.NodeId,Set<String>>	getNodeLabels() Get mapping of nodes to labels
Map<org.apache.hadoop.yarn.api.records.NodeId,Set<org.apache.hadoop.yarn.api.records.NodeLabel>>	getNodeLabelsInfo() Get mapping of nodes to label info
Set<org.apache.hadoop.yarn.api.records.NodeId>	getNodesWithoutALabel() Get nodes that have no labels.
protected void	handleStoreEvent(NodeLabelsStoreEvent event)
protected void	initDispatcher(org.apache.hadoop.conf.Configuration conf)
protected void	initNodeLabelStore(org.apache.hadoop.conf.Configuration conf)
protected void	internalRemoveFromClusterNodeLabels(Collection<String> labelsToRemove)
protected void	internalUpdateLabelsOnNodes(Map<org.apache.hadoop.yarn.api.records.NodeId,Set<String>> nodeToLabels, org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager.NodeLabelUpdateOperation op)
boolean	isCentralizedConfiguration()
boolean	isExclusiveNodeLabel(String nodeLabel)
protected boolean	isInitNodeLabelStoreInProgress()
protected boolean	isNodeLabelExplicit(org.apache.hadoop.yarn.api.records.NodeId nodeId)
protected String	normalizeLabel(String label)
protected Map<org.apache.hadoop.yarn.api.records.NodeId,Set<String>>	normalizeNodeIdToLabels(Map<org.apache.hadoop.yarn.api.records.NodeId,Set<String>> nodeIdToLabels)
void	removeFromClusterNodeLabels(Collection<String> labelsToRemove) Remove multiple node labels from repository
void	removeLabelsFromNode(Map<org.apache.hadoop.yarn.api.records.NodeId,Set<String>> removeLabelsFromNode) remove labels from nodes, labels being removed most be contained by these nodes.
protected void	removeLabelsFromNodeInHost(org.apache.hadoop.yarn.api.records.NodeId node, Set<String> labels)
protected void	removeNodeFromLabels(org.apache.hadoop.yarn.api.records.NodeId node, Set<String> labels)
void	replaceLabelsOnNode(Map<org.apache.hadoop.yarn.api.records.NodeId,Set<String>> replaceLabelsToNode) replace labels to nodes
protected void	serviceInit(org.apache.hadoop.conf.Configuration conf)
protected void	serviceStart()
protected void	serviceStop()
void	setInitNodeLabelStoreInProgress(boolean initNodeLabelStoreInProgress)
protected void	startDispatcher()
protected void	stopDispatcher()

Methods inherited from class org.apache.hadoop.service.AbstractService

close, getBlockers, getConfig, getFailureCause, getFailureState, getLifecycleHistory, getName, getServiceState, getStartTime, init, isInState, noteFailure, putBlocker, registerGlobalListener, registerServiceListener, removeBlocker, setConfig, start, stop, toString, unregisterGlobalListener, unregisterServiceListener, waitForServiceToStop

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait

Field Detail

LOG

protected static final org.slf4j.Logger LOG

EMPTY_STRING_SET

public static final Set<String> EMPTY_STRING_SET

EMPTY_NODELABEL_SET

public static final Set<org.apache.hadoop.yarn.api.records.NodeLabel> EMPTY_NODELABEL_SET

ANY

public static final String ANY

See Also:

Constant Field Values

ACCESS_ANY_LABEL_SET

public static final Set<String> ACCESS_ANY_LABEL_SET

WILDCARD_PORT

public static final int WILDCARD_PORT

See Also:

Constant Field Values

NODE_LABELS_NOT_ENABLED_ERR

@VisibleForTesting
public static final String NODE_LABELS_NOT_ENABLED_ERR

Error messages

See Also:

Constant Field Values

NO_LABEL

public static final String NO_LABEL

If a user doesn't specify label of a queue or node, it belongs DEFAULT_LABEL

See Also:

Constant Field Values

dispatcher

protected Dispatcher dispatcher

labelCollections

protected ConcurrentMap<String,RMNodeLabel> labelCollections

nodeCollections

protected ConcurrentMap<String,CommonNodeLabelsManager.Host> nodeCollections

noNodeLabel

protected RMNodeLabel noNodeLabel

readLock

protected final ReentrantReadWriteLock.ReadLock readLock

writeLock

protected final ReentrantReadWriteLock.WriteLock writeLock

store

protected NodeLabelsStore store

Constructor Detail

CommonNodeLabelsManager

public CommonNodeLabelsManager()

Method Detail

handleStoreEvent

protected void handleStoreEvent(NodeLabelsStoreEvent event)

initDispatcher

protected void initDispatcher(org.apache.hadoop.conf.Configuration conf)

serviceInit

protected void serviceInit(org.apache.hadoop.conf.Configuration conf)
                    throws Exception

Overrides:

serviceInit in class org.apache.hadoop.service.AbstractService

Throws:

Exception

isInitNodeLabelStoreInProgress

protected boolean isInitNodeLabelStoreInProgress()

Returns:

the isStartup

isCentralizedConfiguration

public boolean isCentralizedConfiguration()

Returns:

true if node label configuration type is not distributed.

initNodeLabelStore

protected void initNodeLabelStore(org.apache.hadoop.conf.Configuration conf)
                           throws Exception

Throws:

Exception

startDispatcher

protected void startDispatcher()

serviceStart

protected void serviceStart()
                     throws Exception

Overrides:

serviceStart in class org.apache.hadoop.service.AbstractService

Throws:

Exception

stopDispatcher

protected void stopDispatcher()

serviceStop

protected void serviceStop()
                    throws Exception

Overrides:

serviceStop in class org.apache.hadoop.service.AbstractService

Throws:

Exception

addToCluserNodeLabels

public void addToCluserNodeLabels(Collection<org.apache.hadoop.yarn.api.records.NodeLabel> labels)
                           throws IOException

Throws:

IOException

addToCluserNodeLabelsWithDefaultExclusivity

@VisibleForTesting
public void addToCluserNodeLabelsWithDefaultExclusivity(Set<String> labels)
                                                                    throws IOException

Add multiple node labels to repository.

Parameters:

labels - new node labels added

Throws:

IOException - io error occur.

checkAddLabelsToNode

protected void checkAddLabelsToNode(Map<org.apache.hadoop.yarn.api.records.NodeId,Set<String>> addedLabelsToNode)
                             throws IOException

Throws:

IOException

addLabelsToNode

public void addLabelsToNode(Map<org.apache.hadoop.yarn.api.records.NodeId,Set<String>> addedLabelsToNode)
                     throws IOException

add more labels to nodes.

Parameters:

addedLabelsToNode - node -> labels map

Throws:

IOException - io error occur.

checkRemoveFromClusterNodeLabels

protected void checkRemoveFromClusterNodeLabels(Collection<String> labelsToRemove)
                                         throws IOException

Throws:

IOException

internalRemoveFromClusterNodeLabels

protected void internalRemoveFromClusterNodeLabels(Collection<String> labelsToRemove)

removeFromClusterNodeLabels

public void removeFromClusterNodeLabels(Collection<String> labelsToRemove)
                                 throws IOException

Remove multiple node labels from repository

Parameters:

labelsToRemove - node labels to remove

Throws:

IOException - io error occur.

checkRemoveLabelsFromNode

protected void checkRemoveLabelsFromNode(Map<org.apache.hadoop.yarn.api.records.NodeId,Set<String>> removeLabelsFromNode)
                                  throws IOException

Throws:

IOException

removeNodeFromLabels

protected void removeNodeFromLabels(org.apache.hadoop.yarn.api.records.NodeId node,
                                    Set<String> labels)

removeLabelsFromNodeInHost

protected void removeLabelsFromNodeInHost(org.apache.hadoop.yarn.api.records.NodeId node,
                                          Set<String> labels)
                                   throws IOException

Throws:

IOException

isNodeLabelExplicit

protected boolean isNodeLabelExplicit(org.apache.hadoop.yarn.api.records.NodeId nodeId)

internalUpdateLabelsOnNodes

protected void internalUpdateLabelsOnNodes(Map<org.apache.hadoop.yarn.api.records.NodeId,Set<String>> nodeToLabels,
                                           org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager.NodeLabelUpdateOperation op)
                                    throws IOException

Throws:

IOException

removeLabelsFromNode

public void removeLabelsFromNode(Map<org.apache.hadoop.yarn.api.records.NodeId,Set<String>> removeLabelsFromNode)
                          throws IOException

remove labels from nodes, labels being removed most be contained by these nodes.

Parameters:

removeLabelsFromNode - node -> labels map

Throws:

IOException - io error occur.

checkReplaceLabelsOnNode

protected void checkReplaceLabelsOnNode(Map<org.apache.hadoop.yarn.api.records.NodeId,Set<String>> replaceLabelsToNode)
                                 throws IOException

Throws:

IOException

replaceLabelsOnNode

public void replaceLabelsOnNode(Map<org.apache.hadoop.yarn.api.records.NodeId,Set<String>> replaceLabelsToNode)
                         throws IOException

replace labels to nodes

Parameters:

replaceLabelsToNode - node -> labels map

Throws:

IOException - io error occur.

getNodeLabels

public Map<org.apache.hadoop.yarn.api.records.NodeId,Set<String>> getNodeLabels()

Get mapping of nodes to labels

Returns:

nodes to labels map

getNodeLabelsInfo

public Map<org.apache.hadoop.yarn.api.records.NodeId,Set<org.apache.hadoop.yarn.api.records.NodeLabel>> getNodeLabelsInfo()

Get mapping of nodes to label info

Returns:

nodes to labels map

getNodesWithoutALabel

public Set<org.apache.hadoop.yarn.api.records.NodeId> getNodesWithoutALabel()

Get nodes that have no labels.

Returns:

set of nodes with no labels

getLabelsToNodes

public Map<String,Set<org.apache.hadoop.yarn.api.records.NodeId>> getLabelsToNodes()

Get mapping of labels to nodes for all the labels.

Returns:

labels to nodes map

getLabelsToNodes

public Map<String,Set<org.apache.hadoop.yarn.api.records.NodeId>> getLabelsToNodes(Set<String> labels)

Get mapping of labels to nodes for specified set of labels.

Parameters:

labels - set of labels for which labels to nodes mapping will be returned.

Returns:

labels to nodes map

getLabelsInfoToNodes

public Map<org.apache.hadoop.yarn.api.records.NodeLabel,Set<org.apache.hadoop.yarn.api.records.NodeId>> getLabelsInfoToNodes()

Get mapping of labels to nodes for all the labels.

Returns:

labels to nodes map

getLabelsInfoToNodes

public Map<org.apache.hadoop.yarn.api.records.NodeLabel,Set<org.apache.hadoop.yarn.api.records.NodeId>> getLabelsInfoToNodes(Set<String> labels)

Get mapping of labels info to nodes for specified set of labels.

Parameters:

labels - set of nodelabels for which labels to nodes mapping will be returned.

Returns:

labels to nodes map

getClusterNodeLabelNames

public Set<String> getClusterNodeLabelNames()

Get existing valid labels in repository

Returns:

existing valid labels in repository

getClusterNodeLabels

public List<org.apache.hadoop.yarn.api.records.NodeLabel> getClusterNodeLabels()

isExclusiveNodeLabel

public boolean isExclusiveNodeLabel(String nodeLabel)
                             throws IOException

Throws:

IOException

normalizeLabel

protected String normalizeLabel(String label)

getNMInNodeSet

protected CommonNodeLabelsManager.Node getNMInNodeSet(org.apache.hadoop.yarn.api.records.NodeId nodeId)

getNMInNodeSet

protected CommonNodeLabelsManager.Node getNMInNodeSet(org.apache.hadoop.yarn.api.records.NodeId nodeId,
                                                      Map<String,CommonNodeLabelsManager.Host> map)

getNMInNodeSet

protected CommonNodeLabelsManager.Node getNMInNodeSet(org.apache.hadoop.yarn.api.records.NodeId nodeId,
                                                      Map<String,CommonNodeLabelsManager.Host> map,
                                                      boolean checkRunning)

getLabelsByNode

protected Set<String> getLabelsByNode(org.apache.hadoop.yarn.api.records.NodeId nodeId)

getLabelsByNode

protected Set<String> getLabelsByNode(org.apache.hadoop.yarn.api.records.NodeId nodeId,
                                      Map<String,CommonNodeLabelsManager.Host> map)

getLabelsInfoByNode

public Set<org.apache.hadoop.yarn.api.records.NodeLabel> getLabelsInfoByNode(org.apache.hadoop.yarn.api.records.NodeId nodeId)

createNodeIfNonExisted

protected void createNodeIfNonExisted(org.apache.hadoop.yarn.api.records.NodeId nodeId)
                               throws IOException

Throws:

IOException

createHostIfNonExisted

protected void createHostIfNonExisted(String hostName)

normalizeNodeIdToLabels

protected Map<org.apache.hadoop.yarn.api.records.NodeId,Set<String>> normalizeNodeIdToLabels(Map<org.apache.hadoop.yarn.api.records.NodeId,Set<String>> nodeIdToLabels)

setInitNodeLabelStoreInProgress

public void setInitNodeLabelStoreInProgress(boolean initNodeLabelStoreInProgress)