public class FederationInterceptor extends AbstractRequestInterceptor
Modifier and Type | Field and Description |
---|---|
static String |
NMSS_CLASS_PREFIX |
static String |
NMSS_REG_REQUEST_KEY |
static String |
NMSS_REG_RESPONSE_KEY |
static String |
NMSS_SECONDARY_SC_PREFIX
When AMRMProxy HA is enabled, secondary AMRMTokens will be stored in Yarn
Registry.
|
static String |
STRING_TO_BYTE_FORMAT |
Constructor and Description |
---|
FederationInterceptor()
Creates an instance of the FederationInterceptor class.
|
Modifier and Type | Method and Description |
---|---|
org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse |
allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest request)
Sends the heart beats to the home RM and the secondary sub-cluster RMs that
are being used by the application.
|
protected void |
cleanupRegistry()
Only for unit test cleanup.
|
protected org.apache.hadoop.yarn.server.AMHeartbeatRequestHandler |
createHomeHeartbeartHandler(org.apache.hadoop.conf.Configuration conf,
org.apache.hadoop.yarn.api.records.ApplicationId appId,
org.apache.hadoop.yarn.server.AMRMClientRelayer rmProxyRelayer) |
protected <T> T |
createHomeRMProxy(AMRMProxyApplicationContext appContext,
Class<T> protocol,
org.apache.hadoop.security.UserGroupInformation user)
Create a proxy instance that is used to connect to the Home resource
manager.
|
protected org.apache.hadoop.yarn.server.uam.UnmanagedAMPoolManager |
createUnmanagedAMPoolManager(ExecutorService threadPool)
Create the UAM pool manager for secondary sub-clsuters.
|
org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse |
finishApplicationMaster(org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest request)
Sends the finish application master request to all the resource managers
used by the application.
|
protected org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse |
generateBaseAllocationResponse()
Prepare the base allocation response.
|
Map<org.apache.hadoop.yarn.server.federation.store.records.SubClusterId,List<org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse>> |
getAsyncResponseSink() |
protected org.apache.hadoop.yarn.api.records.ApplicationAttemptId |
getAttemptId() |
protected org.apache.hadoop.yarn.server.AMHeartbeatRequestHandler |
getHomeHeartbeartHandler() |
protected org.apache.hadoop.yarn.server.federation.utils.FederationRegistryClient |
getRegistryClient() |
protected Set<org.apache.hadoop.yarn.server.federation.store.records.SubClusterId> |
getTimedOutSCs(boolean verbose) |
protected Map<org.apache.hadoop.yarn.server.federation.store.records.SubClusterId,Future<?>> |
getUamRegisterFutures() |
protected org.apache.hadoop.yarn.server.uam.UnmanagedAMPoolManager |
getUnmanagedAMPool() |
protected int |
getUnmanagedAMPoolSize() |
void |
init(AMRMProxyApplicationContext appContext)
Initializes the instance using specified context.
|
static <T> boolean |
isNullOrEmpty(Collection<T> c)
Utility method to check if the specified Collection is null or empty.
|
static <T1,T2> boolean |
isNullOrEmpty(Map<T1,T2> c)
Utility method to check if the specified Collection is null or empty.
|
protected void |
mergeAllocateResponse(org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse homeResponse,
org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse otherResponse,
org.apache.hadoop.yarn.server.federation.store.records.SubClusterId otherRMAddress) |
protected void |
reAttachUAMAndMergeRegisterResponse(org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse homeResponse,
org.apache.hadoop.yarn.api.records.ApplicationId appId)
Try re-attach to all existing and running UAMs in secondary sub-clusters
launched by previous application attempts if any.
|
void |
recover(Map<String,byte[]> recoveredDataMap)
Recover
RequestInterceptor state from store. |
org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse |
registerApplicationMaster(org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest request)
Sends the application master's registration request to the home RM.
|
void |
setNextInterceptor(RequestInterceptor next)
Sets the
RequestInterceptor in the chain. |
void |
shutdown()
This is called when the application pipeline is being destroyed.
|
protected Map<org.apache.hadoop.yarn.server.federation.store.records.SubClusterId,List<org.apache.hadoop.yarn.api.records.ResourceRequest>> |
splitResourceRequests(List<org.apache.hadoop.yarn.api.records.ResourceRequest> askList)
Splits the specified request to send it to different sub clusters.
|
allocateForDistributedScheduling, getApplicationContext, getConf, getNextInterceptor, getNMStateStore, registerApplicationMasterForDistributedScheduling, setConf
public static final String NMSS_CLASS_PREFIX
public static final String NMSS_REG_REQUEST_KEY
public static final String NMSS_REG_RESPONSE_KEY
public static final String NMSS_SECONDARY_SC_PREFIX
public static final String STRING_TO_BYTE_FORMAT
public FederationInterceptor()
public void init(AMRMProxyApplicationContext appContext)
init
in interface RequestInterceptor
init
in class AbstractRequestInterceptor
appContext
- AMRMProxy application contextpublic void recover(Map<String,byte[]> recoveredDataMap)
AbstractRequestInterceptor
RequestInterceptor
state from store.recover
in interface RequestInterceptor
recover
in class AbstractRequestInterceptor
recoveredDataMap
- states for all intercepters recovered from NMSSpublic org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse registerApplicationMaster(org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
InvalidApplicationMasterRequestException
. Furthermore, we present
to AM as if we are the RM that never fails over (except when AMRMProxy
restarts). When actual RM fails over, we always re-register automatically.
We did this because FederationInterceptor can receive concurrent register
requests from AM because of timeout between AM and AMRMProxy, which is
shorter than the timeout + failOver between FederationInterceptor
(AMRMProxy) and RM.
For the same reason, this method needs to be synchronized.org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse finishApplicationMaster(org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
public void setNextInterceptor(RequestInterceptor next)
AbstractRequestInterceptor
RequestInterceptor
in the chain.setNextInterceptor
in interface RequestInterceptor
setNextInterceptor
in class AbstractRequestInterceptor
next
- the next intercepter to setpublic void shutdown()
shutdown
in interface RequestInterceptor
shutdown
in class AbstractRequestInterceptor
protected void cleanupRegistry()
protected org.apache.hadoop.yarn.server.federation.utils.FederationRegistryClient getRegistryClient()
protected org.apache.hadoop.yarn.api.records.ApplicationAttemptId getAttemptId()
protected org.apache.hadoop.yarn.server.AMHeartbeatRequestHandler getHomeHeartbeartHandler()
protected org.apache.hadoop.yarn.server.uam.UnmanagedAMPoolManager createUnmanagedAMPoolManager(ExecutorService threadPool)
threadPool
- the thread pool to useprotected org.apache.hadoop.yarn.server.AMHeartbeatRequestHandler createHomeHeartbeartHandler(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.yarn.api.records.ApplicationId appId, org.apache.hadoop.yarn.server.AMRMClientRelayer rmProxyRelayer)
protected <T> T createHomeRMProxy(AMRMProxyApplicationContext appContext, Class<T> protocol, org.apache.hadoop.security.UserGroupInformation user)
T
- the type of the proxyappContext
- AMRMProxyApplicationContextprotocol
- the protocol class for the proxyuser
- the ugi for the proxyprotected void reAttachUAMAndMergeRegisterResponse(org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse homeResponse, org.apache.hadoop.yarn.api.records.ApplicationId appId)
protected org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse generateBaseAllocationResponse()
protected void mergeAllocateResponse(org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse homeResponse, org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse otherResponse, org.apache.hadoop.yarn.server.federation.store.records.SubClusterId otherRMAddress)
protected Set<org.apache.hadoop.yarn.server.federation.store.records.SubClusterId> getTimedOutSCs(boolean verbose)
protected Map<org.apache.hadoop.yarn.server.federation.store.records.SubClusterId,List<org.apache.hadoop.yarn.api.records.ResourceRequest>> splitResourceRequests(List<org.apache.hadoop.yarn.api.records.ResourceRequest> askList) throws org.apache.hadoop.yarn.exceptions.YarnException
askList
- the ask list to splitorg.apache.hadoop.yarn.exceptions.YarnException
- if split failsprotected int getUnmanagedAMPoolSize()
protected org.apache.hadoop.yarn.server.uam.UnmanagedAMPoolManager getUnmanagedAMPool()
protected Map<org.apache.hadoop.yarn.server.federation.store.records.SubClusterId,Future<?>> getUamRegisterFutures()
public Map<org.apache.hadoop.yarn.server.federation.store.records.SubClusterId,List<org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse>> getAsyncResponseSink()
public static <T> boolean isNullOrEmpty(Collection<T> c)
T
- element type of the collectionc
- the collection objectpublic static <T1,T2> boolean isNullOrEmpty(Map<T1,T2> c)
T1
- key type of the mapT2
- value type of the mapc
- the map objectCopyright © 2008–2023 Apache Software Foundation. All rights reserved.