public abstract class ContainerExecutor extends Object implements org.apache.hadoop.conf.Configurable
Modifier and Type | Class and Description |
---|---|
static class |
ContainerExecutor.DelayedProcessKiller
This class will signal a target container after a specified delay.
|
static class |
ContainerExecutor.ExitCode
The container exit code.
|
static class |
ContainerExecutor.Signal
The constants for the signals.
|
Modifier and Type | Field and Description |
---|---|
static String |
DIRECTORY_CONTENTS
The relative path to which debug information will be written.
|
static org.apache.hadoop.fs.permission.FsPermission |
TASK_LAUNCH_SCRIPT_PERMISSION
The permissions to use when creating the launch script.
|
static String |
TOKEN_FILE_NAME_FMT |
protected static String |
WILDCARD |
Constructor and Description |
---|
ContainerExecutor() |
Modifier and Type | Method and Description |
---|---|
void |
activateContainer(org.apache.hadoop.yarn.api.records.ContainerId containerId,
org.apache.hadoop.fs.Path pidFilePath)
Mark the container as active.
|
void |
cleanupBeforeRelaunch(Container container)
Perform any cleanup before the next launch of the container.
|
void |
deactivateContainer(org.apache.hadoop.yarn.api.records.ContainerId containerId)
Mark the container as inactive.
|
abstract void |
deleteAsUser(DeletionAsUserContext ctx)
Delete specified directories as a given user.
|
abstract org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair |
execContainer(ContainerExecContext ctx)
Perform interactive docker command into running container.
|
org.apache.hadoop.conf.Configuration |
getConf() |
String |
getExposedPorts(Container container) |
String[] |
getIpAndHost(Container container) |
static String[] |
getLocalIpAndHost(Container container) |
Map<String,org.apache.hadoop.yarn.api.records.LocalResource> |
getLocalResources(Container container) |
protected String |
getNMEnvVar(String varname) |
protected org.apache.hadoop.fs.Path |
getPidFilePath(org.apache.hadoop.yarn.api.records.ContainerId containerId)
Get the pidFile of the container.
|
String |
getProcessId(org.apache.hadoop.yarn.api.records.ContainerId containerID)
Get the process-identifier for the container.
|
protected String[] |
getRunCommand(String command,
String groupId,
String userName,
org.apache.hadoop.fs.Path pidFile,
org.apache.hadoop.conf.Configuration config)
Return a command line to execute the given command in the OS shell.
|
protected String[] |
getRunCommand(String command,
String groupId,
String userName,
org.apache.hadoop.fs.Path pidFile,
org.apache.hadoop.conf.Configuration config,
org.apache.hadoop.yarn.api.records.Resource resource)
Return a command line to execute the given command in the OS shell.
|
protected String[] |
getRunCommandForOther(String command,
org.apache.hadoop.conf.Configuration config)
Return a command line to execute the given command in the OS shell.
|
protected String[] |
getRunCommandForWindows(String command,
String groupId,
String userName,
org.apache.hadoop.fs.Path pidFile,
org.apache.hadoop.conf.Configuration config,
org.apache.hadoop.yarn.api.records.Resource resource)
Return a command line to execute the given command in the OS shell.
|
abstract void |
init(Context nmContext)
Run the executor initialization steps.
|
protected boolean |
isContainerActive(org.apache.hadoop.yarn.api.records.ContainerId containerId)
Return whether the container is still active.
|
abstract boolean |
isContainerAlive(ContainerLivenessContext ctx)
Check if a container is alive.
|
abstract int |
launchContainer(ContainerStartContext ctx)
Launch the container on the node.
|
org.apache.hadoop.fs.Path |
localizeClasspathJar(org.apache.hadoop.fs.Path jarPath,
org.apache.hadoop.fs.Path target,
String owner)
This function localizes the JAR file on-demand.
|
protected void |
logOutput(String output)
Log each line of the output string as INFO level log messages.
|
void |
pauseContainer(Container container)
Pause the container.
|
void |
prepareContainer(ContainerPrepareContext ctx)
Prepare the container prior to the launch environment being written.
|
int |
reacquireContainer(ContainerReacquisitionContext ctx)
Recover an already existing container.
|
protected File[] |
readDirAsUser(String user,
org.apache.hadoop.fs.Path dir)
Return the files in the target directory.
|
abstract boolean |
reapContainer(ContainerReapContext ctx)
Perform the steps necessary to reap the container.
|
abstract int |
relaunchContainer(ContainerStartContext ctx)
Relaunch the container on the node.
|
void |
resumeContainer(Container container)
Resume the container from pause state.
|
void |
setConf(org.apache.hadoop.conf.Configuration conf) |
abstract boolean |
signalContainer(ContainerSignalContext ctx)
Signal container with the specified signal.
|
void |
start() |
abstract void |
startLocalizer(LocalizerStartContext ctx)
Prepare the environment for containers in this application to execute.
|
void |
stop() |
abstract void |
symLink(String target,
String symlink)
Create a symlink file which points to the target.
|
abstract void |
updateYarnSysFS(Context ctx,
String user,
String appId,
String spec)
Update cluster information inside container.
|
void |
writeLaunchEnv(OutputStream out,
Map<String,String> environment,
Map<org.apache.hadoop.fs.Path,List<String>> resources,
List<String> command,
org.apache.hadoop.fs.Path logDir,
String user,
LinkedHashSet<String> nmVars)
This method writes out the launch environment of a container to the
default container launch script.
|
void |
writeLaunchEnv(OutputStream out,
Map<String,String> environment,
Map<org.apache.hadoop.fs.Path,List<String>> resources,
List<String> command,
org.apache.hadoop.fs.Path logDir,
String user,
String outFilename,
LinkedHashSet<String> nmVars)
This method writes out the launch environment of a container to a specified
path.
|
protected static final String WILDCARD
public static final String TOKEN_FILE_NAME_FMT
public static final org.apache.hadoop.fs.permission.FsPermission TASK_LAUNCH_SCRIPT_PERMISSION
public static final String DIRECTORY_CONTENTS
public void setConf(org.apache.hadoop.conf.Configuration conf)
setConf
in interface org.apache.hadoop.conf.Configurable
public org.apache.hadoop.conf.Configuration getConf()
getConf
in interface org.apache.hadoop.conf.Configurable
public abstract void init(Context nmContext) throws IOException
nmContext
- Context of NMIOException
- if initialization failspublic void start()
public void stop()
public org.apache.hadoop.fs.Path localizeClasspathJar(org.apache.hadoop.fs.Path jarPath, org.apache.hadoop.fs.Path target, String owner) throws IOException
jarPath
- the path to the JAR to localizetarget
- the directory where the JAR file should be localizedowner
- the name of the user who should own the localized fileIOException
- if localization failspublic abstract void startLocalizer(LocalizerStartContext ctx) throws IOException, InterruptedException
For $x in local.dirs create $x/$user/$appId Copy $nmLocal/appTokens -> $N/$user/$appId For $rsrc in private resources Copy $rsrc -> $N/$user/filecache/[idef] For $rsrc in job resources Copy $rsrc -> $N/$user/$appId/filecache/idef
ctx
- LocalizerStartContext that encapsulates necessary information
for starting a localizer.IOException
- for most application init failuresInterruptedException
- if application init thread is halted by NMpublic void prepareContainer(ContainerPrepareContext ctx) throws IOException
ctx
- Encapsulates information necessary for launching containers.IOException
- if errors occur during container preparationpublic abstract int launchContainer(ContainerStartContext ctx) throws IOException, org.apache.hadoop.yarn.exceptions.ConfigurationException
ctx
- Encapsulates information necessary for launching containers.IOException
- if the container launch failsorg.apache.hadoop.yarn.exceptions.ConfigurationException
- if config error was foundpublic abstract int relaunchContainer(ContainerStartContext ctx) throws IOException, org.apache.hadoop.yarn.exceptions.ConfigurationException
ctx
- Encapsulates information necessary for relaunching containers.IOException
- if the container relaunch failsorg.apache.hadoop.yarn.exceptions.ConfigurationException
- if config error was foundpublic abstract boolean signalContainer(ContainerSignalContext ctx) throws IOException
ctx
- Encapsulates information necessary for signaling containers.IOException
- if signaling the container failspublic abstract boolean reapContainer(ContainerReapContext ctx) throws IOException
ctx
- Encapsulates information necessary for reaping containers.IOException
- if reaping the container fails.public abstract org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair execContainer(ContainerExecContext ctx) throws ContainerExecutionException
ctx
- Encapsulates information necessary for exec containers.ContainerExecutionException
- if container exec fails.public abstract void deleteAsUser(DeletionAsUserContext ctx) throws IOException, InterruptedException
ctx
- Encapsulates information necessary for deletion.IOException
- if delete failsInterruptedException
- if interrupted while waiting for the deletion
operation to completepublic abstract void symLink(String target, String symlink) throws IOException
target
- The target for symlinksymlink
- the symlink fileIOException
- Error when creating symlinkspublic abstract boolean isContainerAlive(ContainerLivenessContext ctx) throws IOException
ctx
- Encapsulates information necessary for container liveness check.IOException
- if there is a failure while checking the container
statuspublic Map<String,org.apache.hadoop.yarn.api.records.LocalResource> getLocalResources(Container container) throws IOException
IOException
public abstract void updateYarnSysFS(Context ctx, String user, String appId, String spec) throws IOException
ctx
- ContainerRuntimeContextuser
- Owner of applicationappId
- YARN application IDspec
- Service SpecificationIOException
- if there is a failure while writing spec to diskpublic int reacquireContainer(ContainerReacquisitionContext ctx) throws IOException, InterruptedException
ctx
- encapsulates information necessary to reacquire containerIOException
- if there is a failure while reacquiring the containerInterruptedException
- if interrupted while waiting to reacquire
the containerpublic void writeLaunchEnv(OutputStream out, Map<String,String> environment, Map<org.apache.hadoop.fs.Path,List<String>> resources, List<String> command, org.apache.hadoop.fs.Path logDir, String user, LinkedHashSet<String> nmVars) throws IOException
ContainerLaunch.CONTAINER_SCRIPT
.out
- the output stream to which the environment is written (usually
a script file which will be executed by the Launcher)environment
- the environment variables and their valuesresources
- the resources which have been localized for this
container. Symlinks will be created to these localized resourcescommand
- the command that will be runlogDir
- the log dir to which to copy debugging informationuser
- the username of the job ownernmVars
- the set of environment vars that are explicitly set by NMIOException
- if any errors happened writing to the OutputStream,
while creating symlinkspublic void writeLaunchEnv(OutputStream out, Map<String,String> environment, Map<org.apache.hadoop.fs.Path,List<String>> resources, List<String> command, org.apache.hadoop.fs.Path logDir, String user, String outFilename, LinkedHashSet<String> nmVars) throws IOException
out
- the output stream to which the environment is written (usually
a script file which will be executed by the Launcher)environment
- the environment variables and their valuesresources
- the resources which have been localized for this
container. Symlinks will be created to these localized resourcescommand
- the command that will be runlogDir
- the log dir to which to copy debugging informationuser
- the username of the job owneroutFilename
- the path to which to write the launch environmentnmVars
- the set of environment vars that are explicitly set by NMIOException
- if any errors happened writing to the OutputStream,
while creating symlinksprotected File[] readDirAsUser(String user, org.apache.hadoop.fs.Path dir)
user
- the user as whom to access the target directorydir
- the target directoryprotected void logOutput(String output)
output
- the output string to logprotected org.apache.hadoop.fs.Path getPidFilePath(org.apache.hadoop.yarn.api.records.ContainerId containerId)
containerId
- the container IDprotected String[] getRunCommand(String command, String groupId, String userName, org.apache.hadoop.fs.Path pidFile, org.apache.hadoop.conf.Configuration config)
command
- the command to executegroupId
- the job owner's GIDuserName
- the job owner's usernamepidFile
- the path to the container's PID fileconfig
- the configurationprotected String[] getRunCommand(String command, String groupId, String userName, org.apache.hadoop.fs.Path pidFile, org.apache.hadoop.conf.Configuration config, org.apache.hadoop.yarn.api.records.Resource resource)
command
- the command to executegroupId
- the job owner's GID for Windows. On other operating systems
it is ignored.userName
- the job owner's username for Windows. On other operating
systems it is ignored.pidFile
- the path to the container's PID file on Windows. On other
operating systems it is ignored.config
- the configurationresource
- on Windows this parameter controls memory and CPU limits.
If null, no limits are set. On other operating systems it is ignored.protected String[] getRunCommandForWindows(String command, String groupId, String userName, org.apache.hadoop.fs.Path pidFile, org.apache.hadoop.conf.Configuration config, org.apache.hadoop.yarn.api.records.Resource resource)
command
- the command to executegroupId
- the job owner's GIDuserName
- the job owner's usernamepidFile
- the path to the container's PID fileconfig
- the configurationresource
- this parameter controls memory and CPU limits.
If null, no limits are set.protected String[] getRunCommandForOther(String command, org.apache.hadoop.conf.Configuration config)
command
- the command to executeconfig
- the configurationprotected boolean isContainerActive(org.apache.hadoop.yarn.api.records.ContainerId containerId)
containerId
- the target container's IDpublic void activateContainer(org.apache.hadoop.yarn.api.records.ContainerId containerId, org.apache.hadoop.fs.Path pidFilePath)
containerId
- the container IDpidFilePath
- the path where the executor should write the PID
of the launched processpublic String[] getIpAndHost(Container container) throws ContainerExecutionException
ContainerExecutionException
public void deactivateContainer(org.apache.hadoop.yarn.api.records.ContainerId containerId)
containerId
- the container IDpublic void pauseContainer(Container container)
container
- the Containerpublic void resumeContainer(Container container)
container
- the Containerpublic void cleanupBeforeRelaunch(Container container) throws IOException, InterruptedException
container
- containerIOException
InterruptedException
public String getProcessId(org.apache.hadoop.yarn.api.records.ContainerId containerID)
containerID
- the container IDpublic String getExposedPorts(Container container) throws ContainerExecutionException
ContainerExecutionException
Copyright © 2008–2023 Apache Software Foundation. All rights reserved.