public abstract class ContainerExecutor extends Object implements org.apache.hadoop.conf.Configurable
Modifier and Type | Class and Description |
---|---|
static class |
ContainerExecutor.DelayedProcessKiller
This class will signal a target container after a specified delay.
|
static class |
ContainerExecutor.ExitCode
The container exit code.
|
static class |
ContainerExecutor.Signal
The constants for the signals.
|
Modifier and Type | Field and Description |
---|---|
static String |
DIRECTORY_CONTENTS
The relative path to which debug information will be written.
|
static org.apache.hadoop.fs.permission.FsPermission |
TASK_LAUNCH_SCRIPT_PERMISSION
The permissions to use when creating the launch script.
|
protected static String |
WILDCARD |
Constructor and Description |
---|
ContainerExecutor() |
Modifier and Type | Method and Description |
---|---|
void |
activateContainer(org.apache.hadoop.yarn.api.records.ContainerId containerId,
org.apache.hadoop.fs.Path pidFilePath)
Mark the container as active.
|
void |
deactivateContainer(org.apache.hadoop.yarn.api.records.ContainerId containerId)
Mark the container as inactive.
|
abstract void |
deleteAsUser(DeletionAsUserContext ctx)
Delete specified directories as a given user.
|
org.apache.hadoop.conf.Configuration |
getConf() |
String[] |
getIpAndHost(Container container) |
static String[] |
getLocalIpAndHost(Container container) |
protected String |
getNMEnvVar(String varname) |
protected org.apache.hadoop.fs.Path |
getPidFilePath(org.apache.hadoop.yarn.api.records.ContainerId containerId)
Get the pidFile of the container.
|
String |
getProcessId(org.apache.hadoop.yarn.api.records.ContainerId containerID)
Get the process-identifier for the container.
|
protected String[] |
getRunCommand(String command,
String groupId,
String userName,
org.apache.hadoop.fs.Path pidFile,
org.apache.hadoop.conf.Configuration config)
Return a command line to execute the given command in the OS shell.
|
protected String[] |
getRunCommand(String command,
String groupId,
String userName,
org.apache.hadoop.fs.Path pidFile,
org.apache.hadoop.conf.Configuration config,
org.apache.hadoop.yarn.api.records.Resource resource)
Return a command line to execute the given command in the OS shell.
|
protected String[] |
getRunCommandForOther(String command,
org.apache.hadoop.conf.Configuration config)
Return a command line to execute the given command in the OS shell.
|
protected String[] |
getRunCommandForWindows(String command,
String groupId,
String userName,
org.apache.hadoop.fs.Path pidFile,
org.apache.hadoop.conf.Configuration config,
org.apache.hadoop.yarn.api.records.Resource resource)
Return a command line to execute the given command in the OS shell.
|
abstract void |
init()
Run the executor initialization steps.
|
protected boolean |
isContainerActive(org.apache.hadoop.yarn.api.records.ContainerId containerId)
Return whether the container is still active.
|
abstract boolean |
isContainerAlive(ContainerLivenessContext ctx)
Check if a container is alive.
|
abstract int |
launchContainer(ContainerStartContext ctx)
Launch the container on the node.
|
org.apache.hadoop.fs.Path |
localizeClasspathJar(org.apache.hadoop.fs.Path jarPath,
org.apache.hadoop.fs.Path target,
String owner)
This function localizes the JAR file on-demand.
|
protected void |
logOutput(String output)
Log each line of the output string as INFO level log messages.
|
void |
pauseContainer(Container container)
Pause the container.
|
int |
reacquireContainer(ContainerReacquisitionContext ctx)
Recover an already existing container.
|
protected File[] |
readDirAsUser(String user,
org.apache.hadoop.fs.Path dir)
Return the files in the target directory.
|
void |
resumeContainer(Container container)
Resume the container from pause state.
|
void |
setConf(org.apache.hadoop.conf.Configuration conf) |
abstract boolean |
signalContainer(ContainerSignalContext ctx)
Signal container with the specified signal.
|
abstract void |
startLocalizer(LocalizerStartContext ctx)
Prepare the environment for containers in this application to execute.
|
abstract void |
symLink(String target,
String symlink)
Create a symlink file which points to the target.
|
protected void |
updateEnvForWhitelistVars(Map<String,String> env)
Propagate variables from the nodemanager's environment into the
container's environment if unspecified by the container.
|
void |
writeLaunchEnv(OutputStream out,
Map<String,String> environment,
Map<org.apache.hadoop.fs.Path,List<String>> resources,
List<String> command,
org.apache.hadoop.fs.Path logDir,
String user)
This method writes out the launch environment of a container to the
default container launch script.
|
void |
writeLaunchEnv(OutputStream out,
Map<String,String> environment,
Map<org.apache.hadoop.fs.Path,List<String>> resources,
List<String> command,
org.apache.hadoop.fs.Path logDir,
String user,
String outFilename)
This method writes out the launch environment of a container to a specified
path.
|
protected static final String WILDCARD
public static final org.apache.hadoop.fs.permission.FsPermission TASK_LAUNCH_SCRIPT_PERMISSION
public static final String DIRECTORY_CONTENTS
ContainerLaunch.ShellScriptBuilder#listDebugInformation
,
Constant Field Valuespublic void setConf(org.apache.hadoop.conf.Configuration conf)
setConf
in interface org.apache.hadoop.conf.Configurable
public org.apache.hadoop.conf.Configuration getConf()
getConf
in interface org.apache.hadoop.conf.Configurable
public abstract void init() throws IOException
IOException
- if initialization failspublic org.apache.hadoop.fs.Path localizeClasspathJar(org.apache.hadoop.fs.Path jarPath, org.apache.hadoop.fs.Path target, String owner) throws IOException
jarPath
- the path to the JAR to localizetarget
- the directory where the JAR file should be localizedowner
- the name of the user who should own the localized fileIOException
- if localization failspublic abstract void startLocalizer(LocalizerStartContext ctx) throws IOException, InterruptedException
For $x in local.dirs create $x/$user/$appId Copy $nmLocal/appTokens -> $N/$user/$appId For $rsrc in private resources Copy $rsrc -> $N/$user/filecache/[idef] For $rsrc in job resources Copy $rsrc -> $N/$user/$appId/filecache/idef
ctx
- LocalizerStartContext that encapsulates necessary information
for starting a localizer.IOException
- for most application init failuresInterruptedException
- if application init thread is halted by NMpublic abstract int launchContainer(ContainerStartContext ctx) throws IOException, org.apache.hadoop.yarn.exceptions.ConfigurationException
ctx
- Encapsulates information necessary for launching containers.IOException
- if the container launch failsorg.apache.hadoop.yarn.exceptions.ConfigurationException
- if config error was foundpublic abstract boolean signalContainer(ContainerSignalContext ctx) throws IOException
ctx
- Encapsulates information necessary for signaling containers.IOException
- if signaling the container failspublic abstract void deleteAsUser(DeletionAsUserContext ctx) throws IOException, InterruptedException
ctx
- Encapsulates information necessary for deletion.IOException
- if delete failsInterruptedException
- if interrupted while waiting for the deletion
operation to completepublic abstract void symLink(String target, String symlink) throws IOException
target
- The target for symlinksymlink
- the symlink fileIOException
- Error when creating symlinkspublic abstract boolean isContainerAlive(ContainerLivenessContext ctx) throws IOException
ctx
- Encapsulates information necessary for container liveness check.IOException
- if there is a failure while checking the container
statuspublic int reacquireContainer(ContainerReacquisitionContext ctx) throws IOException, InterruptedException
ctx
- encapsulates information necessary to reacquire containerIOException
- if there is a failure while reacquiring the containerInterruptedException
- if interrupted while waiting to reacquire
the containerpublic void writeLaunchEnv(OutputStream out, Map<String,String> environment, Map<org.apache.hadoop.fs.Path,List<String>> resources, List<String> command, org.apache.hadoop.fs.Path logDir, String user) throws IOException
ContainerLaunch.CONTAINER_SCRIPT
.out
- the output stream to which the environment is written (usually
a script file which will be executed by the Launcher)environment
- the environment variables and their valuesresources
- the resources which have been localized for this
container. Symlinks will be created to these localized resourcescommand
- the command that will be runlogDir
- the log dir to which to copy debugging informationuser
- the username of the job ownerIOException
- if any errors happened writing to the OutputStream,
while creating symlinkspublic void writeLaunchEnv(OutputStream out, Map<String,String> environment, Map<org.apache.hadoop.fs.Path,List<String>> resources, List<String> command, org.apache.hadoop.fs.Path logDir, String user, String outFilename) throws IOException
out
- the output stream to which the environment is written (usually
a script file which will be executed by the Launcher)environment
- the environment variables and their valuesresources
- the resources which have been localized for this
container. Symlinks will be created to these localized resourcescommand
- the command that will be runlogDir
- the log dir to which to copy debugging informationuser
- the username of the job owneroutFilename
- the path to which to write the launch environmentIOException
- if any errors happened writing to the OutputStream,
while creating symlinksprotected File[] readDirAsUser(String user, org.apache.hadoop.fs.Path dir)
user
- the user as whom to access the target directorydir
- the target directoryprotected void logOutput(String output)
output
- the output string to logprotected org.apache.hadoop.fs.Path getPidFilePath(org.apache.hadoop.yarn.api.records.ContainerId containerId)
containerId
- the container IDprotected String[] getRunCommand(String command, String groupId, String userName, org.apache.hadoop.fs.Path pidFile, org.apache.hadoop.conf.Configuration config)
command
- the command to executegroupId
- the job owner's GIDuserName
- the job owner's usernamepidFile
- the path to the container's PID fileconfig
- the configurationprotected String[] getRunCommand(String command, String groupId, String userName, org.apache.hadoop.fs.Path pidFile, org.apache.hadoop.conf.Configuration config, org.apache.hadoop.yarn.api.records.Resource resource)
command
- the command to executegroupId
- the job owner's GID for Windows. On other operating systems
it is ignored.userName
- the job owner's username for Windows. On other operating
systems it is ignored.pidFile
- the path to the container's PID file on Windows. On other
operating systems it is ignored.config
- the configurationresource
- on Windows this parameter controls memory and CPU limits.
If null, no limits are set. On other operating systems it is ignored.protected String[] getRunCommandForWindows(String command, String groupId, String userName, org.apache.hadoop.fs.Path pidFile, org.apache.hadoop.conf.Configuration config, org.apache.hadoop.yarn.api.records.Resource resource)
command
- the command to executegroupId
- the job owner's GIDuserName
- the job owner's usernamepidFile
- the path to the container's PID fileconfig
- the configurationresource
- this parameter controls memory and CPU limits.
If null, no limits are set.protected String[] getRunCommandForOther(String command, org.apache.hadoop.conf.Configuration config)
command
- the command to executeconfig
- the configurationprotected boolean isContainerActive(org.apache.hadoop.yarn.api.records.ContainerId containerId)
containerId
- the target container's IDprotected void updateEnvForWhitelistVars(Map<String,String> env)
env
- the environment to updateYarnConfiguration.NM_ENV_WHITELIST
public void activateContainer(org.apache.hadoop.yarn.api.records.ContainerId containerId, org.apache.hadoop.fs.Path pidFilePath)
containerId
- the container IDpidFilePath
- the path where the executor should write the PID
of the launched processpublic String[] getIpAndHost(Container container) throws ContainerExecutionException
ContainerExecutionException
public void deactivateContainer(org.apache.hadoop.yarn.api.records.ContainerId containerId)
containerId
- the container IDpublic void pauseContainer(Container container)
container
- the Containerpublic void resumeContainer(Container container)
container
- the Containerpublic String getProcessId(org.apache.hadoop.yarn.api.records.ContainerId containerID)
containerID
- the container IDCopyright © 2018 Apache Software Foundation. All Rights Reserved.