org.apache.hadoop.mapred

Class LocalJobRunner

java.lang.Object

org.apache.hadoop.mapred.LocalJobRunner

All Implemented Interfaces:

org.apache.hadoop.ipc.VersionedProtocol, org.apache.hadoop.mapreduce.protocol.ClientProtocol

@InterfaceAudience.Private
 @InterfaceStability.Unstable
public class LocalJobRunner
extends Object
implements org.apache.hadoop.mapreduce.protocol.ClientProtocol

Implements MapReduce locally, in-process, for debugging.

Field Summary

Fields

Modifier and Type	Field and Description
static String	INTERMEDIATE_DATA_ENCRYPTION_ALGO
static String	LOCAL_MAX_MAPS The maximum number of map tasks to run in parallel in LocalJobRunner
static String	LOCAL_MAX_REDUCES The maximum number of reduce tasks to run in parallel in LocalJobRunner
static org.slf4j.Logger	LOG

Fields inherited from interface org.apache.hadoop.mapreduce.protocol.ClientProtocol

versionID

Constructor Summary

Constructors

Constructor and Description
LocalJobRunner(org.apache.hadoop.conf.Configuration conf)
LocalJobRunner(org.apache.hadoop.mapred.JobConf conf) Deprecated.

Method Summary

Modifier and Type	Method and Description
void	cancelDelegationToken(org.apache.hadoop.security.token.Token<org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier> token)
org.apache.hadoop.mapreduce.TaskTrackerInfo[]	getActiveTrackers() Get all active trackers in cluster.
org.apache.hadoop.mapreduce.JobStatus[]	getAllJobs()
org.apache.hadoop.mapreduce.TaskTrackerInfo[]	getBlacklistedTrackers() Get all blacklisted trackers in cluster.
org.apache.hadoop.mapreduce.QueueInfo[]	getChildQueues(String queueName)
org.apache.hadoop.mapreduce.ClusterMetrics	getClusterMetrics()
org.apache.hadoop.security.token.Token<org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier>	getDelegationToken(org.apache.hadoop.io.Text renewer)
String	getFilesystemName()
org.apache.hadoop.mapreduce.Counters	getJobCounters(org.apache.hadoop.mapreduce.JobID id)
String	getJobHistoryDir()
org.apache.hadoop.mapreduce.JobStatus	getJobStatus(org.apache.hadoop.mapreduce.JobID id)
org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus	getJobTrackerStatus()
static int	getLocalMaxRunningMaps(org.apache.hadoop.mapreduce.JobContext job)
static int	getLocalMaxRunningReduces(org.apache.hadoop.mapreduce.JobContext job)
org.apache.hadoop.mapreduce.v2.LogParams	getLogFileParams(org.apache.hadoop.mapreduce.JobID jobID, org.apache.hadoop.mapreduce.TaskAttemptID taskAttemptID)
org.apache.hadoop.mapreduce.JobID	getNewJobID()
org.apache.hadoop.ipc.ProtocolSignature	getProtocolSignature(String protocol, long clientVersion, int clientMethodsHash)
long	getProtocolVersion(String protocol, long clientVersion)
org.apache.hadoop.mapreduce.QueueInfo	getQueue(String queue)
org.apache.hadoop.mapreduce.QueueAclsInfo[]	getQueueAclsForCurrentUser()
org.apache.hadoop.security.authorize.AccessControlList	getQueueAdmins(String queueName)
org.apache.hadoop.mapreduce.QueueInfo[]	getQueues()
org.apache.hadoop.mapreduce.QueueInfo[]	getRootQueues()
String	getStagingAreaDir()
String	getSystemDir()
org.apache.hadoop.mapreduce.TaskCompletionEvent[]	getTaskCompletionEvents(org.apache.hadoop.mapreduce.JobID jobid, int fromEventId, int maxEvents)
String[]	getTaskDiagnostics(org.apache.hadoop.mapreduce.TaskAttemptID taskid) Returns the diagnostic information for a particular task in the given job.
org.apache.hadoop.mapreduce.TaskReport[]	getTaskReports(org.apache.hadoop.mapreduce.JobID id, org.apache.hadoop.mapreduce.TaskType type)
long	getTaskTrackerExpiryInterval()
void	killJob(org.apache.hadoop.mapreduce.JobID id)
boolean	killTask(org.apache.hadoop.mapreduce.TaskAttemptID taskId, boolean shouldFail) Throws UnsupportedOperationException
long	renewDelegationToken(org.apache.hadoop.security.token.Token<org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier> token)
void	setJobPriority(org.apache.hadoop.mapreduce.JobID id, String jp)
static void	setLocalMaxRunningMaps(org.apache.hadoop.mapreduce.JobContext job, int maxMaps) Set the max number of map tasks to run concurrently in the LocalJobRunner.
static void	setLocalMaxRunningReduces(org.apache.hadoop.mapreduce.JobContext job, int maxReduces) Set the max number of reduce tasks to run concurrently in the LocalJobRunner.
org.apache.hadoop.mapreduce.JobStatus	submitJob(org.apache.hadoop.mapreduce.JobID jobid, String jobSubmitDir, org.apache.hadoop.security.Credentials credentials)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

LOG

public static final org.slf4j.Logger LOG

LOCAL_MAX_MAPS

public static final String LOCAL_MAX_MAPS

The maximum number of map tasks to run in parallel in LocalJobRunner

See Also:

Constant Field Values

LOCAL_MAX_REDUCES

public static final String LOCAL_MAX_REDUCES

The maximum number of reduce tasks to run in parallel in LocalJobRunner

See Also:

Constant Field Values

INTERMEDIATE_DATA_ENCRYPTION_ALGO

public static final String INTERMEDIATE_DATA_ENCRYPTION_ALGO

See Also:

Constant Field Values

Constructor Detail

LocalJobRunner

public LocalJobRunner(org.apache.hadoop.conf.Configuration conf)
               throws IOException

Throws:

IOException

LocalJobRunner

@Deprecated
public LocalJobRunner(org.apache.hadoop.mapred.JobConf conf)
                           throws IOException

Deprecated.

Throws:

IOException

Method Detail

getProtocolVersion

public long getProtocolVersion(String protocol,
                               long clientVersion)

Specified by:

getProtocolVersion in interface org.apache.hadoop.ipc.VersionedProtocol

getProtocolSignature

public org.apache.hadoop.ipc.ProtocolSignature getProtocolSignature(String protocol,
                                                                    long clientVersion,
                                                                    int clientMethodsHash)
                                                             throws IOException

Specified by:

getProtocolSignature in interface org.apache.hadoop.ipc.VersionedProtocol

Throws:

IOException

getNewJobID

public org.apache.hadoop.mapreduce.JobID getNewJobID()

Specified by:

getNewJobID in interface org.apache.hadoop.mapreduce.protocol.ClientProtocol

submitJob

public org.apache.hadoop.mapreduce.JobStatus submitJob(org.apache.hadoop.mapreduce.JobID jobid,
                                                       String jobSubmitDir,
                                                       org.apache.hadoop.security.Credentials credentials)
                                                throws IOException

Specified by:

submitJob in interface org.apache.hadoop.mapreduce.protocol.ClientProtocol

Throws:

IOException

killJob

public void killJob(org.apache.hadoop.mapreduce.JobID id)

Specified by:

killJob in interface org.apache.hadoop.mapreduce.protocol.ClientProtocol

setJobPriority

public void setJobPriority(org.apache.hadoop.mapreduce.JobID id,
                           String jp)
                    throws IOException

Specified by:

setJobPriority in interface org.apache.hadoop.mapreduce.protocol.ClientProtocol

Throws:

IOException

killTask

public boolean killTask(org.apache.hadoop.mapreduce.TaskAttemptID taskId,
                        boolean shouldFail)
                 throws IOException

Throws UnsupportedOperationException

Specified by:

killTask in interface org.apache.hadoop.mapreduce.protocol.ClientProtocol

Throws:

IOException

getTaskReports

public org.apache.hadoop.mapreduce.TaskReport[] getTaskReports(org.apache.hadoop.mapreduce.JobID id,
                                                               org.apache.hadoop.mapreduce.TaskType type)

Specified by:

getTaskReports in interface org.apache.hadoop.mapreduce.protocol.ClientProtocol

getJobStatus

public org.apache.hadoop.mapreduce.JobStatus getJobStatus(org.apache.hadoop.mapreduce.JobID id)

Specified by:

getJobStatus in interface org.apache.hadoop.mapreduce.protocol.ClientProtocol

getJobCounters

public org.apache.hadoop.mapreduce.Counters getJobCounters(org.apache.hadoop.mapreduce.JobID id)

Specified by:

getJobCounters in interface org.apache.hadoop.mapreduce.protocol.ClientProtocol

getFilesystemName

public String getFilesystemName()
                         throws IOException

Specified by:

getFilesystemName in interface org.apache.hadoop.mapreduce.protocol.ClientProtocol

Throws:

IOException

getClusterMetrics

public org.apache.hadoop.mapreduce.ClusterMetrics getClusterMetrics()

Specified by:

getClusterMetrics in interface org.apache.hadoop.mapreduce.protocol.ClientProtocol

getJobTrackerStatus

public org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus getJobTrackerStatus()

Specified by:

getJobTrackerStatus in interface org.apache.hadoop.mapreduce.protocol.ClientProtocol

getTaskTrackerExpiryInterval

public long getTaskTrackerExpiryInterval()
                                  throws IOException,
                                         InterruptedException

Specified by:

getTaskTrackerExpiryInterval in interface org.apache.hadoop.mapreduce.protocol.ClientProtocol

Throws:

IOException

InterruptedException

getActiveTrackers

public org.apache.hadoop.mapreduce.TaskTrackerInfo[] getActiveTrackers()
                                                                throws IOException,
                                                                       InterruptedException

Get all active trackers in cluster.

Specified by:

getActiveTrackers in interface org.apache.hadoop.mapreduce.protocol.ClientProtocol

Returns:

array of TaskTrackerInfo

Throws:

IOException

InterruptedException

getBlacklistedTrackers

public org.apache.hadoop.mapreduce.TaskTrackerInfo[] getBlacklistedTrackers()
                                                                     throws IOException,
                                                                            InterruptedException

Get all blacklisted trackers in cluster.

Specified by:

getBlacklistedTrackers in interface org.apache.hadoop.mapreduce.protocol.ClientProtocol

Returns:

array of TaskTrackerInfo

Throws:

IOException

InterruptedException

getTaskCompletionEvents

public org.apache.hadoop.mapreduce.TaskCompletionEvent[] getTaskCompletionEvents(org.apache.hadoop.mapreduce.JobID jobid,
                                                                                 int fromEventId,
                                                                                 int maxEvents)
                                                                          throws IOException

Specified by:

getTaskCompletionEvents in interface org.apache.hadoop.mapreduce.protocol.ClientProtocol

Throws:

IOException

getAllJobs

public org.apache.hadoop.mapreduce.JobStatus[] getAllJobs()

Specified by:

getAllJobs in interface org.apache.hadoop.mapreduce.protocol.ClientProtocol

getTaskDiagnostics

public String[] getTaskDiagnostics(org.apache.hadoop.mapreduce.TaskAttemptID taskid)
                            throws IOException

Returns the diagnostic information for a particular task in the given job. To be implemented

Specified by:

getTaskDiagnostics in interface org.apache.hadoop.mapreduce.protocol.ClientProtocol

Throws:

IOException

getSystemDir

public String getSystemDir()

Specified by:

getSystemDir in interface org.apache.hadoop.mapreduce.protocol.ClientProtocol

See Also:

ClientProtocol.getSystemDir()

getQueueAdmins

public org.apache.hadoop.security.authorize.AccessControlList getQueueAdmins(String queueName)
                                                                      throws IOException

Specified by:

getQueueAdmins in interface org.apache.hadoop.mapreduce.protocol.ClientProtocol

Throws:

IOException

See Also:

ClientProtocol.getQueueAdmins(String)

getStagingAreaDir

public String getStagingAreaDir()
                         throws IOException

Specified by:

getStagingAreaDir in interface org.apache.hadoop.mapreduce.protocol.ClientProtocol

Throws:

IOException

See Also:

ClientProtocol.getStagingAreaDir()

getJobHistoryDir

public String getJobHistoryDir()

Specified by:

getJobHistoryDir in interface org.apache.hadoop.mapreduce.protocol.ClientProtocol

getChildQueues

public org.apache.hadoop.mapreduce.QueueInfo[] getChildQueues(String queueName)
                                                       throws IOException

Specified by:

getChildQueues in interface org.apache.hadoop.mapreduce.protocol.ClientProtocol

Throws:

IOException

getRootQueues

public org.apache.hadoop.mapreduce.QueueInfo[] getRootQueues()
                                                      throws IOException

Specified by:

getRootQueues in interface org.apache.hadoop.mapreduce.protocol.ClientProtocol

Throws:

IOException

getQueues

public org.apache.hadoop.mapreduce.QueueInfo[] getQueues()
                                                  throws IOException

Specified by:

getQueues in interface org.apache.hadoop.mapreduce.protocol.ClientProtocol

Throws:

IOException

getQueue

public org.apache.hadoop.mapreduce.QueueInfo getQueue(String queue)
                                               throws IOException

Specified by:

getQueue in interface org.apache.hadoop.mapreduce.protocol.ClientProtocol

Throws:

IOException

getQueueAclsForCurrentUser

public org.apache.hadoop.mapreduce.QueueAclsInfo[] getQueueAclsForCurrentUser()
                                                                       throws IOException

Specified by:

getQueueAclsForCurrentUser in interface org.apache.hadoop.mapreduce.protocol.ClientProtocol

Throws:

IOException

setLocalMaxRunningMaps

public static void setLocalMaxRunningMaps(org.apache.hadoop.mapreduce.JobContext job,
                                          int maxMaps)

Set the max number of map tasks to run concurrently in the LocalJobRunner.

Parameters:

job - the job to configure

maxMaps - the maximum number of map tasks to allow.

getLocalMaxRunningMaps

public static int getLocalMaxRunningMaps(org.apache.hadoop.mapreduce.JobContext job)

Returns:

the max number of map tasks to run concurrently in the LocalJobRunner.

setLocalMaxRunningReduces

public static void setLocalMaxRunningReduces(org.apache.hadoop.mapreduce.JobContext job,
                                             int maxReduces)

Set the max number of reduce tasks to run concurrently in the LocalJobRunner.

Parameters:

job - the job to configure

maxReduces - the maximum number of reduce tasks to allow.

getLocalMaxRunningReduces

public static int getLocalMaxRunningReduces(org.apache.hadoop.mapreduce.JobContext job)

Returns:

the max number of reduce tasks to run concurrently in the LocalJobRunner.

cancelDelegationToken

public void cancelDelegationToken(org.apache.hadoop.security.token.Token<org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier> token)
                           throws IOException,
                                  InterruptedException

Specified by:

cancelDelegationToken in interface org.apache.hadoop.mapreduce.protocol.ClientProtocol

Throws:

IOException

InterruptedException

getDelegationToken

public org.apache.hadoop.security.token.Token<org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier> getDelegationToken(org.apache.hadoop.io.Text renewer)
                                                                                                                                           throws IOException,
                                                                                                                                                  InterruptedException

Specified by:

getDelegationToken in interface org.apache.hadoop.mapreduce.protocol.ClientProtocol

Throws:

IOException

InterruptedException

renewDelegationToken

public long renewDelegationToken(org.apache.hadoop.security.token.Token<org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier> token)
                          throws IOException,
                                 InterruptedException

Specified by:

renewDelegationToken in interface org.apache.hadoop.mapreduce.protocol.ClientProtocol

Throws:

IOException

InterruptedException

getLogFileParams

public org.apache.hadoop.mapreduce.v2.LogParams getLogFileParams(org.apache.hadoop.mapreduce.JobID jobID,
                                                                 org.apache.hadoop.mapreduce.TaskAttemptID taskAttemptID)
                                                          throws IOException,
                                                                 InterruptedException

Specified by:

getLogFileParams in interface org.apache.hadoop.mapreduce.protocol.ClientProtocol

Throws:

IOException

InterruptedException