org.apache.hadoop.mapred
Class JobTracker

java.lang.Object
  extended by org.apache.hadoop.mapred.JobTracker
All Implemented Interfaces:
VersionedProtocol, AdminOperationsProtocol, JobTrackerMXBean, RefreshAuthorizationPolicyProtocol, RefreshUserMappingsProtocol

public class JobTracker
extends Object
implements RefreshUserMappingsProtocol, RefreshAuthorizationPolicyProtocol, AdminOperationsProtocol, JobTrackerMXBean

JobTracker is the central location for submitting and tracking MR jobs in a network environment.


Nested Class Summary
static class JobTracker.IllegalStateException
          A client tried to submit a job before the Job Tracker was ready.
static class JobTracker.SafeModeAction
          JobTracker SafeMode
static class JobTracker.State
           
 
Field Summary
static long COUNTER_UPDATE_INTERVAL
           
static long DEFAULT_DISK_HEALTH_CHECK_INTERVAL
          How often TaskTracker needs to check the health of its disks, if not configured using mapred.disk.healthChecker.interval
static long DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT
           
static String DELEGATION_KEY_UPDATE_INTERVAL_KEY
           
static long DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT
           
static String DELEGATION_TOKEN_MAX_LIFETIME_KEY
           
static long DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT
           
static String DELEGATION_TOKEN_RENEW_INTERVAL_KEY
           
static int FILE_NOT_FOUND
           
static String FOR_REDUCE_TASK
          The reduce task number for which this map output is being transferred
static String FROM_MAP_TASK
          The map task from which the map output data is being transferred
static int HEARTBEAT_INTERVAL_MIN
           
static String JT_KEYTAB_FILE
           
static String JT_USER_NAME
           
static org.apache.commons.logging.Log LOG
           
static String MAP_OUTPUT_LENGTH
          The custom http header used for the map output length.
static String MAPRED_CLUSTER_MAP_MEMORY_MB_PROPERTY
           
static String MAPRED_CLUSTER_MAX_MAP_MEMORY_MB_PROPERTY
           
static String MAPRED_CLUSTER_MAX_REDUCE_MEMORY_MB_PROPERTY
           
static String MAPRED_CLUSTER_REDUCE_MEMORY_MB_PROPERTY
           
static String MAX_USER_JOBCONF_SIZE_KEY
          the config key for max user jobconf size
static String RAW_MAP_OUTPUT_LENGTH
          The custom http header used for the "raw" map output length.
static int SUCCESS
           
static int TRACKERS_OK
           
static int UNKNOWN_TASKTRACKER
           
static long versionID
          version 3 introduced to replace emitHearbeat/pollForNewTask/pollForTaskWithClosedJob with heartbeat(TaskTrackerStatus, boolean, boolean, boolean, short) version 4 changed TaskReport for HADOOP-549.
static long versionID
           
static String WORKDIR
           
 
Fields inherited from interface org.apache.hadoop.security.RefreshUserMappingsProtocol
versionID
 
Fields inherited from interface org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol
versionID
 
Fields inherited from interface org.apache.hadoop.mapred.AdminOperationsProtocol
versionID
 
Method Summary
 Collection<TaskTrackerStatus> activeTaskTrackers()
          Get the active task tracker statuses in the cluster
 void addJobInProgressListener(org.apache.hadoop.mapred.JobInProgressListener listener)
          Registers a JobInProgressListener for updates from this TaskTrackerManager.
 Collection<TaskTrackerStatus> blacklistedTaskTrackers()
          Get the statuses of the blacklisted task trackers in the cluster.
 void cancelDelegationToken(Token<DelegationTokenIdentifier> token)
          Discard a current delegation token.
 Vector<JobInProgress> completedJobs()
           
 Vector<JobInProgress> failedJobs()
           
 void failJob(JobInProgress job)
          Fail a job and inform the listeners.
static InetSocketAddress getAddress(Configuration conf)
           
 String getAliveNodesInfoJson()
           
 JobStatus[] getAllJobs()
          Get all the jobs submitted.
 String getAssignedTracker(TaskAttemptID taskId)
          Get tracker name for a given task id.
 String getBlacklistedNodesInfoJson()
           
 String getBuildVersion()
          Returns the VersionInfo build version of the JobTracker
 TaskReport[] getCleanupTaskReports(JobID jobid)
          Grab a bunch of info on the cleanup tasks that make up the job
 org.apache.hadoop.mapred.Clock getClock()
           
 ClusterMetrics getClusterMetrics()
           
 ClusterStatus getClusterStatus()
          Deprecated. use getClusterStatus(boolean)
 ClusterStatus getClusterStatus(boolean detailed)
          Get the current status of the cluster
 JobConf getConf()
          Returns a handle to the JobTracker's Configuration
 String getConfigVersion()
           
 Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
          Get a new delegation token.
 DelegationTokenSecretManager getDelegationTokenSecretManager()
           
 String getFilesystemName()
          Grab the local fs name
 String getGraylistedNodesInfoJson()
           
 String getHostname()
           
 int getInfoPort()
           
 JobInProgress getJob(JobID jobid)
          Obtain the job object identified by jobid
 Counters getJobCounters(JobID jobid)
          Grab the current job counters
 JobProfile getJobProfile(JobID jobid)
          Grab a handle to a job that is already known to the JobTracker.
 JobStatus[] getJobsFromQueue(String queue)
          Gets all the jobs submitted to the particular Queue
 JobStatus getJobStatus(JobID jobid)
          Grab a handle to a job that is already known to the JobTracker.
 String getJobTrackerMachine()
           
static String getLocalJobFilePath(JobID jobId)
          Get the localized job file path on the job trackers local file system
 TaskReport[] getMapTaskReports(JobID jobid)
          Grab a bunch of info on the map tasks that make up the job
 JobID getNewJobId()
          Allocates a new JobId string.
 int getNextHeartbeatInterval()
          Calculates next heartbeat interval using cluster size.
 Node getNode(String name)
          Return the Node in the network topology that corresponds to the hostname
 Collection<Node> getNodesAtMaxLevel()
          Returns a collection of nodes at the max level
 int getNumberOfUniqueHosts()
           
 int getNumResolvedTaskTrackers()
           
 int getNumTaskCacheLevels()
           
static Node getParentNode(Node node, int level)
           
 long getProtocolVersion(String protocol, long clientVersion)
          Return protocol version corresponding to protocol interface.
 QueueAclsInfo[] getQueueAclsForCurrentUser()
          Gets the Queue ACLs for current user
 AccessControlList getQueueAdmins(String queueName)
          Get the administrators of the given job-queue.
 JobQueueInfo getQueueInfo(String queue)
          Gets scheduling information associated with the particular Job queue
 String getQueueInfoJson()
           
 org.apache.hadoop.mapred.QueueManager getQueueManager()
          Return the QueueManager associated with the JobTracker.
 JobQueueInfo[] getQueues()
          Gets set of Job Queues associated with the Job Tracker
 String getReasonsForBlacklisting(String host)
           
 String getReasonsForGraylisting(String host)
           
 long getRecoveryDuration()
          How long the jobtracker took to recover from restart.
 TaskReport[] getReduceTaskReports(JobID jobid)
          Grab a bunch of info on the reduce tasks that make up the job
 List<JobInProgress> getRunningJobs()
          Version that is called from a timer thread, and therefore needs to be careful to synchronize.
 TaskReport[] getSetupTaskReports(JobID jobid)
          Grab a bunch of info on the setup tasks that make up the job
 String getStagingAreaDir()
          Get a hint from the JobTracker where job-specific files are to be placed.
 long getStartTime()
           
 String getSummaryJson()
           
 String getSystemDir()
          Grab the jobtracker system directory path where job-specific files are to be placed.
 TaskCompletionEvent[] getTaskCompletionEvents(JobID jobid, int fromEventId, int maxEvents)
          Get task completion events for the jobid, starting from fromEventId.
 String[] getTaskDiagnostics(TaskAttemptID taskId)
          Get the diagnostics for a given task
 TaskTracker getTaskTracker(String trackerID)
           
 TaskTrackerStatus getTaskTrackerStatus(String trackerID)
           
 int getThreadCount()
           
 org.apache.hadoop.mapred.TaskInProgress getTip(TaskID tipid)
          Returns specified TaskInProgress, or null.
 int getTotalSubmissions()
           
 String getTrackerIdentifier()
          Get the unique identifier (ie.
 int getTrackerPort()
           
 String getVersion()
           
 String getVIVersion()
          Returns the VersionInfo version of the JobTracker
 Collection<TaskTrackerStatus> graylistedTaskTrackers()
          Get the statuses of the graylisted task trackers in the cluster.
 boolean hasRecovered()
          Whether the JT has recovered upon restart
 boolean hasRestarted()
          Whether the JT has restarted
 org.apache.hadoop.mapred.HeartbeatResponse heartbeat(TaskTrackerStatus status, boolean restarted, boolean initialContact, boolean acceptNewTasks, short responseId)
          The periodic heartbeat mechanism between the TaskTracker and the JobTracker.
 void initJob(JobInProgress job)
          Initialize the Job
 boolean isBlacklisted(String trackerID)
          Whether the tracker is blacklisted or not
 boolean isGraylisted(String trackerID)
          Whether the tracker is graylisted or not
 boolean isInSafeMode()
          Get safe mode.
 JobStatus[] jobsToComplete()
          Get the jobs that are not completed and not failed
 void killJob(JobID jobid)
          Kill the indicated job
 boolean killTask(TaskAttemptID taskid, boolean shouldFail)
          Kill indicated task attempt.
static void main(String[] argv)
          Start the JobTracker process.
 void offerService()
          Run forever
 void refreshNodes()
          Rereads the config to get hosts and exclude list file names.
 void refreshQueues()
          Refresh the queue acls in use currently.
 void refreshServiceAcl()
          Refresh the service-level authorization policy in-effect.
 void refreshSuperUserGroupsConfiguration()
          Refresh superuser proxy group list
 void refreshUserToGroupsMappings()
          Refresh user to group mappings.
 void removeJobInProgressListener(org.apache.hadoop.mapred.JobInProgressListener listener)
          Unregisters a JobInProgressListener from this TaskTrackerManager.
 long renewDelegationToken(Token<DelegationTokenIdentifier> token)
          Renew a delegation token to extend its lifetime.
 void reportTaskTrackerError(String taskTracker, String errorClass, String errorMessage)
          Report a problem to the job tracker.
 Node resolveAndAddToTopology(String name)
           
 Vector<JobInProgress> runningJobs()
           
 void setJobPriority(JobID jobid, String priority)
          Set the priority of the specified job
 boolean setSafeMode(JobTracker.SafeModeAction safeModeAction)
          Set safe mode for the JobTracker.
static JobTracker startTracker(JobConf conf)
          Start the JobTracker with given configuration.
static JobTracker startTracker(JobConf conf, String identifier)
           
static JobTracker startTracker(JobConf conf, String identifier, boolean initialize)
           
 void stopTracker()
           
 JobStatus submitJob(JobID jobId, String jobSubmitDir, Credentials ts)
          JobTracker.submitJob() kicks off a new job.
 JobStatus submitJob(JobID jobId, String jobSubmitDir, UserGroupInformation ugi, Credentials ts, boolean recovered)
          JobTracker.submitJob() kicks off a new job.
 List<List<String>> taskTrackerNames()
          Get the active, blacklisted, and graylisted task tracker names in the cluster.
 Collection<TaskTrackerStatus> taskTrackers()
          Get all the task trackers in the cluster
 
Methods inherited from class java.lang.Object
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
 

Field Detail

MAX_USER_JOBCONF_SIZE_KEY

public static final String MAX_USER_JOBCONF_SIZE_KEY
the config key for max user jobconf size

See Also:
Constant Field Values

DELEGATION_KEY_UPDATE_INTERVAL_KEY

public static final String DELEGATION_KEY_UPDATE_INTERVAL_KEY
See Also:
Constant Field Values

DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT

public static final long DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT
See Also:
Constant Field Values

DELEGATION_TOKEN_RENEW_INTERVAL_KEY

public static final String DELEGATION_TOKEN_RENEW_INTERVAL_KEY
See Also:
Constant Field Values

DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT

public static final long DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT
See Also:
Constant Field Values

DELEGATION_TOKEN_MAX_LIFETIME_KEY

public static final String DELEGATION_TOKEN_MAX_LIFETIME_KEY
See Also:
Constant Field Values

DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT

public static final long DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT
See Also:
Constant Field Values

LOG

public static final org.apache.commons.logging.Log LOG

JT_USER_NAME

public static final String JT_USER_NAME
See Also:
Constant Field Values

JT_KEYTAB_FILE

public static final String JT_KEYTAB_FILE
See Also:
Constant Field Values

MAPRED_CLUSTER_MAP_MEMORY_MB_PROPERTY

public static final String MAPRED_CLUSTER_MAP_MEMORY_MB_PROPERTY
See Also:
Constant Field Values

MAPRED_CLUSTER_REDUCE_MEMORY_MB_PROPERTY

public static final String MAPRED_CLUSTER_REDUCE_MEMORY_MB_PROPERTY
See Also:
Constant Field Values

MAPRED_CLUSTER_MAX_MAP_MEMORY_MB_PROPERTY

public static final String MAPRED_CLUSTER_MAX_MAP_MEMORY_MB_PROPERTY
See Also:
Constant Field Values

MAPRED_CLUSTER_MAX_REDUCE_MEMORY_MB_PROPERTY

public static final String MAPRED_CLUSTER_MAX_REDUCE_MEMORY_MB_PROPERTY
See Also:
Constant Field Values

HEARTBEAT_INTERVAL_MIN

public static final int HEARTBEAT_INTERVAL_MIN
See Also:
Constant Field Values

COUNTER_UPDATE_INTERVAL

public static final long COUNTER_UPDATE_INTERVAL
See Also:
Constant Field Values

DEFAULT_DISK_HEALTH_CHECK_INTERVAL

public static final long DEFAULT_DISK_HEALTH_CHECK_INTERVAL
How often TaskTracker needs to check the health of its disks, if not configured using mapred.disk.healthChecker.interval

See Also:
Constant Field Values

SUCCESS

public static final int SUCCESS
See Also:
Constant Field Values

FILE_NOT_FOUND

public static final int FILE_NOT_FOUND
See Also:
Constant Field Values

MAP_OUTPUT_LENGTH

public static final String MAP_OUTPUT_LENGTH
The custom http header used for the map output length.

See Also:
Constant Field Values

RAW_MAP_OUTPUT_LENGTH

public static final String RAW_MAP_OUTPUT_LENGTH
The custom http header used for the "raw" map output length.

See Also:
Constant Field Values

FROM_MAP_TASK

public static final String FROM_MAP_TASK
The map task from which the map output data is being transferred

See Also:
Constant Field Values

FOR_REDUCE_TASK

public static final String FOR_REDUCE_TASK
The reduce task number for which this map output is being transferred

See Also:
Constant Field Values

WORKDIR

public static final String WORKDIR
See Also:
Constant Field Values

versionID

public static final long versionID
version 3 introduced to replace emitHearbeat/pollForNewTask/pollForTaskWithClosedJob with heartbeat(TaskTrackerStatus, boolean, boolean, boolean, short) version 4 changed TaskReport for HADOOP-549. version 5 introduced that removes locateMapOutputs and instead uses getTaskCompletionEvents to figure finished maps and fetch the outputs version 6 adds maxTasks to TaskTrackerStatus for HADOOP-1245 version 7 replaces maxTasks by maxMapTasks and maxReduceTasks in TaskTrackerStatus for HADOOP-1274 Version 8: HeartbeatResponse is added with the next heartbeat interval. version 9 changes the counter representation for HADOOP-2248 version 10 changes the TaskStatus representation for HADOOP-2208 version 11 changes string to JobID in getTaskCompletionEvents(). version 12 changes the counters representation for HADOOP-1915 version 13 added call getBuildVersion() for HADOOP-236 Version 14: replaced getFilesystemName with getSystemDir for HADOOP-3135 Version 15: Changed format of Task and TaskStatus for HADOOP-153 Version 16: adds ResourceStatus to TaskTrackerStatus for HADOOP-3759 Version 17: Changed format of Task and TaskStatus for HADOOP-3150 Version 18: Changed status message due to changes in TaskStatus Version 19: Changed heartbeat to piggyback JobTracker restart information so that the TaskTracker can synchronize itself. Version 20: Changed status message due to changes in TaskStatus (HADOOP-4232) Version 21: Changed information reported in TaskTrackerStatus' ResourceStatus and the corresponding accessor methods (HADOOP-4035) Version 22: Replaced parameter 'initialContact' with 'restarted' in heartbeat method (HADOOP-4305) Version 23: Added parameter 'initialContact' again in heartbeat method (HADOOP-4869) Version 24: Changed format of Task and TaskStatus for HADOOP-4759 Version 25: JobIDs are passed in response to JobTracker restart Version 26: Added numRequiredSlots to TaskStatus for MAPREDUCE-516 Version 27: Adding node health status to TaskStatus for MAPREDUCE-211 Version 28: Adding user name to the serialized Task for use by TT. Version 29: Adding available memory and CPU usage information on TT to TaskTrackerStatus for MAPREDUCE-1218 Version 30: Adding disk failure to TaskTrackerStatus for MAPREDUCE-3015 Version 31: Adding version methods for HADOOP-8209

See Also:
Constant Field Values

TRACKERS_OK

public static final int TRACKERS_OK
See Also:
Constant Field Values

UNKNOWN_TASKTRACKER

public static final int UNKNOWN_TASKTRACKER
See Also:
Constant Field Values

versionID

public static final long versionID
See Also:
Constant Field Values
Method Detail

getClock

public org.apache.hadoop.mapred.Clock getClock()

startTracker

public static JobTracker startTracker(JobConf conf)
                               throws IOException,
                                      InterruptedException
Start the JobTracker with given configuration. The conf will be modified to reflect the actual ports on which the JobTracker is up and running if the user passes the port as zero.

Parameters:
conf - configuration for the JobTracker.
Throws:
IOException
InterruptedException

startTracker

public static JobTracker startTracker(JobConf conf,
                                      String identifier)
                               throws IOException,
                                      InterruptedException
Throws:
IOException
InterruptedException

startTracker

public static JobTracker startTracker(JobConf conf,
                                      String identifier,
                                      boolean initialize)
                               throws IOException,
                                      InterruptedException
Throws:
IOException
InterruptedException

stopTracker

public void stopTracker()
                 throws IOException
Throws:
IOException

getProtocolVersion

public long getProtocolVersion(String protocol,
                               long clientVersion)
                        throws IOException
Description copied from interface: VersionedProtocol
Return protocol version corresponding to protocol interface.

Specified by:
getProtocolVersion in interface VersionedProtocol
Parameters:
protocol - The classname of the protocol interface
clientVersion - The version of the protocol that the client speaks
Returns:
the version that the server will speak
Throws:
IOException

getDelegationTokenSecretManager

public DelegationTokenSecretManager getDelegationTokenSecretManager()

hasRestarted

public boolean hasRestarted()
Whether the JT has restarted


hasRecovered

public boolean hasRecovered()
Whether the JT has recovered upon restart


getRecoveryDuration

public long getRecoveryDuration()
How long the jobtracker took to recover from restart.


getAddress

public static InetSocketAddress getAddress(Configuration conf)

offerService

public void offerService()
                  throws InterruptedException,
                         IOException
Run forever

Throws:
InterruptedException
IOException

getTotalSubmissions

public int getTotalSubmissions()

getJobTrackerMachine

public String getJobTrackerMachine()

getTrackerIdentifier

public String getTrackerIdentifier()
Get the unique identifier (ie. timestamp) of this job tracker start.

Returns:
a string with a unique identifier

getTrackerPort

public int getTrackerPort()

getInfoPort

public int getInfoPort()

getStartTime

public long getStartTime()

runningJobs

public Vector<JobInProgress> runningJobs()

getRunningJobs

public List<JobInProgress> getRunningJobs()
Version that is called from a timer thread, and therefore needs to be careful to synchronize.


failedJobs

public Vector<JobInProgress> failedJobs()

completedJobs

public Vector<JobInProgress> completedJobs()

taskTrackers

public Collection<TaskTrackerStatus> taskTrackers()
Get all the task trackers in the cluster

Returns:
Collection of TaskTrackerStatus

activeTaskTrackers

public Collection<TaskTrackerStatus> activeTaskTrackers()
Get the active task tracker statuses in the cluster

Returns:
Collection of active TaskTrackerStatus

taskTrackerNames

public List<List<String>> taskTrackerNames()
Get the active, blacklisted, and graylisted task tracker names in the cluster. The first element in the returned list contains the list of active tracker names; the second element in the returned list contains the list of blacklisted tracker names; and the third contains the list of graylisted tracker names. Note that the blacklist is disjoint from the active list, but the graylist is not: initially, graylisted trackers are still active and therefore will appear in both lists. (Graylisted trackers can later be blacklisted, in which case they'll be removed from the active list and added to the blacklist, but they remain on the graylist in this case. Blacklisting comes about via the health-check script, while graylisting is heuristically based on the number of per-job blacklistings in a specified time interval.)


blacklistedTaskTrackers

public Collection<TaskTrackerStatus> blacklistedTaskTrackers()
Get the statuses of the blacklisted task trackers in the cluster.

Returns:
Collection of blacklisted TaskTrackerStatus

graylistedTaskTrackers

public Collection<TaskTrackerStatus> graylistedTaskTrackers()
Get the statuses of the graylisted task trackers in the cluster.

Returns:
Collection of graylisted TaskTrackerStatus

isBlacklisted

public boolean isBlacklisted(String trackerID)
Whether the tracker is blacklisted or not

Parameters:
trackerID -
Returns:
true if blacklisted, false otherwise

isGraylisted

public boolean isGraylisted(String trackerID)
Whether the tracker is graylisted or not

Parameters:
trackerID -
Returns:
true if graylisted, false otherwise

getTaskTrackerStatus

public TaskTrackerStatus getTaskTrackerStatus(String trackerID)

getTaskTracker

public TaskTracker getTaskTracker(String trackerID)

resolveAndAddToTopology

public Node resolveAndAddToTopology(String name)
                             throws UnknownHostException
Throws:
UnknownHostException

getNodesAtMaxLevel

public Collection<Node> getNodesAtMaxLevel()
Returns a collection of nodes at the max level


getParentNode

public static Node getParentNode(Node node,
                                 int level)

getNode

public Node getNode(String name)
Return the Node in the network topology that corresponds to the hostname


getNumTaskCacheLevels

public int getNumTaskCacheLevels()

getNumResolvedTaskTrackers

public int getNumResolvedTaskTrackers()

getNumberOfUniqueHosts

public int getNumberOfUniqueHosts()
Returns:
The number of unique hosts running tasktrackers.

addJobInProgressListener

public void addJobInProgressListener(org.apache.hadoop.mapred.JobInProgressListener listener)
Registers a JobInProgressListener for updates from this TaskTrackerManager.


removeJobInProgressListener

public void removeJobInProgressListener(org.apache.hadoop.mapred.JobInProgressListener listener)
Unregisters a JobInProgressListener from this TaskTrackerManager.


getQueueManager

public org.apache.hadoop.mapred.QueueManager getQueueManager()
Return the QueueManager associated with the JobTracker.

Returns:
the QueueManager

getVIVersion

public String getVIVersion()
                    throws IOException
Returns the VersionInfo version of the JobTracker

Throws:
IOException

getBuildVersion

public String getBuildVersion()
                       throws IOException
Returns the VersionInfo build version of the JobTracker

Throws:
IOException

heartbeat

public org.apache.hadoop.mapred.HeartbeatResponse heartbeat(TaskTrackerStatus status,
                                                            boolean restarted,
                                                            boolean initialContact,
                                                            boolean acceptNewTasks,
                                                            short responseId)
                                                     throws IOException
The periodic heartbeat mechanism between the TaskTracker and the JobTracker. The JobTracker processes the status information sent by the TaskTracker and responds with instructions to start/stop tasks or jobs, and also 'reset' instructions during contingencies.

Parameters:
status - the status update
restarted - true if the process has just started or restarted, false otherwise
initialContact - true if this is first interaction since 'refresh', false otherwise.
acceptNewTasks - true if the TaskTracker is ready to accept new tasks to run.
responseId - the last responseId successfully acted upon by the TaskTracker.
Returns:
a HeartbeatResponse with fresh instructions.
Throws:
IOException

getNextHeartbeatInterval

public int getNextHeartbeatInterval()
Calculates next heartbeat interval using cluster size. Heartbeat interval is incremented by 1 second for every 100 nodes by default.

Returns:
next heartbeat interval.

getFilesystemName

public String getFilesystemName()
                         throws IOException
Grab the local fs name

Throws:
IOException

getConf

public JobConf getConf()
Returns a handle to the JobTracker's Configuration


reportTaskTrackerError

public void reportTaskTrackerError(String taskTracker,
                                   String errorClass,
                                   String errorMessage)
                            throws IOException
Report a problem to the job tracker.

Parameters:
taskTracker - the name of the task tracker
errorClass - the kind of error (eg. the class that was thrown)
errorMessage - the human readable error message
Throws:
IOException - if there was a problem in communication or on the remote side

getNewJobId

public JobID getNewJobId()
                  throws IOException
Allocates a new JobId string.

Returns:
a unique job name for submitting jobs.
Throws:
IOException

submitJob

public JobStatus submitJob(JobID jobId,
                           String jobSubmitDir,
                           Credentials ts)
                    throws IOException
JobTracker.submitJob() kicks off a new job. Create a 'JobInProgress' object, which contains both JobProfile and JobStatus. Those two sub-objects are sometimes shipped outside of the JobTracker. But JobInProgress adds info that's useful for the JobTracker alone.

Throws:
IOException

submitJob

public JobStatus submitJob(JobID jobId,
                           String jobSubmitDir,
                           UserGroupInformation ugi,
                           Credentials ts,
                           boolean recovered)
                    throws IOException
JobTracker.submitJob() kicks off a new job. Create a 'JobInProgress' object, which contains both JobProfile and JobStatus. Those two sub-objects are sometimes shipped outside of the JobTracker. But JobInProgress adds info that's useful for the JobTracker alone.

Throws:
IOException

getStagingAreaDir

public String getStagingAreaDir()
                         throws IOException
Get a hint from the JobTracker where job-specific files are to be placed.

Returns:
the directory where job-specific files are to be placed.
Throws:
IOException
See Also:
JobSubmissionProtocol.getStagingAreaDir()

getClusterStatus

@Deprecated
public ClusterStatus getClusterStatus()
Deprecated. use getClusterStatus(boolean)

Returns:
a summary of the cluster's status.

getClusterStatus

public ClusterStatus getClusterStatus(boolean detailed)
Get the current status of the cluster

Parameters:
detailed - if true then report tracker names and memory usage
Returns:
summary of the state of the cluster

getClusterMetrics

public ClusterMetrics getClusterMetrics()

killJob

public void killJob(JobID jobid)
             throws IOException
Kill the indicated job

Throws:
IOException
See Also:
JobSubmissionProtocol.killJob(org.apache.hadoop.mapred.JobID)

cancelDelegationToken

public void cancelDelegationToken(Token<DelegationTokenIdentifier> token)
                           throws IOException,
                                  InterruptedException
Discard a current delegation token.

Parameters:
token - the token to cancel
Throws:
IOException
InterruptedException

getDelegationToken

public Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
                                                    throws IOException,
                                                           InterruptedException
Get a new delegation token.

Parameters:
renewer - the user other than the creator (if any) that can renew the token
Returns:
the new delegation token
Throws:
IOException
InterruptedException

renewDelegationToken

public long renewDelegationToken(Token<DelegationTokenIdentifier> token)
                          throws IOException,
                                 InterruptedException
Renew a delegation token to extend its lifetime.

Parameters:
token - the token to renew
Returns:
the new expiration time
Throws:
IOException
InterruptedException

initJob

public void initJob(JobInProgress job)
Initialize the Job

Parameters:
job - JobInProgress object

failJob

public void failJob(JobInProgress job)
Fail a job and inform the listeners. Other components in the framework should use this to fail a job.

Parameters:
job - JobInProgress object

setJobPriority

public void setJobPriority(JobID jobid,
                           String priority)
                    throws IOException
Set the priority of the specified job

Parameters:
jobid - ID of the job
priority - Priority to be set for the job
Throws:
IOException

getJobProfile

public JobProfile getJobProfile(JobID jobid)
Grab a handle to a job that is already known to the JobTracker.

Returns:
Profile of the job, or null if not found.

getJobStatus

public JobStatus getJobStatus(JobID jobid)
Grab a handle to a job that is already known to the JobTracker.

Returns:
Status of the job, or null if not found.

getJobCounters

public Counters getJobCounters(JobID jobid)
                        throws IOException
Grab the current job counters

Throws:
IOException

getMapTaskReports

public TaskReport[] getMapTaskReports(JobID jobid)
                               throws IOException
Grab a bunch of info on the map tasks that make up the job

Throws:
IOException

getReduceTaskReports

public TaskReport[] getReduceTaskReports(JobID jobid)
                                  throws IOException
Grab a bunch of info on the reduce tasks that make up the job

Throws:
IOException

getCleanupTaskReports

public TaskReport[] getCleanupTaskReports(JobID jobid)
                                   throws IOException
Grab a bunch of info on the cleanup tasks that make up the job

Throws:
IOException

getSetupTaskReports

public TaskReport[] getSetupTaskReports(JobID jobid)
                                 throws IOException
Grab a bunch of info on the setup tasks that make up the job

Throws:
IOException

getTaskCompletionEvents

public TaskCompletionEvent[] getTaskCompletionEvents(JobID jobid,
                                                     int fromEventId,
                                                     int maxEvents)
                                              throws IOException
Get task completion events for the jobid, starting from fromEventId. Returns empty aray if no events are available.

Parameters:
jobid - job id
fromEventId - event id to start from.
maxEvents - the max number of events we want to look at
Returns:
array of task completion events.
Throws:
IOException

getTaskDiagnostics

public String[] getTaskDiagnostics(TaskAttemptID taskId)
                            throws IOException
Get the diagnostics for a given task

Parameters:
taskId - the id of the task
Returns:
an array of the diagnostic messages
Throws:
IOException

getTip

public org.apache.hadoop.mapred.TaskInProgress getTip(TaskID tipid)
Returns specified TaskInProgress, or null.


killTask

public boolean killTask(TaskAttemptID taskid,
                        boolean shouldFail)
                 throws IOException
Kill indicated task attempt.

Parameters:
taskid - the id of the task to kill.
shouldFail - if true the task is failed and added to failed tasks list, otherwise it is just killed, w/o affecting job failure status.
Returns:
true if the task was found and successfully marked to kill
Throws:
IOException
See Also:
JobSubmissionProtocol.killTask(TaskAttemptID, boolean)

getAssignedTracker

public String getAssignedTracker(TaskAttemptID taskId)
Get tracker name for a given task id.

Parameters:
taskId - the name of the task
Returns:
The name of the task tracker

jobsToComplete

public JobStatus[] jobsToComplete()
Get the jobs that are not completed and not failed

Returns:
array of JobStatus for the running/to-be-run jobs.

getAllJobs

public JobStatus[] getAllJobs()
Get all the jobs submitted.

Returns:
array of JobStatus for the submitted jobs
See Also:
JobSubmissionProtocol.getAllJobs()

getSystemDir

public String getSystemDir()
Grab the jobtracker system directory path where job-specific files are to be placed.

Returns:
the system directory where job-specific files are to be placed.
See Also:
JobSubmissionProtocol.getSystemDir()

getQueueAdmins

public AccessControlList getQueueAdmins(String queueName)
                                 throws IOException
Get the administrators of the given job-queue. This method is for hadoop internal use only.

Returns:
Queue administrators ACL for the queue to which job is submitted to
Throws:
IOException
See Also:
JobSubmissionProtocol.getQueueAdmins(String)

getJob

public JobInProgress getJob(JobID jobid)
Obtain the job object identified by jobid

Returns:
jobInProgress object

refreshNodes

public void refreshNodes()
                  throws IOException
Rereads the config to get hosts and exclude list file names. Rereads the files to update the hosts and exclude lists.

Specified by:
refreshNodes in interface AdminOperationsProtocol
Throws:
IOException

getLocalJobFilePath

public static String getLocalJobFilePath(JobID jobId)
Get the localized job file path on the job trackers local file system

Parameters:
jobId - id of the job
Returns:
the path of the job conf file on the local file system

main

public static void main(String[] argv)
                 throws IOException,
                        InterruptedException
Start the JobTracker process. This is used only for debugging. As a rule, JobTracker should be run as part of the DFS Namenode process.

Throws:
IOException
InterruptedException

getQueues

public JobQueueInfo[] getQueues()
                         throws IOException
Gets set of Job Queues associated with the Job Tracker

Returns:
Array of the Job Queue Information Object
Throws:
IOException

getQueueInfo

public JobQueueInfo getQueueInfo(String queue)
                          throws IOException
Gets scheduling information associated with the particular Job queue

Parameters:
queue - Queue Name
Returns:
Scheduling Information of the Queue
Throws:
IOException

getJobsFromQueue

public JobStatus[] getJobsFromQueue(String queue)
                             throws IOException
Gets all the jobs submitted to the particular Queue

Parameters:
queue - Queue name
Returns:
array of JobStatus for the submitted jobs
Throws:
IOException

getQueueAclsForCurrentUser

public QueueAclsInfo[] getQueueAclsForCurrentUser()
                                           throws IOException
Gets the Queue ACLs for current user

Returns:
array of QueueAclsInfo object for current user.
Throws:
IOException

refreshServiceAcl

public void refreshServiceAcl()
                       throws IOException
Description copied from interface: RefreshAuthorizationPolicyProtocol
Refresh the service-level authorization policy in-effect.

Specified by:
refreshServiceAcl in interface RefreshAuthorizationPolicyProtocol
Throws:
IOException

refreshSuperUserGroupsConfiguration

public void refreshSuperUserGroupsConfiguration()
Description copied from interface: RefreshUserMappingsProtocol
Refresh superuser proxy group list

Specified by:
refreshSuperUserGroupsConfiguration in interface RefreshUserMappingsProtocol

refreshUserToGroupsMappings

public void refreshUserToGroupsMappings()
                                 throws IOException
Description copied from interface: RefreshUserMappingsProtocol
Refresh user to group mappings.

Specified by:
refreshUserToGroupsMappings in interface RefreshUserMappingsProtocol
Throws:
IOException

refreshQueues

public void refreshQueues()
                   throws IOException
Description copied from interface: AdminOperationsProtocol
Refresh the queue acls in use currently. Refresh the queues used by the jobtracker and scheduler. Access control lists and queue states are refreshed.

Specified by:
refreshQueues in interface AdminOperationsProtocol
Throws:
IOException

getReasonsForBlacklisting

public String getReasonsForBlacklisting(String host)

getReasonsForGraylisting

public String getReasonsForGraylisting(String host)

getHostname

public String getHostname()
Specified by:
getHostname in interface JobTrackerMXBean
Returns:
hostname of the jobtracker

getVersion

public String getVersion()
Specified by:
getVersion in interface JobTrackerMXBean
Returns:
version of the code base

getConfigVersion

public String getConfigVersion()
Specified by:
getConfigVersion in interface JobTrackerMXBean
Returns:
the config version (from a config property)

getThreadCount

public int getThreadCount()
Specified by:
getThreadCount in interface JobTrackerMXBean
Returns:
number of threads of the jobtracker jvm

getSummaryJson

public String getSummaryJson()
Specified by:
getSummaryJson in interface JobTrackerMXBean
Returns:
the summary info in json

getAliveNodesInfoJson

public String getAliveNodesInfoJson()
Specified by:
getAliveNodesInfoJson in interface JobTrackerMXBean
Returns:
the alive nodes info in json

getBlacklistedNodesInfoJson

public String getBlacklistedNodesInfoJson()
Specified by:
getBlacklistedNodesInfoJson in interface JobTrackerMXBean
Returns:
the blacklisted nodes info in json

getGraylistedNodesInfoJson

public String getGraylistedNodesInfoJson()
Specified by:
getGraylistedNodesInfoJson in interface JobTrackerMXBean
Returns:
the graylisted nodes info in json

getQueueInfoJson

public String getQueueInfoJson()
Specified by:
getQueueInfoJson in interface JobTrackerMXBean
Returns:
the queue info json

setSafeMode

public boolean setSafeMode(JobTracker.SafeModeAction safeModeAction)
                    throws IOException
Description copied from interface: AdminOperationsProtocol
Set safe mode for the JobTracker.

Specified by:
setSafeMode in interface AdminOperationsProtocol
Parameters:
safeModeAction - safe mode action
Returns:
current safemode
Throws:
IOException

isInSafeMode

public boolean isInSafeMode()
Get safe mode.

Returns:


Copyright © 2009 The Apache Software Foundation