|
||||||||||
PREV CLASS NEXT CLASS | FRAMES NO FRAMES | |||||||||
SUMMARY: NESTED | FIELD | CONSTR | METHOD | DETAIL: FIELD | CONSTR | METHOD |
java.lang.Object org.apache.hadoop.mapred.JobInProgress
public class JobInProgress
JobInProgress maintains all the info for keeping a Job on the straight and narrow. It keeps its JobProfile and its latest JobStatus, plus a set of tables for doing bookkeeping of its Tasks. ***********************************************************
Nested Class Summary | |
---|---|
static class |
JobInProgress.Counter
|
Field Summary | |
---|---|
protected Credentials |
tokenStorage
|
Constructor Summary | |
---|---|
protected |
JobInProgress(JobID jobid,
JobConf conf,
JobTracker tracker)
Create an almost empty JobInProgress, which can be used only for tests |
Method Summary | |
---|---|
void |
cleanUpMetrics()
Called when the job is complete |
boolean |
completedTask(org.apache.hadoop.mapred.TaskInProgress tip,
TaskStatus status)
A taskid assigned to this JobInProgress has reported in successfully. |
int |
desiredMaps()
|
int |
desiredReduces()
|
int |
desiredTasks()
Return total number of map and reduce tasks desired by the job. |
void |
failedTask(org.apache.hadoop.mapred.TaskInProgress tip,
TaskAttemptID taskid,
String reason,
TaskStatus.Phase phase,
TaskStatus.State state,
String trackerName)
Fail a task with a given reason, but without a status object. |
TaskStatus |
findFinishedMap(int mapId)
Find the details of someplace where a map has finished |
protected org.apache.hadoop.mapred.TaskInProgress |
findSpeculativeTask(Collection<org.apache.hadoop.mapred.TaskInProgress> list,
TaskTrackerStatus ttStatus,
double avgProgress,
long currentTime,
boolean shouldRemove)
Find a speculative task |
int |
finishedMaps()
|
int |
finishedReduces()
|
boolean |
getCounters(Counters result)
Returns the total job counters, by adding together the job, the map and the reduce counters. |
long |
getFinishTime()
|
Counters |
getJobCounters()
Returns the job-level counters. |
JobID |
getJobID()
|
String |
getJobSubmitHostAddress()
|
String |
getJobSubmitHostName()
|
long |
getLaunchTime()
|
boolean |
getMapCounters(Counters counters)
Returns map phase counters by summing over all map tasks in progress. |
int |
getNumReservedTaskTrackersForMaps()
|
int |
getNumReservedTaskTrackersForReduces()
|
long |
getNumSchedulingOpportunities()
|
int |
getNumSlotsPerTask(TaskType taskType)
|
JobPriority |
getPriority()
|
JobProfile |
getProfile()
|
org.apache.hadoop.mapred.QueueMetrics |
getQueueMetrics()
Get the QueueMetrics object associated with this job |
boolean |
getReduceCounters(Counters counters)
Returns map phase counters by summing over all map tasks in progress. |
Object |
getSchedulingInfo()
|
long |
getStartTime()
|
JobStatus |
getStatus()
|
TaskCompletionEvent[] |
getTaskCompletionEvents(int fromEventId,
int maxEvents)
|
org.apache.hadoop.mapred.TaskInProgress |
getTaskInProgress(TaskID tipid)
Return the TaskInProgress that matches the tipid. |
String |
getUser()
Get the user for the job |
boolean |
inited()
Check if the job has been initialized. |
void |
initTasks()
Construct the splits, etc. |
void |
kill()
Kill the job and all its component tasks. |
Task |
obtainJobCleanupTask(TaskTrackerStatus tts,
int clusterSize,
int numUniqueHosts,
boolean isMapSlot)
Return a CleanupTask, if appropriate, to run on the given tasktracker |
Task |
obtainJobSetupTask(TaskTrackerStatus tts,
int clusterSize,
int numUniqueHosts,
boolean isMapSlot)
Return a SetupTask, if appropriate, to run on the given tasktracker |
Task |
obtainNewMapTask(TaskTrackerStatus tts,
int clusterSize,
int numUniqueHosts)
Return a MapTask, if appropriate, to run on the given tasktracker |
Task |
obtainNewMapTaskCommon(TaskTrackerStatus tts,
int clusterSize,
int numUniqueHosts,
int maxCacheLevel)
Return a MapTask with locality level that smaller or equal than a given locality level to tasktracker. |
Task |
obtainNewNodeLocalMapTask(TaskTrackerStatus tts,
int clusterSize,
int numUniqueHosts)
|
Task |
obtainNewNodeOrRackLocalMapTask(TaskTrackerStatus tts,
int clusterSize,
int numUniqueHosts)
|
Task |
obtainNewNonLocalMapTask(TaskTrackerStatus tts,
int clusterSize,
int numUniqueHosts)
|
Task |
obtainNewReduceTask(TaskTrackerStatus tts,
int clusterSize,
int numUniqueHosts)
Return a ReduceTask, if appropriate, to run on the given tasktracker. |
Task |
obtainTaskCleanupTask(TaskTrackerStatus tts,
boolean isMapSlot)
|
void |
overrideSchedulingOpportunities()
|
int |
pendingMaps()
|
int |
pendingReduces()
|
Vector<org.apache.hadoop.mapred.TaskInProgress> |
reportCleanupTIPs(boolean shouldBeComplete)
Return a vector of cleanup TaskInProgress objects |
Vector<org.apache.hadoop.mapred.TaskInProgress> |
reportSetupTIPs(boolean shouldBeComplete)
Return a vector of setup TaskInProgress objects |
Vector<org.apache.hadoop.mapred.TaskInProgress> |
reportTasksInProgress(boolean shouldBeMap,
boolean shouldBeComplete)
Return a vector of completed TaskInProgress objects |
void |
reserveTaskTracker(TaskTracker taskTracker,
TaskType type,
int numSlots)
|
void |
resetSchedulingOpportunities()
|
int |
runningMaps()
|
int |
runningReduces()
|
protected void |
scheduleMap(org.apache.hadoop.mapred.TaskInProgress tip)
Adds a map tip to the list of running maps. |
boolean |
scheduleOffSwitch(int numTaskTrackers)
Check if we can schedule an off-switch task for this job. |
protected void |
scheduleReduce(org.apache.hadoop.mapred.TaskInProgress tip)
Adds a reduce tip to the list of running reduces |
boolean |
scheduleReduces()
|
void |
schedulingOpportunity()
|
void |
setPriority(JobPriority priority)
|
void |
setSchedulingInfo(Object schedulingInfo)
|
void |
unreserveTaskTracker(TaskTracker taskTracker,
TaskType type)
|
void |
updateTaskStatus(org.apache.hadoop.mapred.TaskInProgress tip,
TaskStatus status)
Assuming JobTracker is locked on entry. |
Methods inherited from class java.lang.Object |
---|
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait |
Field Detail |
---|
protected Credentials tokenStorage
Constructor Detail |
---|
protected JobInProgress(JobID jobid, JobConf conf, JobTracker tracker) throws IOException
IOException
Method Detail |
---|
public org.apache.hadoop.mapred.QueueMetrics getQueueMetrics()
public void cleanUpMetrics()
public boolean inited()
true
if the job has been initialized,
false
otherwisepublic String getUser()
public void initTasks() throws IOException, org.apache.hadoop.mapred.JobInProgress.KillInterruptedException, UnknownHostException
IOException
org.apache.hadoop.mapred.JobInProgress.KillInterruptedException
UnknownHostException
public JobProfile getProfile()
public JobStatus getStatus()
public long getLaunchTime()
public long getStartTime()
public long getFinishTime()
public int desiredMaps()
public int finishedMaps()
public int desiredReduces()
public int runningMaps()
public int runningReduces()
public int finishedReduces()
public int pendingMaps()
public int pendingReduces()
public int desiredTasks()
public int getNumSlotsPerTask(TaskType taskType)
public JobPriority getPriority()
public void setPriority(JobPriority priority)
public Vector<org.apache.hadoop.mapred.TaskInProgress> reportTasksInProgress(boolean shouldBeMap, boolean shouldBeComplete)
public Vector<org.apache.hadoop.mapred.TaskInProgress> reportCleanupTIPs(boolean shouldBeComplete)
public Vector<org.apache.hadoop.mapred.TaskInProgress> reportSetupTIPs(boolean shouldBeComplete)
public void updateTaskStatus(org.apache.hadoop.mapred.TaskInProgress tip, TaskStatus status)
JobTracker
is locked on entry.
public Counters getJobCounters()
public boolean getMapCounters(Counters counters)
public boolean getReduceCounters(Counters counters)
public boolean getCounters(Counters result)
public Task obtainNewMapTask(TaskTrackerStatus tts, int clusterSize, int numUniqueHosts) throws IOException
IOException
public Task obtainNewMapTaskCommon(TaskTrackerStatus tts, int clusterSize, int numUniqueHosts, int maxCacheLevel) throws IOException
tts
- The task tracker that is asking for a taskclusterSize
- The number of task trackers in the clusternumUniqueHosts
- The number of hosts that run task trackersavgProgress
- The average progress of this kind of task in this jobmaxCacheLevel
- The maximum topology level until which to schedule
maps.
IOException
public Task obtainTaskCleanupTask(TaskTrackerStatus tts, boolean isMapSlot) throws IOException
IOException
public Task obtainNewNodeLocalMapTask(TaskTrackerStatus tts, int clusterSize, int numUniqueHosts) throws IOException
IOException
public Task obtainNewNodeOrRackLocalMapTask(TaskTrackerStatus tts, int clusterSize, int numUniqueHosts) throws IOException
IOException
public Task obtainNewNonLocalMapTask(TaskTrackerStatus tts, int clusterSize, int numUniqueHosts) throws IOException
IOException
public void schedulingOpportunity()
public void resetSchedulingOpportunities()
public long getNumSchedulingOpportunities()
public void overrideSchedulingOpportunities()
public boolean scheduleOffSwitch(int numTaskTrackers)
numTaskTrackers
- number of tasktrackers
true
if we can schedule off-switch,
false
otherwise
We check the number of missed opportunities for the job.
If it has 'waited' long enough we go ahead and schedule.public Task obtainJobCleanupTask(TaskTrackerStatus tts, int clusterSize, int numUniqueHosts, boolean isMapSlot) throws IOException
IOException
public Task obtainJobSetupTask(TaskTrackerStatus tts, int clusterSize, int numUniqueHosts, boolean isMapSlot) throws IOException
IOException
public boolean scheduleReduces()
public Task obtainNewReduceTask(TaskTrackerStatus tts, int clusterSize, int numUniqueHosts) throws IOException
IOException
public void reserveTaskTracker(TaskTracker taskTracker, TaskType type, int numSlots)
public void unreserveTaskTracker(TaskTracker taskTracker, TaskType type)
public int getNumReservedTaskTrackersForMaps()
public int getNumReservedTaskTrackersForReduces()
protected void scheduleMap(org.apache.hadoop.mapred.TaskInProgress tip)
tip
- the tip that needs to be scheduled as runningprotected void scheduleReduce(org.apache.hadoop.mapred.TaskInProgress tip)
tip
- the tip that needs to be scheduled as runningprotected org.apache.hadoop.mapred.TaskInProgress findSpeculativeTask(Collection<org.apache.hadoop.mapred.TaskInProgress> list, TaskTrackerStatus ttStatus, double avgProgress, long currentTime, boolean shouldRemove)
list
- a list of tipsttStatus
- status of the tracker that has requested a tipavgProgress
- the average progress for speculationcurrentTime
- current time in millisecondsshouldRemove
- whether to remove the tips
public boolean completedTask(org.apache.hadoop.mapred.TaskInProgress tip, TaskStatus status)
public void kill()
public void failedTask(org.apache.hadoop.mapred.TaskInProgress tip, TaskAttemptID taskid, String reason, TaskStatus.Phase phase, TaskStatus.State state, String trackerName)
JobTracker
is locked on entry.
tip
- The task's tiptaskid
- The task idreason
- The reason that the task failedtrackerName
- The task tracker the task failed onpublic org.apache.hadoop.mapred.TaskInProgress getTaskInProgress(TaskID tipid)
public TaskStatus findFinishedMap(int mapId)
mapId
- the id of the map
public TaskCompletionEvent[] getTaskCompletionEvents(int fromEventId, int maxEvents)
public JobID getJobID()
public String getJobSubmitHostName()
public String getJobSubmitHostAddress()
public Object getSchedulingInfo()
public void setSchedulingInfo(Object schedulingInfo)
|
||||||||||
PREV CLASS NEXT CLASS | FRAMES NO FRAMES | |||||||||
SUMMARY: NESTED | FIELD | CONSTR | METHOD | DETAIL: FIELD | CONSTR | METHOD |