org.apache.hadoop.mapreduce.v2.hs

Class HistoryFileManager

java.lang.Object

org.apache.hadoop.service.AbstractService

org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager

All Implemented Interfaces:

Closeable, AutoCloseable, org.apache.hadoop.service.Service

@InterfaceAudience.Public
 @InterfaceStability.Unstable
public class HistoryFileManager
extends org.apache.hadoop.service.AbstractService

This class provides a way to interact with history files in a thread safe manor.

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
class	HistoryFileManager.HistoryFileInfo

Nested classes/interfaces inherited from interface org.apache.hadoop.service.Service

org.apache.hadoop.service.Service.STATE

Field Summary

Fields

Modifier and Type	Field and Description
protected org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.JobListCache	jobListCache
protected ThreadPoolExecutor	moveToDoneExecutor

Constructor Summary

Constructors

Constructor and Description
HistoryFileManager()

Method Summary

Modifier and Type	Method and Description
protected HistoryFileManager.HistoryFileInfo	createHistoryFileInfo(org.apache.hadoop.fs.Path historyFile, org.apache.hadoop.fs.Path confFile, org.apache.hadoop.fs.Path summaryFile, org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo jobIndexInfo, boolean isInDone)
protected org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.JobListCache	createJobListCache()
protected ThreadPoolExecutor	createMoveToDoneThreadPool(int numMoveThreads)
protected boolean	deleteDir(org.apache.hadoop.fs.FileStatus serialDir)
protected List<org.apache.hadoop.fs.FileStatus>	findTimestampedDirectories() Finds all history directories with a timestamp component by scanning the filesystem.
Collection<HistoryFileManager.HistoryFileInfo>	getAllFileInfo()
HistoryFileManager.HistoryFileInfo	getFileInfo(org.apache.hadoop.mapreduce.v2.api.records.JobId jobId)
protected static List<org.apache.hadoop.fs.FileStatus>	scanDirectory(org.apache.hadoop.fs.Path path, org.apache.hadoop.fs.FileContext fc, org.apache.hadoop.fs.PathFilter pathFilter)
protected List<org.apache.hadoop.fs.FileStatus>	scanDirectoryForHistoryFiles(org.apache.hadoop.fs.Path path, org.apache.hadoop.fs.FileContext fc)
protected void	serviceInit(org.apache.hadoop.conf.Configuration conf)
void	serviceStop()

Methods inherited from class org.apache.hadoop.service.AbstractService

close, getBlockers, getConfig, getFailureCause, getFailureState, getLifecycleHistory, getName, getServiceState, getStartTime, init, isInState, noteFailure, putBlocker, registerGlobalListener, registerServiceListener, removeBlocker, serviceStart, setConfig, start, stop, toString, unregisterGlobalListener, unregisterServiceListener, waitForServiceToStop

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait

Field Detail

jobListCache

protected org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.JobListCache jobListCache

moveToDoneExecutor

@VisibleForTesting
protected ThreadPoolExecutor moveToDoneExecutor

Constructor Detail

HistoryFileManager

public HistoryFileManager()

Method Detail

serviceInit

protected void serviceInit(org.apache.hadoop.conf.Configuration conf)
                    throws Exception

Overrides:

serviceInit in class org.apache.hadoop.service.AbstractService

Throws:

Exception

createMoveToDoneThreadPool

protected ThreadPoolExecutor createMoveToDoneThreadPool(int numMoveThreads)

serviceStop

public void serviceStop()
                 throws Exception

Overrides:

serviceStop in class org.apache.hadoop.service.AbstractService

Throws:

Exception

createJobListCache

protected org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.JobListCache createJobListCache()

createHistoryFileInfo

protected HistoryFileManager.HistoryFileInfo createHistoryFileInfo(org.apache.hadoop.fs.Path historyFile,
                                                                   org.apache.hadoop.fs.Path confFile,
                                                                   org.apache.hadoop.fs.Path summaryFile,
                                                                   org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo jobIndexInfo,
                                                                   boolean isInDone)

scanDirectory

@VisibleForTesting
protected static List<org.apache.hadoop.fs.FileStatus> scanDirectory(org.apache.hadoop.fs.Path path,
                                                                                        org.apache.hadoop.fs.FileContext fc,
                                                                                        org.apache.hadoop.fs.PathFilter pathFilter)
                                                                                 throws IOException

Throws:

IOException

scanDirectoryForHistoryFiles

protected List<org.apache.hadoop.fs.FileStatus> scanDirectoryForHistoryFiles(org.apache.hadoop.fs.Path path,
                                                                             org.apache.hadoop.fs.FileContext fc)
                                                                      throws IOException

Throws:

IOException

findTimestampedDirectories

protected List<org.apache.hadoop.fs.FileStatus> findTimestampedDirectories()
                                                                    throws IOException

Finds all history directories with a timestamp component by scanning the filesystem. Used when the JobHistory server is started.

Returns:

list of history directories

Throws:

IOException

getAllFileInfo

public Collection<HistoryFileManager.HistoryFileInfo> getAllFileInfo()
                                                              throws IOException

Throws:

IOException

getFileInfo

public HistoryFileManager.HistoryFileInfo getFileInfo(org.apache.hadoop.mapreduce.v2.api.records.JobId jobId)
                                               throws IOException

Throws:

IOException

deleteDir

protected boolean deleteDir(org.apache.hadoop.fs.FileStatus serialDir)
                     throws org.apache.hadoop.security.AccessControlException,
                            FileNotFoundException,
                            org.apache.hadoop.fs.UnsupportedFileSystemException,
                            IOException

Throws:

org.apache.hadoop.security.AccessControlException

FileNotFoundException

org.apache.hadoop.fs.UnsupportedFileSystemException

IOException