org.apache.hadoop.mapred
Class JobLocalizer

java.lang.Object
  extended by org.apache.hadoop.mapred.JobLocalizer

public class JobLocalizer
extends Object

Internal class responsible for initializing the job, not intended for users. Creates the following hierarchy:

  • $mapred.local.dir/taskTracker/$user
  • $mapred.local.dir/taskTracker/$user/jobcache
  • $mapred.local.dir/taskTracker/$user/jobcache/$jobid/work
  • $mapred.local.dir/taskTracker/$user/jobcache/$jobid/jars
  • $mapred.local.dir/taskTracker/$user/jobcache/$jobid/jars/job.jar
  • $mapred.local.dir/taskTracker/$user/jobcache/$jobid/job.xml
  • $mapred.local.dir/taskTracker/$user/jobcache/$jobid/jobToken
  • $mapred.local.dir/taskTracker/$user/distcache

  • Field Summary
    protected static String JOB_LOCAL_CTXT
               
    protected  JobConf ttConf
               
     
    Constructor Summary
    JobLocalizer(JobConf ttConf, String user, String jobid)
               
    JobLocalizer(JobConf ttConf, String user, String jobid, String... localDirs)
               
     
    Method Summary
     void createJobDirs()
              Prepare the job directories for a given job.
     void createLocalDirs()
               
     void createUserDirs()
              Initialize the local directories for a particular user on this TT.
     void createWorkDir(JobConf jConf)
               
    static long[] downloadPrivateCache(Configuration conf)
              Download the parts of the distributed cache that are private.
     Path findCredentials()
               
     void initializeJobLogDir()
              Create job log directory and set appropriate permissions for the directory.
     void localizeJobFiles(JobID jobid, JobConf jConf, Path localJobFile, Path localJobTokenFile, TaskUmbilicalProtocol taskTracker)
               
     void localizeJobFiles(JobID jobid, JobConf jConf, Path localJobTokenFile, TaskUmbilicalProtocol taskTracker)
               
    static void main(String[] argv)
               
     int runSetup(String user, String jobid, Path localJobTokenFile, TaskUmbilicalProtocol taskTracker)
               
    static void writeLocalJobFile(Path jobFile, JobConf conf)
              Write the task specific job-configuration file.
     
    Methods inherited from class java.lang.Object
    clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
     

    Field Detail

    ttConf

    protected final JobConf ttConf

    JOB_LOCAL_CTXT

    protected static final String JOB_LOCAL_CTXT
    See Also:
    Constant Field Values
    Constructor Detail

    JobLocalizer

    public JobLocalizer(JobConf ttConf,
                        String user,
                        String jobid)
                 throws IOException
    Throws:
    IOException

    JobLocalizer

    public JobLocalizer(JobConf ttConf,
                        String user,
                        String jobid,
                        String... localDirs)
                 throws IOException
    Throws:
    IOException
    Method Detail

    createLocalDirs

    public void createLocalDirs()
                         throws IOException
    Throws:
    IOException

    createUserDirs

    public void createUserDirs()
                        throws IOException
    Initialize the local directories for a particular user on this TT. This involves creation and setting permissions of the following directories

    Throws:
    IOException

    createJobDirs

    public void createJobDirs()
                       throws IOException
    Prepare the job directories for a given job. To be called by the job localization code, only if the job is not already localized.
    Here, we set 700 permissions on the job directories created on all disks. This we do so as to avoid any misuse by other users till the time TaskController.initializeJob(java.lang.String, java.lang.String, org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path, org.apache.hadoop.mapred.TaskUmbilicalProtocol, java.net.InetSocketAddress) is run at a later time to set proper private permissions on the job directories.

    Throws:
    IOException

    initializeJobLogDir

    public void initializeJobLogDir()
                             throws IOException
    Create job log directory and set appropriate permissions for the directory.

    Throws:
    IOException

    downloadPrivateCache

    public static long[] downloadPrivateCache(Configuration conf)
                                       throws IOException
    Download the parts of the distributed cache that are private.

    Parameters:
    conf - the job's configuration
    Returns:
    the size of the archive objects
    Throws:
    IOException

    localizeJobFiles

    public void localizeJobFiles(JobID jobid,
                                 JobConf jConf,
                                 Path localJobTokenFile,
                                 TaskUmbilicalProtocol taskTracker)
                          throws IOException,
                                 InterruptedException
    Throws:
    IOException
    InterruptedException

    localizeJobFiles

    public void localizeJobFiles(JobID jobid,
                                 JobConf jConf,
                                 Path localJobFile,
                                 Path localJobTokenFile,
                                 TaskUmbilicalProtocol taskTracker)
                          throws IOException,
                                 InterruptedException
    Throws:
    IOException
    InterruptedException

    createWorkDir

    public void createWorkDir(JobConf jConf)
                       throws IOException
    Throws:
    IOException

    findCredentials

    public Path findCredentials()
                         throws IOException
    Throws:
    IOException

    runSetup

    public int runSetup(String user,
                        String jobid,
                        Path localJobTokenFile,
                        TaskUmbilicalProtocol taskTracker)
                 throws IOException,
                        InterruptedException
    Throws:
    IOException
    InterruptedException

    main

    public static void main(String[] argv)
                     throws IOException,
                            InterruptedException
    Throws:
    IOException
    InterruptedException

    writeLocalJobFile

    public static void writeLocalJobFile(Path jobFile,
                                         JobConf conf)
                                  throws IOException
    Write the task specific job-configuration file.

    Throws:
    IOException


    Copyright © 2009 The Apache Software Foundation