Package org.apache.hadoop.yarn.logaggregation

Class LogAggregationUtils

java.lang.Object
org.apache.hadoop.yarn.logaggregation.LogAggregationUtils
@Private public class LogAggregationUtils extends Object

  • Field Summary

    Fields
    Modifier and Type
    Field
    Description
    static final String
    TMP_FILE_SUFFIX
     

  • Constructor Summary

    Constructors
    Constructor
    Description
    LogAggregationUtils()
     

  • Method Summary

    Modifier and Type
    Method
    Description
    static String
    getBucketSuffix()
    Returns the bucket suffix component of the log dir.
    static org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.FileStatus>
    getNodeFiles(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.Path remoteAppLogDir, org.apache.hadoop.yarn.api.records.ApplicationId appId, String appOwner)
    Get all available log files under remote app log directory.
    static String
    getNodeString(String nodeId)
     
    static String
    getNodeString(org.apache.hadoop.yarn.api.records.NodeId nodeId)
    Converts a nodeId to a form used in the app log file name.
    static org.apache.hadoop.fs.Path
    getOlderRemoteAppLogDir(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.yarn.api.records.ApplicationId appId, String user, org.apache.hadoop.fs.Path remoteRootLogDir, String suffix)
     
    static org.apache.hadoop.fs.Path
    getOlderRemoteAppLogDir(org.apache.hadoop.yarn.api.records.ApplicationId appId, String user, org.apache.hadoop.fs.Path remoteRootLogDir, String suffix)
    Gets the older remote app log dir.
    static org.apache.hadoop.fs.Path
    getOlderRemoteLogSuffixedDir(org.apache.hadoop.fs.Path remoteRootLogDir, String user, String suffix)
    Gets the older remote suffixed log dir for the user.
    static org.apache.hadoop.fs.Path
    getRemoteAppLogDir(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.yarn.api.records.ApplicationId appId, String appOwner, org.apache.hadoop.fs.Path remoteRootLogDir, String suffix)
    Return the remote application log directory.
    static org.apache.hadoop.fs.Path
    getRemoteAppLogDir(org.apache.hadoop.fs.Path remoteRootLogDir, org.apache.hadoop.yarn.api.records.ApplicationId appId, String user, String suffix)
    Gets the remote app log dir.
    static org.apache.hadoop.fs.Path
    getRemoteBucketDir(org.apache.hadoop.fs.Path remoteRootLogDir, String user, String suffix, org.apache.hadoop.yarn.api.records.ApplicationId appId)
    Gets the remote log user's bucket dir.
    static org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.FileStatus>
    getRemoteFiles(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.Path appPath)
     
    static org.apache.hadoop.fs.Path
    getRemoteLogSuffixedDir(org.apache.hadoop.fs.Path remoteRootLogDir, String user, String suffix)
    Gets the remote suffixed log dir for the user.
    static org.apache.hadoop.fs.Path
    getRemoteLogUserDir(org.apache.hadoop.fs.Path remoteRootLogDir, String user)
    Gets the remote log user dir.
    static org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.FileStatus>
    getRemoteNodeFileDir(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.yarn.api.records.ApplicationId appId, String appOwner, org.apache.hadoop.fs.Path remoteRootLogDir, String suffix)
    Get all available log files under remote app log directory.
    static List<org.apache.hadoop.fs.FileStatus>
    getRemoteNodeFileList(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.yarn.api.records.ApplicationId appId, String appOwner, org.apache.hadoop.fs.Path remoteRootLogDir, String suffix)
    Get all available log files under remote app log directory.
    static org.apache.hadoop.fs.Path
    getRemoteNodeLogFileForApp(org.apache.hadoop.fs.Path remoteRootLogDir, org.apache.hadoop.yarn.api.records.ApplicationId appId, String user, org.apache.hadoop.yarn.api.records.NodeId nodeId, String suffix)
    Constructs the full filename for an application's log file per node.
    static org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.FileStatus>
    getUserRemoteLogDir(org.apache.hadoop.conf.Configuration conf, String user, org.apache.hadoop.fs.Path remoteRootLogDir, String remoteRootLogDirSuffix)
     
    static boolean
    isOlderPathEnabled(org.apache.hadoop.conf.Configuration conf)
    Check if older Application Log Directory has to be included.

    Methods inherited from class java.lang.Object

    clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

  • Field Details

  • Constructor Details

    • LogAggregationUtils

      public LogAggregationUtils()

  • Method Details

    • getRemoteNodeLogFileForApp

      public static org.apache.hadoop.fs.Path getRemoteNodeLogFileForApp(org.apache.hadoop.fs.Path remoteRootLogDir, org.apache.hadoop.yarn.api.records.ApplicationId appId, String user, org.apache.hadoop.yarn.api.records.NodeId nodeId, String suffix)
      Constructs the full filename for an application's log file per node.
      Parameters:
      remoteRootLogDir - the aggregated remote root log dir
      appId - the application Id
      user - the application owner
      nodeId - the node id
      suffix - the log dir suffix
      Returns:
      the remote log file.

    • getRemoteAppLogDir

      public static org.apache.hadoop.fs.Path getRemoteAppLogDir(org.apache.hadoop.fs.Path remoteRootLogDir, org.apache.hadoop.yarn.api.records.ApplicationId appId, String user, String suffix)
      Gets the remote app log dir.
      Parameters:
      remoteRootLogDir - the aggregated log remote root log dir
      appId - the application id
      user - the application owner
      suffix - the log directory suffix
      Returns:
      the remote application specific log dir.

    • getOlderRemoteAppLogDir

      public static org.apache.hadoop.fs.Path getOlderRemoteAppLogDir(org.apache.hadoop.yarn.api.records.ApplicationId appId, String user, org.apache.hadoop.fs.Path remoteRootLogDir, String suffix)
      Gets the older remote app log dir.
      Parameters:
      appId - the application id
      user - the application owner
      remoteRootLogDir - the aggregated log remote root log dir
      suffix - the log directory suffix
      Returns:
      the remote application specific log dir.

    • getOlderRemoteAppLogDir

      public static org.apache.hadoop.fs.Path getOlderRemoteAppLogDir(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.yarn.api.records.ApplicationId appId, String user, org.apache.hadoop.fs.Path remoteRootLogDir, String suffix) throws IOException
      Throws:
      IOException

    • getRemoteLogSuffixedDir

      public static org.apache.hadoop.fs.Path getRemoteLogSuffixedDir(org.apache.hadoop.fs.Path remoteRootLogDir, String user, String suffix)
      Gets the remote suffixed log dir for the user.
      Parameters:
      remoteRootLogDir - the aggregated log remote root log dir
      user - the application owner
      suffix - the log dir suffix
      Returns:
      the remote suffixed log dir.

    • getOlderRemoteLogSuffixedDir

      public static org.apache.hadoop.fs.Path getOlderRemoteLogSuffixedDir(org.apache.hadoop.fs.Path remoteRootLogDir, String user, String suffix)
      Gets the older remote suffixed log dir for the user.
      Parameters:
      remoteRootLogDir - the aggregated log remote root log dir
      user - the application owner
      suffix - the log dir suffix
      Returns:
      the older remote suffixed log dir.

    • getRemoteLogUserDir

      public static org.apache.hadoop.fs.Path getRemoteLogUserDir(org.apache.hadoop.fs.Path remoteRootLogDir, String user)
      Gets the remote log user dir.
      Parameters:
      remoteRootLogDir - the aggregated log remote root log dir
      user - the application owner
      Returns:
      the remote per user log dir.

    • getRemoteBucketDir

      public static org.apache.hadoop.fs.Path getRemoteBucketDir(org.apache.hadoop.fs.Path remoteRootLogDir, String user, String suffix, org.apache.hadoop.yarn.api.records.ApplicationId appId)
      Gets the remote log user's bucket dir.
      Parameters:
      remoteRootLogDir - the aggregated log remote root log dir
      user - the application owner
      suffix - the log dir suffix
      appId - the application id
      Returns:
      the remote log per user per cluster timestamp per bucket dir.

    • isOlderPathEnabled

      public static boolean isOlderPathEnabled(org.apache.hadoop.conf.Configuration conf)
      Check if older Application Log Directory has to be included.
      Parameters:
      conf - the configuration
      Returns:
      Is Older App Log Dir enabled?

    • getBucketSuffix

      public static String getBucketSuffix()
      Returns the bucket suffix component of the log dir.
      Returns:
      the bucket suffix which appended to user log dir

    • getNodeString

      @VisibleForTesting public static String getNodeString(org.apache.hadoop.yarn.api.records.NodeId nodeId)
      Converts a nodeId to a form used in the app log file name.
      Parameters:
      nodeId - the nodeId
      Returns:
      the node string to be used to construct the file name.

    • getNodeString

      @VisibleForTesting public static String getNodeString(String nodeId)

    • getRemoteAppLogDir

      public static org.apache.hadoop.fs.Path getRemoteAppLogDir(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.yarn.api.records.ApplicationId appId, String appOwner, org.apache.hadoop.fs.Path remoteRootLogDir, String suffix) throws IOException
      Return the remote application log directory.
      Parameters:
      conf - the configuration
      appId - the application
      appOwner - the application owner
      remoteRootLogDir - the remote root log directory
      suffix - the log directory suffix
      Returns:
      the remote application log directory path
      Throws:
      IOException - if we can not find remote application log directory

    • getNodeFiles

      public static org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.FileStatus> getNodeFiles(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.Path remoteAppLogDir, org.apache.hadoop.yarn.api.records.ApplicationId appId, String appOwner) throws IOException
      Get all available log files under remote app log directory.
      Parameters:
      conf - the configuration
      remoteAppLogDir - the application log directory
      appId - the applicationId
      appOwner - the application owner
      Returns:
      the iterator of available log files
      Throws:
      IOException - if there is no log file directory

    • getRemoteNodeFileDir

      public static org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.FileStatus> getRemoteNodeFileDir(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.yarn.api.records.ApplicationId appId, String appOwner, org.apache.hadoop.fs.Path remoteRootLogDir, String suffix) throws IOException
      Get all available log files under remote app log directory.
      Parameters:
      conf - the configuration
      appId - the applicationId
      appOwner - the application owner
      remoteRootLogDir - the remote root log directory
      suffix - the log directory suffix
      Returns:
      the iterator of available log files
      Throws:
      IOException - if there is no log file available

    • getRemoteNodeFileList

      public static List<org.apache.hadoop.fs.FileStatus> getRemoteNodeFileList(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.yarn.api.records.ApplicationId appId, String appOwner, org.apache.hadoop.fs.Path remoteRootLogDir, String suffix) throws IOException
      Get all available log files under remote app log directory.
      Parameters:
      conf - the configuration
      appId - the applicationId
      appOwner - the application owner
      remoteRootLogDir - the remote root log directory
      suffix - the log directory suffix
      Returns:
      the list of available log files
      Throws:
      IOException - if there is no log file available

    • getRemoteFiles

      public static org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.FileStatus> getRemoteFiles(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.Path appPath) throws IOException
      Throws:
      IOException

    • getUserRemoteLogDir

      public static org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.FileStatus> getUserRemoteLogDir(org.apache.hadoop.conf.Configuration conf, String user, org.apache.hadoop.fs.Path remoteRootLogDir, String remoteRootLogDirSuffix) throws IOException
      Throws:
      IOException