001    /**
002    * Licensed to the Apache Software Foundation (ASF) under one
003    * or more contributor license agreements.  See the NOTICE file
004    * distributed with this work for additional information
005    * regarding copyright ownership.  The ASF licenses this file
006    * to you under the Apache License, Version 2.0 (the
007    * "License"); you may not use this file except in compliance
008    * with the License.  You may obtain a copy of the License at
009    *
010    *     http://www.apache.org/licenses/LICENSE-2.0
011    *
012    * Unless required by applicable law or agreed to in writing, software
013    * distributed under the License is distributed on an "AS IS" BASIS,
014    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015    * See the License for the specific language governing permissions and
016    * limitations under the License.
017    */
018    
019    package org.apache.hadoop.yarn.client.cli;
020    
021    import java.io.IOException;
022    
023    import org.apache.commons.cli.CommandLine;
024    import org.apache.commons.cli.CommandLineParser;
025    import org.apache.commons.cli.GnuParser;
026    import org.apache.commons.cli.HelpFormatter;
027    import org.apache.commons.cli.Option;
028    import org.apache.commons.cli.Options;
029    import org.apache.commons.cli.ParseException;
030    import org.apache.hadoop.classification.InterfaceAudience.Public;
031    import org.apache.hadoop.classification.InterfaceStability.Evolving;
032    import org.apache.hadoop.conf.Configuration;
033    import org.apache.hadoop.conf.Configured;
034    import org.apache.hadoop.fs.Path;
035    import org.apache.hadoop.security.UserGroupInformation;
036    import org.apache.hadoop.util.Tool;
037    import org.apache.hadoop.yarn.api.records.ApplicationId;
038    import org.apache.hadoop.yarn.api.records.ApplicationReport;
039    import org.apache.hadoop.yarn.client.api.YarnClient;
040    import org.apache.hadoop.yarn.conf.YarnConfiguration;
041    import org.apache.hadoop.yarn.exceptions.YarnException;
042    import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat;
043    import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils;
044    import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers;
045    import org.apache.hadoop.yarn.util.ConverterUtils;
046    
047    import com.google.common.annotations.VisibleForTesting;
048    
049    @Public
050    @Evolving
051    public class LogsCLI extends Configured implements Tool {
052    
053      private static final String CONTAINER_ID_OPTION = "containerId";
054      private static final String APPLICATION_ID_OPTION = "applicationId";
055      private static final String NODE_ADDRESS_OPTION = "nodeAddress";
056      private static final String APP_OWNER_OPTION = "appOwner";
057    
058      @Override
059      public int run(String[] args) throws Exception {
060    
061        Options opts = new Options();
062        Option appIdOpt = new Option(APPLICATION_ID_OPTION, true, "ApplicationId (required)");
063        appIdOpt.setRequired(true);
064        opts.addOption(appIdOpt);
065        opts.addOption(CONTAINER_ID_OPTION, true,
066          "ContainerId (must be specified if node address is specified)");
067        opts.addOption(NODE_ADDRESS_OPTION, true, "NodeAddress in the format "
068          + "nodename:port (must be specified if container id is specified)");
069        opts.addOption(APP_OWNER_OPTION, true,
070          "AppOwner (assumed to be current user if not specified)");
071        opts.getOption(APPLICATION_ID_OPTION).setArgName("Application ID");
072        opts.getOption(CONTAINER_ID_OPTION).setArgName("Container ID");
073        opts.getOption(NODE_ADDRESS_OPTION).setArgName("Node Address");
074        opts.getOption(APP_OWNER_OPTION).setArgName("Application Owner");
075    
076        Options printOpts = new Options();
077        printOpts.addOption(opts.getOption(CONTAINER_ID_OPTION));
078        printOpts.addOption(opts.getOption(NODE_ADDRESS_OPTION));
079        printOpts.addOption(opts.getOption(APP_OWNER_OPTION));
080    
081        if (args.length < 1) {
082          printHelpMessage(printOpts);
083          return -1;
084        }
085    
086        CommandLineParser parser = new GnuParser();
087        String appIdStr = null;
088        String containerIdStr = null;
089        String nodeAddress = null;
090        String appOwner = null;
091        try {
092          CommandLine commandLine = parser.parse(opts, args, true);
093          appIdStr = commandLine.getOptionValue(APPLICATION_ID_OPTION);
094          containerIdStr = commandLine.getOptionValue(CONTAINER_ID_OPTION);
095          nodeAddress = commandLine.getOptionValue(NODE_ADDRESS_OPTION);
096          appOwner = commandLine.getOptionValue(APP_OWNER_OPTION);
097        } catch (ParseException e) {
098          System.err.println("options parsing failed: " + e.getMessage());
099          printHelpMessage(printOpts);
100          return -1;
101        }
102    
103        if (appIdStr == null) {
104          System.err.println("ApplicationId cannot be null!");
105          printHelpMessage(printOpts);
106          return -1;
107        }
108    
109        ApplicationId appId = null;
110        try {
111          appId = ConverterUtils.toApplicationId(appIdStr);
112        } catch (Exception e) {
113          System.err.println("Invalid ApplicationId specified");
114          return -1;
115        }
116        
117        try {
118          int resultCode = verifyApplicationState(appId);
119          if (resultCode != 0) {
120            System.out.println("Application has not completed." +
121                            " Logs are only available after an application completes");
122            return resultCode;
123          }
124        } catch (Exception e) {
125          System.err.println("Unable to get ApplicationState." +
126                    " Attempting to fetch logs directly from the filesystem.");
127        }
128    
129        LogCLIHelpers logCliHelper = new LogCLIHelpers();
130        logCliHelper.setConf(getConf());
131        
132        if (appOwner == null || appOwner.isEmpty()) {
133          appOwner = UserGroupInformation.getCurrentUser().getShortUserName();
134        }
135        int resultCode = 0;
136        if (containerIdStr == null && nodeAddress == null) {
137          resultCode = logCliHelper.dumpAllContainersLogs(appId, appOwner, System.out);
138        } else if ((containerIdStr == null && nodeAddress != null)
139            || (containerIdStr != null && nodeAddress == null)) {
140          System.out.println("ContainerId or NodeAddress cannot be null!");
141          printHelpMessage(printOpts);
142          resultCode = -1;
143        } else {
144          Path remoteRootLogDir =
145            new Path(getConf().get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
146                YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));
147          AggregatedLogFormat.LogReader reader =
148              new AggregatedLogFormat.LogReader(getConf(),
149                  LogAggregationUtils.getRemoteNodeLogFileForApp(
150                      remoteRootLogDir,
151                      appId,
152                      appOwner,
153                      ConverterUtils.toNodeId(nodeAddress),
154                      LogAggregationUtils.getRemoteNodeLogDirSuffix(getConf())));
155          resultCode = logCliHelper.dumpAContainerLogs(containerIdStr, reader, System.out);
156        }
157    
158        return resultCode;
159      }
160    
161      private int verifyApplicationState(ApplicationId appId) throws IOException,
162          YarnException {
163        YarnClient yarnClient = createYarnClient();
164    
165        try {
166          ApplicationReport appReport = yarnClient.getApplicationReport(appId);
167          switch (appReport.getYarnApplicationState()) {
168          case NEW:
169          case NEW_SAVING:
170          case ACCEPTED:
171          case SUBMITTED:
172          case RUNNING:
173            return -1;
174          case FAILED:
175          case FINISHED:
176          case KILLED:
177          default:
178            break;
179    
180          }
181        } finally {
182          yarnClient.close();
183        }
184        return 0;
185      }
186      
187      @VisibleForTesting
188      protected YarnClient createYarnClient() {
189        YarnClient yarnClient = YarnClient.createYarnClient();
190        yarnClient.init(getConf());
191        yarnClient.start();
192        return yarnClient;
193      }
194    
195      public static void main(String[] args) throws Exception {
196        Configuration conf = new YarnConfiguration();
197        LogsCLI logDumper = new LogsCLI();
198        logDumper.setConf(conf);
199        int exitCode = logDumper.run(args);
200        System.exit(exitCode);
201      }
202    
203      private void printHelpMessage(Options options) {
204        System.out.println("Retrieve logs for completed YARN applications.");
205        HelpFormatter formatter = new HelpFormatter();
206        formatter.printHelp("yarn logs -applicationId <application ID> [OPTIONS]", new Options());
207        formatter.setSyntaxPrefix("");
208        formatter.printHelp("general options are:", options);
209      }
210    }