001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.util;
019
020import java.io.BufferedReader;
021import java.io.File;
022import java.io.FileNotFoundException;
023import java.io.IOException;
024import java.io.InputStreamReader;
025import java.io.InputStream;
026import java.io.InterruptedIOException;
027import java.nio.charset.Charset;
028import java.util.Arrays;
029import java.util.Map;
030import java.util.Timer;
031import java.util.TimerTask;
032import java.util.concurrent.atomic.AtomicBoolean;
033
034import com.google.common.annotations.VisibleForTesting;
035import org.apache.hadoop.classification.InterfaceAudience;
036import org.apache.hadoop.classification.InterfaceStability;
037import org.apache.hadoop.security.alias.AbstractJavaKeyStoreProvider;
038import org.slf4j.Logger;
039import org.slf4j.LoggerFactory;
040
041/**
042 * A base class for running a Shell command.
043 *
044 * <code>Shell</code> can be used to run shell commands like <code>du</code> or
045 * <code>df</code>. It also offers facilities to gate commands by
046 * time-intervals.
047 */
048@InterfaceAudience.Public
049@InterfaceStability.Evolving
050public abstract class Shell {
051  public static final Logger LOG = LoggerFactory.getLogger(Shell.class);
052
053  /**
054   * Text to include when there are windows-specific problems.
055   * {@value}
056   */
057  private static final String WINDOWS_PROBLEMS =
058      "https://wiki.apache.org/hadoop/WindowsProblems";
059
060  /**
061   * Name of the windows utils binary: {@value}.
062   */
063  static final String WINUTILS_EXE = "winutils.exe";
064
065  /**
066   * System property for the Hadoop home directory: {@value}.
067   */
068  public static final String SYSPROP_HADOOP_HOME_DIR = "hadoop.home.dir";
069
070  /**
071   * Environment variable for Hadoop's home dir: {@value}.
072   */
073  public static final String ENV_HADOOP_HOME = "HADOOP_HOME";
074
075  /**
076   * query to see if system is Java 7 or later.
077   * Now that Hadoop requires Java 7 or later, this always returns true.
078   * @deprecated This call isn't needed any more: please remove uses of it.
079   * @return true, always.
080   */
081  @Deprecated
082  public static boolean isJava7OrAbove() {
083    return true;
084  }
085
086  /**
087   * Maximum command line length in Windows
088   * KB830473 documents this as 8191
089   */
090  public static final int WINDOWS_MAX_SHELL_LENGTH = 8191;
091
092  /**
093   * mis-spelling of {@link #WINDOWS_MAX_SHELL_LENGTH}.
094   * @deprecated use the correctly spelled constant.
095   */
096  @Deprecated
097  public static final int WINDOWS_MAX_SHELL_LENGHT = WINDOWS_MAX_SHELL_LENGTH;
098
099  /**
100   * Checks if a given command (String[]) fits in the Windows maximum command
101   * line length Note that the input is expected to already include space
102   * delimiters, no extra count will be added for delimiters.
103   *
104   * @param commands command parts, including any space delimiters
105   */
106  public static void checkWindowsCommandLineLength(String...commands)
107      throws IOException {
108    int len = 0;
109    for (String s: commands) {
110      len += s.length();
111    }
112    if (len > WINDOWS_MAX_SHELL_LENGTH) {
113      throw new IOException(String.format(
114        "The command line has a length of %d exceeds maximum allowed length" +
115            " of %d. Command starts with: %s",
116        len, WINDOWS_MAX_SHELL_LENGTH,
117        StringUtils.join("", commands).substring(0, 100)));
118    }
119  }
120
121  /**
122   * Quote the given arg so that bash will interpret it as a single value.
123   * Note that this quotes it for one level of bash, if you are passing it
124   * into a badly written shell script, you need to fix your shell script.
125   * @param arg the argument to quote
126   * @return the quoted string
127   */
128  static String bashQuote(String arg) {
129    StringBuilder buffer = new StringBuilder(arg.length() + 2);
130    buffer.append('\'');
131    buffer.append(arg.replace("'", "'\\''"));
132    buffer.append('\'');
133    return buffer.toString();
134  }
135
136  /** a Unix command to get the current user's name: {@value}. */
137  public static final String USER_NAME_COMMAND = "whoami";
138
139  /** Windows <code>CreateProcess</code> synchronization object. */
140  public static final Object WindowsProcessLaunchLock = new Object();
141
142  // OSType detection
143
144  public enum OSType {
145    OS_TYPE_LINUX,
146    OS_TYPE_WIN,
147    OS_TYPE_SOLARIS,
148    OS_TYPE_MAC,
149    OS_TYPE_FREEBSD,
150    OS_TYPE_OTHER
151  }
152
153  /**
154   * Get the type of the operating system, as determined from parsing
155   * the <code>os.name</code> property.
156   */
157  public static final OSType osType = getOSType();
158
159  private static OSType getOSType() {
160    String osName = System.getProperty("os.name");
161    if (osName.startsWith("Windows")) {
162      return OSType.OS_TYPE_WIN;
163    } else if (osName.contains("SunOS") || osName.contains("Solaris")) {
164      return OSType.OS_TYPE_SOLARIS;
165    } else if (osName.contains("Mac")) {
166      return OSType.OS_TYPE_MAC;
167    } else if (osName.contains("FreeBSD")) {
168      return OSType.OS_TYPE_FREEBSD;
169    } else if (osName.startsWith("Linux")) {
170      return OSType.OS_TYPE_LINUX;
171    } else {
172      // Some other form of Unix
173      return OSType.OS_TYPE_OTHER;
174    }
175  }
176
177  // Helper static vars for each platform
178  public static final boolean WINDOWS = (osType == OSType.OS_TYPE_WIN);
179  public static final boolean SOLARIS = (osType == OSType.OS_TYPE_SOLARIS);
180  public static final boolean MAC     = (osType == OSType.OS_TYPE_MAC);
181  public static final boolean FREEBSD = (osType == OSType.OS_TYPE_FREEBSD);
182  public static final boolean LINUX   = (osType == OSType.OS_TYPE_LINUX);
183  public static final boolean OTHER   = (osType == OSType.OS_TYPE_OTHER);
184
185  public static final boolean PPC_64
186                = System.getProperties().getProperty("os.arch").contains("ppc64");
187
188  /** a Unix command to get the current user's groups list. */
189  public static String[] getGroupsCommand() {
190    return (WINDOWS)? new String[]{"cmd", "/c", "groups"}
191                    : new String[]{"groups"};
192  }
193
194  /**
195   * A command to get a given user's groups list.
196   * If the OS is not WINDOWS, the command will get the user's primary group
197   * first and finally get the groups list which includes the primary group.
198   * i.e. the user's primary group will be included twice.
199   */
200  public static String[] getGroupsForUserCommand(final String user) {
201    //'groups username' command return is inconsistent across different unixes
202    if (WINDOWS) {
203      return new String[]
204          {getWinUtilsPath(), "groups", "-F", "\"" + user + "\""};
205    } else {
206      String quotedUser = bashQuote(user);
207      return new String[] {"bash", "-c", "id -gn " + quotedUser +
208                            "; id -Gn " + quotedUser};
209    }
210  }
211
212  /**
213   * A command to get a given user's group id list.
214   * The command will get the user's primary group
215   * first and finally get the groups list which includes the primary group.
216   * i.e. the user's primary group will be included twice.
217   * This command does not support Windows and will only return group names.
218   */
219  public static String[] getGroupsIDForUserCommand(final String user) {
220    //'groups username' command return is inconsistent across different unixes
221    if (WINDOWS) {
222      return new String[]{getWinUtilsPath(), "groups", "-F", "\"" + user +
223                           "\""};
224    } else {
225      String quotedUser = bashQuote(user);
226      return new String[] {"bash", "-c", "id -g " + quotedUser + "; id -G " +
227                            quotedUser};
228    }
229  }
230
231  /** A command to get a given netgroup's user list. */
232  public static String[] getUsersForNetgroupCommand(final String netgroup) {
233    //'groups username' command return is non-consistent across different unixes
234    return new String[] {"getent", "netgroup", netgroup};
235  }
236
237  /** Return a command to get permission information. */
238  public static String[] getGetPermissionCommand() {
239    return (WINDOWS) ? new String[] { getWinUtilsPath(), "ls", "-F" }
240                     : new String[] { "ls", "-ld" };
241  }
242
243  /** Return a command to set permission. */
244  public static String[] getSetPermissionCommand(String perm, boolean recursive) {
245    if (recursive) {
246      return (WINDOWS) ?
247          new String[] { getWinUtilsPath(), "chmod", "-R", perm }
248          : new String[] { "chmod", "-R", perm };
249    } else {
250      return (WINDOWS) ?
251          new String[] { getWinUtilsPath(), "chmod", perm }
252          : new String[] { "chmod", perm };
253    }
254  }
255
256  /**
257   * Return a command to set permission for specific file.
258   *
259   * @param perm String permission to set
260   * @param recursive boolean true to apply to all sub-directories recursively
261   * @param file String file to set
262   * @return String[] containing command and arguments
263   */
264  public static String[] getSetPermissionCommand(String perm,
265                                                 boolean recursive,
266                                                 String file) {
267    String[] baseCmd = getSetPermissionCommand(perm, recursive);
268    String[] cmdWithFile = Arrays.copyOf(baseCmd, baseCmd.length + 1);
269    cmdWithFile[cmdWithFile.length - 1] = file;
270    return cmdWithFile;
271  }
272
273  /** Return a command to set owner. */
274  public static String[] getSetOwnerCommand(String owner) {
275    return (WINDOWS) ?
276        new String[] { getWinUtilsPath(), "chown", "\"" + owner + "\"" }
277        : new String[] { "chown", owner };
278  }
279
280  /** Return a command to create symbolic links. */
281  public static String[] getSymlinkCommand(String target, String link) {
282    return WINDOWS ?
283       new String[] { getWinUtilsPath(), "symlink", link, target }
284       : new String[] { "ln", "-s", target, link };
285  }
286
287  /** Return a command to read the target of the a symbolic link. */
288  public static String[] getReadlinkCommand(String link) {
289    return WINDOWS ?
290        new String[] { getWinUtilsPath(), "readlink", link }
291        : new String[] { "readlink", link };
292  }
293
294  /**
295   * Return a command for determining if process with specified pid is alive.
296   * @param pid process ID
297   * @return a <code>kill -0</code> command or equivalent
298   */
299  public static String[] getCheckProcessIsAliveCommand(String pid) {
300    return getSignalKillCommand(0, pid);
301  }
302
303  /** Return a command to send a signal to a given pid. */
304  public static String[] getSignalKillCommand(int code, String pid) {
305    // Code == 0 means check alive
306    if (Shell.WINDOWS) {
307      if (0 == code) {
308        return new String[] {Shell.getWinUtilsPath(), "task", "isAlive", pid };
309      } else {
310        return new String[] {Shell.getWinUtilsPath(), "task", "kill", pid };
311      }
312    }
313
314    // Use the bash-builtin instead of the Unix kill command (usually
315    // /bin/kill) as the bash-builtin supports "--" in all Hadoop supported
316    // OSes.
317    final String quotedPid = bashQuote(pid);
318    if (isSetsidAvailable) {
319      return new String[] { "bash", "-c", "kill -" + code + " -- -" +
320          quotedPid };
321    } else {
322      return new String[] { "bash", "-c", "kill -" + code + " " +
323          quotedPid };
324    }
325  }
326
327  /** Regular expression for environment variables: {@value}. */
328  public static final String ENV_NAME_REGEX = "[A-Za-z_][A-Za-z0-9_]*";
329
330  /** Return a regular expression string that match environment variables. */
331  public static String getEnvironmentVariableRegex() {
332    return (WINDOWS)
333        ? "%(" + ENV_NAME_REGEX + "?)%"
334        : "\\$(" + ENV_NAME_REGEX + ")";
335  }
336
337  /**
338   * Returns a File referencing a script with the given basename, inside the
339   * given parent directory.  The file extension is inferred by platform:
340   * <code>".cmd"</code> on Windows, or <code>".sh"</code> otherwise.
341   *
342   * @param parent File parent directory
343   * @param basename String script file basename
344   * @return File referencing the script in the directory
345   */
346  public static File appendScriptExtension(File parent, String basename) {
347    return new File(parent, appendScriptExtension(basename));
348  }
349
350  /**
351   * Returns a script file name with the given basename.
352   *
353   * The file extension is inferred by platform:
354   * <code>".cmd"</code> on Windows, or <code>".sh"</code> otherwise.
355   *
356   * @param basename String script file basename
357   * @return String script file name
358   */
359  public static String appendScriptExtension(String basename) {
360    return basename + (WINDOWS ? ".cmd" : ".sh");
361  }
362
363  /**
364   * Returns a command to run the given script.  The script interpreter is
365   * inferred by platform: cmd on Windows or bash otherwise.
366   *
367   * @param script File script to run
368   * @return String[] command to run the script
369   */
370  public static String[] getRunScriptCommand(File script) {
371    String absolutePath = script.getAbsolutePath();
372    return WINDOWS ?
373      new String[] {"cmd", "/c", absolutePath }
374      : new String[] {"bash", bashQuote(absolutePath) };
375  }
376
377  /** a Unix command to set permission: {@value}. */
378  public static final String SET_PERMISSION_COMMAND = "chmod";
379  /** a Unix command to set owner: {@value}. */
380  public static final String SET_OWNER_COMMAND = "chown";
381
382  /** a Unix command to set the change user's groups list: {@value}. */
383  public static final String SET_GROUP_COMMAND = "chgrp";
384  /** a Unix command to create a link: {@value}. */
385  public static final String LINK_COMMAND = "ln";
386  /** a Unix command to get a link target: {@value}. */
387  public static final String READ_LINK_COMMAND = "readlink";
388
389  /**Time after which the executing script would be timedout. */
390  protected long timeOutInterval = 0L;
391  /** If or not script timed out*/
392  private final AtomicBoolean timedOut = new AtomicBoolean(false);
393
394  /** Indicates if the parent env vars should be inherited or not*/
395  protected boolean inheritParentEnv = true;
396
397  /**
398   *  Centralized logic to discover and validate the sanity of the Hadoop
399   *  home directory.
400   *
401   *  This does a lot of work so it should only be called
402   *  privately for initialization once per process.
403   *
404   * @return A directory that exists and via was specified on the command line
405   * via <code>-Dhadoop.home.dir</code> or the <code>HADOOP_HOME</code>
406   * environment variable.
407   * @throws FileNotFoundException if the properties are absent or the specified
408   * path is not a reference to a valid directory.
409   */
410  private static File checkHadoopHome() throws FileNotFoundException {
411
412    // first check the Dflag hadoop.home.dir with JVM scope
413    String home = System.getProperty(SYSPROP_HADOOP_HOME_DIR);
414
415    // fall back to the system/user-global env variable
416    if (home == null) {
417      home = System.getenv(ENV_HADOOP_HOME);
418    }
419    return checkHadoopHomeInner(home);
420  }
421
422  /*
423  A set of exception strings used to construct error messages;
424  these are referred to in tests
425  */
426  static final String E_DOES_NOT_EXIST = "does not exist";
427  static final String E_IS_RELATIVE = "is not an absolute path.";
428  static final String E_NOT_DIRECTORY = "is not a directory.";
429  static final String E_NO_EXECUTABLE = "Could not locate Hadoop executable";
430  static final String E_NOT_EXECUTABLE_FILE = "Not an executable file";
431  static final String E_HADOOP_PROPS_UNSET = ENV_HADOOP_HOME + " and "
432      + SYSPROP_HADOOP_HOME_DIR + " are unset.";
433  static final String E_HADOOP_PROPS_EMPTY = ENV_HADOOP_HOME + " or "
434      + SYSPROP_HADOOP_HOME_DIR + " set to an empty string";
435  static final String E_NOT_A_WINDOWS_SYSTEM = "Not a Windows system";
436
437  /**
438   *  Validate the accessibility of the Hadoop home directory.
439   *
440   * @return A directory that is expected to be the hadoop home directory
441   * @throws FileNotFoundException if the specified
442   * path is not a reference to a valid directory.
443   */
444  @VisibleForTesting
445  static File checkHadoopHomeInner(String home) throws FileNotFoundException {
446    // couldn't find either setting for hadoop's home directory
447    if (home == null) {
448      throw new FileNotFoundException(E_HADOOP_PROPS_UNSET);
449    }
450    // strip off leading and trailing double quotes
451    while (home.startsWith("\"")) {
452      home = home.substring(1);
453    }
454    while (home.endsWith("\"")) {
455      home = home.substring(0, home.length() - 1);
456    }
457
458    // after stripping any quotes, check for home dir being non-empty
459    if (home.isEmpty()) {
460      throw new FileNotFoundException(E_HADOOP_PROPS_EMPTY);
461    }
462
463    // check that the hadoop home dir value
464    // is an absolute reference to a directory
465    File homedir = new File(home);
466    if (!homedir.isAbsolute()) {
467      throw new FileNotFoundException("Hadoop home directory " + homedir
468          + " " + E_IS_RELATIVE);
469    }
470    if (!homedir.exists()) {
471      throw new FileNotFoundException("Hadoop home directory " + homedir
472          + " " + E_DOES_NOT_EXIST);
473    }
474    if (!homedir.isDirectory()) {
475      throw new FileNotFoundException("Hadoop home directory " + homedir
476          + " "+ E_NOT_DIRECTORY);
477    }
478    return homedir;
479  }
480
481  /**
482   * The Hadoop home directory.
483   */
484  private static final File HADOOP_HOME_FILE;
485
486  /**
487   * Rethrowable cause for the failure to determine the hadoop
488   * home directory
489   */
490  private static final IOException HADOOP_HOME_DIR_FAILURE_CAUSE;
491
492  static {
493    File home;
494    IOException ex;
495    try {
496      home = checkHadoopHome();
497      ex = null;
498    } catch (IOException ioe) {
499      if (LOG.isDebugEnabled()) {
500        LOG.debug("Failed to detect a valid hadoop home directory", ioe);
501      }
502      ex = ioe;
503      home = null;
504    }
505    HADOOP_HOME_FILE = home;
506    HADOOP_HOME_DIR_FAILURE_CAUSE = ex;
507  }
508
509  /**
510   * Optionally extend an error message with some OS-specific text.
511   * @param message core error message
512   * @return error message, possibly with some extra text
513   */
514  private static String addOsText(String message) {
515    return WINDOWS ? (message + " -see " + WINDOWS_PROBLEMS) : message;
516  }
517
518  /**
519   * Create a {@code FileNotFoundException} with the inner nested cause set
520   * to the given exception. Compensates for the fact that FNFE doesn't
521   * have an initializer that takes an exception.
522   * @param text error text
523   * @param ex inner exception
524   * @return a new exception to throw.
525   */
526  private static FileNotFoundException fileNotFoundException(String text,
527      Exception ex) {
528    return (FileNotFoundException) new FileNotFoundException(text)
529        .initCause(ex);
530  }
531
532  /**
533   * Get the Hadoop home directory. Raises an exception if not found
534   * @return the home dir
535   * @throws IOException if the home directory cannot be located.
536   */
537  public static String getHadoopHome() throws IOException {
538    return getHadoopHomeDir().getCanonicalPath();
539  }
540
541  /**
542   * Get the Hadoop home directory. If it is invalid,
543   * throw an exception.
544   * @return a path referring to hadoop home.
545   * @throws FileNotFoundException if the directory doesn't exist.
546   */
547  private static File getHadoopHomeDir() throws FileNotFoundException {
548    if (HADOOP_HOME_DIR_FAILURE_CAUSE != null) {
549      throw fileNotFoundException(
550          addOsText(HADOOP_HOME_DIR_FAILURE_CAUSE.toString()),
551          HADOOP_HOME_DIR_FAILURE_CAUSE);
552    }
553    return HADOOP_HOME_FILE;
554  }
555
556  /**
557   *  Fully qualify the path to a binary that should be in a known hadoop
558   *  bin location. This is primarily useful for disambiguating call-outs
559   *  to executable sub-components of Hadoop to avoid clashes with other
560   *  executables that may be in the path.  Caveat:  this call doesn't
561   *  just format the path to the bin directory.  It also checks for file
562   *  existence of the composed path. The output of this call should be
563   *  cached by callers.
564   *
565   * @param executable executable
566   * @return executable file reference
567   * @throws FileNotFoundException if the path does not exist
568   */
569  public static File getQualifiedBin(String executable)
570      throws FileNotFoundException {
571    // construct hadoop bin path to the specified executable
572    return getQualifiedBinInner(getHadoopHomeDir(), executable);
573  }
574
575  /**
576   * Inner logic of {@link #getQualifiedBin(String)}, accessible
577   * for tests.
578   * @param hadoopHomeDir home directory (assumed to be valid)
579   * @param executable executable
580   * @return path to the binary
581   * @throws FileNotFoundException if the executable was not found/valid
582   */
583  static File getQualifiedBinInner(File hadoopHomeDir, String executable)
584      throws FileNotFoundException {
585    String binDirText = "Hadoop bin directory ";
586    File bin = new File(hadoopHomeDir, "bin");
587    if (!bin.exists()) {
588      throw new FileNotFoundException(addOsText(binDirText + E_DOES_NOT_EXIST
589          + ": " + bin));
590    }
591    if (!bin.isDirectory()) {
592      throw new FileNotFoundException(addOsText(binDirText + E_NOT_DIRECTORY
593          + ": " + bin));
594    }
595
596    File exeFile = new File(bin, executable);
597    if (!exeFile.exists()) {
598      throw new FileNotFoundException(
599          addOsText(E_NO_EXECUTABLE + ": " + exeFile));
600    }
601    if (!exeFile.isFile()) {
602      throw new FileNotFoundException(
603          addOsText(E_NOT_EXECUTABLE_FILE + ": " + exeFile));
604    }
605    try {
606      return exeFile.getCanonicalFile();
607    } catch (IOException e) {
608      // this isn't going to happen, because of all the upfront checks.
609      // so if it does, it gets converted to a FNFE and rethrown
610      throw fileNotFoundException(e.toString(), e);
611    }
612  }
613
614  /**
615   *  Fully qualify the path to a binary that should be in a known hadoop
616   *  bin location. This is primarily useful for disambiguating call-outs
617   *  to executable sub-components of Hadoop to avoid clashes with other
618   *  executables that may be in the path.  Caveat:  this call doesn't
619   *  just format the path to the bin directory.  It also checks for file
620   *  existence of the composed path. The output of this call should be
621   *  cached by callers.
622   *
623   * @param executable executable
624   * @return executable file reference
625   * @throws FileNotFoundException if the path does not exist
626   * @throws IOException on path canonicalization failures
627   */
628  public static String getQualifiedBinPath(String executable)
629      throws IOException {
630    return getQualifiedBin(executable).getCanonicalPath();
631  }
632
633  /**
634   * Location of winutils as a string; null if not found.
635   * <p>
636   * <i>Important: caller must check for this value being null</i>.
637   * The lack of such checks has led to many support issues being raised.
638   * <p>
639   * @deprecated use one of the exception-raising getter methods,
640   * specifically {@link #getWinUtilsPath()} or {@link #getWinUtilsFile()}
641   */
642  @Deprecated
643  public static final String WINUTILS;
644
645  /** Canonical path to winutils, private to Shell. */
646  private static final String WINUTILS_PATH;
647
648  /** file reference to winutils. */
649  private static final File WINUTILS_FILE;
650
651  /** the exception raised on a failure to init the WINUTILS fields. */
652  private static final IOException WINUTILS_FAILURE;
653
654  /*
655   * Static WINUTILS_* field initializer.
656   * On non-Windows systems sets the paths to null, and
657   * adds a specific exception to the failure cause, so
658   * that on any attempt to resolve the paths will raise
659   * a meaningful exception.
660   */
661  static {
662    IOException ioe = null;
663    String path = null;
664    File file = null;
665    // invariant: either there's a valid file and path,
666    // or there is a cached IO exception.
667    if (WINDOWS) {
668      try {
669        file = getQualifiedBin(WINUTILS_EXE);
670        path = file.getCanonicalPath();
671        ioe = null;
672      } catch (IOException e) {
673        LOG.warn("Did not find {}: {}", WINUTILS_EXE, e);
674        // stack trace comes at debug level
675        LOG.debug("Failed to find " + WINUTILS_EXE, e);
676        file = null;
677        path = null;
678        ioe = e;
679      }
680    } else {
681      // on a non-windows system, the invariant is kept
682      // by adding an explicit exception.
683      ioe = new FileNotFoundException(E_NOT_A_WINDOWS_SYSTEM);
684    }
685    WINUTILS_PATH = path;
686    WINUTILS_FILE = file;
687
688    WINUTILS = path;
689    WINUTILS_FAILURE = ioe;
690  }
691
692  /**
693   * Predicate to indicate whether or not the path to winutils is known.
694   *
695   * If true, then {@link #WINUTILS} is non-null, and both
696   * {@link #getWinUtilsPath()} and {@link #getWinUtilsFile()}
697   * will successfully return this value. Always false on non-windows systems.
698   * @return true if there is a valid path to the binary
699   */
700  public static boolean hasWinutilsPath() {
701    return WINUTILS_PATH != null;
702  }
703
704  /**
705   * Locate the winutils binary, or fail with a meaningful
706   * exception and stack trace as an RTE.
707   * This method is for use in methods which don't explicitly throw
708   * an <code>IOException</code>.
709   * @return the path to {@link #WINUTILS_EXE}
710   * @throws RuntimeException if the path is not resolvable
711   */
712  public static String getWinUtilsPath() {
713    if (WINUTILS_FAILURE == null) {
714      return WINUTILS_PATH;
715    } else {
716      throw new RuntimeException(WINUTILS_FAILURE.toString(),
717          WINUTILS_FAILURE);
718    }
719  }
720
721  /**
722   * Get a file reference to winutils.
723   * Always raises an exception if there isn't one
724   * @return the file instance referring to the winutils bin.
725   * @throws FileNotFoundException on any failure to locate that file.
726   */
727  public static File getWinUtilsFile() throws FileNotFoundException {
728    if (WINUTILS_FAILURE == null) {
729      return WINUTILS_FILE;
730    } else {
731      // raise a new exception to generate a new stack trace
732      throw fileNotFoundException(WINUTILS_FAILURE.toString(),
733          WINUTILS_FAILURE);
734    }
735  }
736
737  public static boolean checkIsBashSupported() throws InterruptedIOException {
738    if (Shell.WINDOWS) {
739      return false;
740    }
741
742    ShellCommandExecutor shexec;
743    boolean supported = true;
744    try {
745      String[] args = {"bash", "-c", "echo 1000"};
746      shexec = new ShellCommandExecutor(args);
747      shexec.execute();
748    } catch (InterruptedIOException iioe) {
749      LOG.warn("Interrupted, unable to determine if bash is supported", iioe);
750      throw iioe;
751    } catch (IOException ioe) {
752      LOG.warn("Bash is not supported by the OS", ioe);
753      supported = false;
754    }
755
756    return supported;
757  }
758
759  /**
760   * Flag which is true if setsid exists.
761   */
762  public static final boolean isSetsidAvailable = isSetsidSupported();
763
764  /**
765   * Look for <code>setsid</code>.
766   * @return true if <code>setsid</code> was present
767   */
768  private static boolean isSetsidSupported() {
769    if (Shell.WINDOWS) {
770      return false;
771    }
772    ShellCommandExecutor shexec = null;
773    boolean setsidSupported = true;
774    try {
775      String[] args = {"setsid", "bash", "-c", "echo $$"};
776      shexec = new ShellCommandExecutor(args);
777      shexec.execute();
778    } catch (IOException ioe) {
779      LOG.debug("setsid is not available on this machine. So not using it.");
780      setsidSupported = false;
781    }  catch (Error err) {
782      if (err.getMessage() != null
783          && err.getMessage().contains("posix_spawn is not " +
784          "a supported process launch mechanism")
785          && (Shell.FREEBSD || Shell.MAC)) {
786        // HADOOP-11924: This is a workaround to avoid failure of class init
787        // by JDK issue on TR locale(JDK-8047340).
788        LOG.info("Avoiding JDK-8047340 on BSD-based systems.", err);
789        setsidSupported = false;
790      }
791    }  finally { // handle the exit code
792      if (LOG.isDebugEnabled()) {
793        LOG.debug("setsid exited with exit code "
794                 + (shexec != null ? shexec.getExitCode() : "(null executor)"));
795      }
796    }
797    return setsidSupported;
798  }
799
800  /** Token separator regex used to parse Shell tool outputs. */
801  public static final String TOKEN_SEPARATOR_REGEX
802                = WINDOWS ? "[|\n\r]" : "[ \t\n\r\f]";
803
804  private long interval;   // refresh interval in msec
805  private long lastTime;   // last time the command was performed
806  private final boolean redirectErrorStream; // merge stdout and stderr
807  private Map<String, String> environment; // env for the command execution
808  private File dir;
809  private Process process; // sub process used to execute the command
810  private int exitCode;
811
812  /** Flag to indicate whether or not the script has finished executing. */
813  private final AtomicBoolean completed = new AtomicBoolean(false);
814
815  /**
816   * Create an instance with no minimum interval between runs; stderr is
817   * not merged with stdout.
818   */
819  protected Shell() {
820    this(0L);
821  }
822
823  /**
824   * Create an instance with a minimum interval between executions; stderr is
825   * not merged with stdout.
826   * @param interval interval in milliseconds between command executions.
827   */
828  protected Shell(long interval) {
829    this(interval, false);
830  }
831
832  /**
833   * Create a shell instance which can be re-executed when the {@link #run()}
834   * method is invoked with a given elapsed time between calls.
835   *
836   * @param interval the minimum duration in milliseconds to wait before
837   *        re-executing the command. If set to 0, there is no minimum.
838   * @param redirectErrorStream should the error stream be merged with
839   *        the normal output stream?
840   */
841  protected Shell(long interval, boolean redirectErrorStream) {
842    this.interval = interval;
843    this.lastTime = (interval < 0) ? 0 : -interval;
844    this.redirectErrorStream = redirectErrorStream;
845  }
846
847  /**
848   * Set the environment for the command.
849   * @param env Mapping of environment variables
850   */
851  protected void setEnvironment(Map<String, String> env) {
852    this.environment = env;
853  }
854
855  /**
856   * Set the working directory.
857   * @param dir The directory where the command will be executed
858   */
859  protected void setWorkingDirectory(File dir) {
860    this.dir = dir;
861  }
862
863  /** Check to see if a command needs to be executed and execute if needed. */
864  protected void run() throws IOException {
865    if (lastTime + interval > Time.monotonicNow()) {
866      return;
867    }
868    exitCode = 0; // reset for next run
869    runCommand();
870  }
871
872  /** Run the command. */
873  private void runCommand() throws IOException {
874    ProcessBuilder builder = new ProcessBuilder(getExecString());
875    Timer timeOutTimer = null;
876    ShellTimeoutTimerTask timeoutTimerTask = null;
877    timedOut.set(false);
878    completed.set(false);
879
880    if (environment != null) {
881      builder.environment().putAll(this.environment);
882    }
883
884    // Remove all env vars from the Builder to prevent leaking of env vars from
885    // the parent process.
886    if (!inheritParentEnv) {
887      // branch-2: Only do this for HADOOP_CREDSTORE_PASSWORD
888      // Sometimes daemons are configured to use the CredentialProvider feature
889      // and given their jceks password via an environment variable.  We need to
890      // make sure to remove it so it doesn't leak to child processes, which
891      // might be owned by a different user.  For example, the NodeManager
892      // running a User's container.
893      builder.environment().remove(
894          AbstractJavaKeyStoreProvider.CREDENTIAL_PASSWORD_ENV_VAR);
895    }
896
897    if (dir != null) {
898      builder.directory(this.dir);
899    }
900
901    builder.redirectErrorStream(redirectErrorStream);
902
903    if (Shell.WINDOWS) {
904      synchronized (WindowsProcessLaunchLock) {
905        // To workaround the race condition issue with child processes
906        // inheriting unintended handles during process launch that can
907        // lead to hangs on reading output and error streams, we
908        // serialize process creation. More info available at:
909        // http://support.microsoft.com/kb/315939
910        process = builder.start();
911      }
912    } else {
913      process = builder.start();
914    }
915
916    if (timeOutInterval > 0) {
917      timeOutTimer = new Timer("Shell command timeout");
918      timeoutTimerTask = new ShellTimeoutTimerTask(
919          this);
920      //One time scheduling.
921      timeOutTimer.schedule(timeoutTimerTask, timeOutInterval);
922    }
923    final BufferedReader errReader =
924            new BufferedReader(new InputStreamReader(
925                process.getErrorStream(), Charset.defaultCharset()));
926    BufferedReader inReader =
927            new BufferedReader(new InputStreamReader(
928                process.getInputStream(), Charset.defaultCharset()));
929    final StringBuffer errMsg = new StringBuffer();
930
931    // read error and input streams as this would free up the buffers
932    // free the error stream buffer
933    Thread errThread = new Thread() {
934      @Override
935      public void run() {
936        try {
937          String line = errReader.readLine();
938          while((line != null) && !isInterrupted()) {
939            errMsg.append(line);
940            errMsg.append(System.getProperty("line.separator"));
941            line = errReader.readLine();
942          }
943        } catch(IOException ioe) {
944          LOG.warn("Error reading the error stream", ioe);
945        }
946      }
947    };
948    try {
949      errThread.start();
950    } catch (IllegalStateException ise) {
951    } catch (OutOfMemoryError oe) {
952      LOG.error("Caught " + oe + ". One possible reason is that ulimit"
953          + " setting of 'max user processes' is too low. If so, do"
954          + " 'ulimit -u <largerNum>' and try again.");
955      throw oe;
956    }
957    try {
958      parseExecResult(inReader); // parse the output
959      // clear the input stream buffer
960      String line = inReader.readLine();
961      while(line != null) {
962        line = inReader.readLine();
963      }
964      // wait for the process to finish and check the exit code
965      exitCode  = process.waitFor();
966      // make sure that the error thread exits
967      joinThread(errThread);
968      completed.set(true);
969      //the timeout thread handling
970      //taken care in finally block
971      if (exitCode != 0) {
972        throw new ExitCodeException(exitCode, errMsg.toString());
973      }
974    } catch (InterruptedException ie) {
975      InterruptedIOException iie = new InterruptedIOException(ie.toString());
976      iie.initCause(ie);
977      throw iie;
978    } finally {
979      if (timeOutTimer != null) {
980        timeOutTimer.cancel();
981      }
982      // close the input stream
983      try {
984        // JDK 7 tries to automatically drain the input streams for us
985        // when the process exits, but since close is not synchronized,
986        // it creates a race if we close the stream first and the same
987        // fd is recycled.  the stream draining thread will attempt to
988        // drain that fd!!  it may block, OOM, or cause bizarre behavior
989        // see: https://bugs.openjdk.java.net/browse/JDK-8024521
990        //      issue is fixed in build 7u60
991        InputStream stdout = process.getInputStream();
992        synchronized (stdout) {
993          inReader.close();
994        }
995      } catch (IOException ioe) {
996        LOG.warn("Error while closing the input stream", ioe);
997      }
998      if (!completed.get()) {
999        errThread.interrupt();
1000        joinThread(errThread);
1001      }
1002      try {
1003        InputStream stderr = process.getErrorStream();
1004        synchronized (stderr) {
1005          errReader.close();
1006        }
1007      } catch (IOException ioe) {
1008        LOG.warn("Error while closing the error stream", ioe);
1009      }
1010      process.destroy();
1011      lastTime = Time.monotonicNow();
1012    }
1013  }
1014
1015  private static void joinThread(Thread t) {
1016    while (t.isAlive()) {
1017      try {
1018        t.join();
1019      } catch (InterruptedException ie) {
1020        if (LOG.isWarnEnabled()) {
1021          LOG.warn("Interrupted while joining on: " + t, ie);
1022        }
1023        t.interrupt(); // propagate interrupt
1024      }
1025    }
1026  }
1027
1028  /** return an array containing the command name and its parameters. */
1029  protected abstract String[] getExecString();
1030
1031  /** Parse the execution result */
1032  protected abstract void parseExecResult(BufferedReader lines)
1033  throws IOException;
1034
1035  /**
1036   * Get an environment variable.
1037   * @param env the environment var
1038   * @return the value or null if it was unset.
1039   */
1040  public String getEnvironment(String env) {
1041    return environment.get(env);
1042  }
1043
1044  /** get the current sub-process executing the given command.
1045   * @return process executing the command
1046   */
1047  public Process getProcess() {
1048    return process;
1049  }
1050
1051  /** get the exit code.
1052   * @return the exit code of the process
1053   */
1054  public int getExitCode() {
1055    return exitCode;
1056  }
1057
1058  /**
1059   * This is an IOException with exit code added.
1060   */
1061  public static class ExitCodeException extends IOException {
1062    private final int exitCode;
1063
1064    public ExitCodeException(int exitCode, String message) {
1065      super(message);
1066      this.exitCode = exitCode;
1067    }
1068
1069    public int getExitCode() {
1070      return exitCode;
1071    }
1072
1073    @Override
1074    public String toString() {
1075      final StringBuilder sb =
1076          new StringBuilder("ExitCodeException ");
1077      sb.append("exitCode=").append(exitCode)
1078        .append(": ");
1079      sb.append(super.getMessage());
1080      return sb.toString();
1081    }
1082  }
1083
1084  public interface CommandExecutor {
1085
1086    void execute() throws IOException;
1087
1088    int getExitCode() throws IOException;
1089
1090    String getOutput() throws IOException;
1091
1092    void close();
1093
1094  }
1095
1096  /**
1097   * A simple shell command executor.
1098   *
1099   * <code>ShellCommandExecutor</code>should be used in cases where the output
1100   * of the command needs no explicit parsing and where the command, working
1101   * directory and the environment remains unchanged. The output of the command
1102   * is stored as-is and is expected to be small.
1103   */
1104  public static class ShellCommandExecutor extends Shell
1105      implements CommandExecutor {
1106
1107    private String[] command;
1108    private StringBuffer output;
1109
1110
1111    public ShellCommandExecutor(String[] execString) {
1112      this(execString, null);
1113    }
1114
1115    public ShellCommandExecutor(String[] execString, File dir) {
1116      this(execString, dir, null);
1117    }
1118
1119    public ShellCommandExecutor(String[] execString, File dir,
1120                                 Map<String, String> env) {
1121      this(execString, dir, env , 0L);
1122    }
1123
1124    public ShellCommandExecutor(String[] execString, File dir,
1125                                Map<String, String> env, long timeout) {
1126      this(execString, dir, env , timeout, true);
1127    }
1128
1129    /**
1130     * Create a new instance of the ShellCommandExecutor to execute a command.
1131     *
1132     * @param execString The command to execute with arguments
1133     * @param dir If not-null, specifies the directory which should be set
1134     *            as the current working directory for the command.
1135     *            If null, the current working directory is not modified.
1136     * @param env If not-null, environment of the command will include the
1137     *            key-value pairs specified in the map. If null, the current
1138     *            environment is not modified.
1139     * @param timeout Specifies the time in milliseconds, after which the
1140     *                command will be killed and the status marked as timed-out.
1141     *                If 0, the command will not be timed out.
1142     * @param inheritParentEnv Indicates if the process should inherit the env
1143     *                         vars from the parent process or not.
1144     */
1145    public ShellCommandExecutor(String[] execString, File dir,
1146        Map<String, String> env, long timeout, boolean inheritParentEnv) {
1147      command = execString.clone();
1148      if (dir != null) {
1149        setWorkingDirectory(dir);
1150      }
1151      if (env != null) {
1152        setEnvironment(env);
1153      }
1154      timeOutInterval = timeout;
1155      this.inheritParentEnv = inheritParentEnv;
1156    }
1157
1158    /**
1159     * Execute the shell command.
1160     * @throws IOException if the command fails, or if the command is
1161     * not well constructed.
1162     */
1163    public void execute() throws IOException {
1164      for (String s : command) {
1165        if (s == null) {
1166          throw new IOException("(null) entry in command string: "
1167              + StringUtils.join(" ", command));
1168        }
1169      }
1170      this.run();
1171    }
1172
1173    @Override
1174    public String[] getExecString() {
1175      return command;
1176    }
1177
1178    @Override
1179    protected void parseExecResult(BufferedReader lines) throws IOException {
1180      output = new StringBuffer();
1181      char[] buf = new char[512];
1182      int nRead;
1183      while ( (nRead = lines.read(buf, 0, buf.length)) > 0 ) {
1184        output.append(buf, 0, nRead);
1185      }
1186    }
1187
1188    /** Get the output of the shell command. */
1189    public String getOutput() {
1190      return (output == null) ? "" : output.toString();
1191    }
1192
1193    /**
1194     * Returns the commands of this instance.
1195     * Arguments with spaces in are presented with quotes round; other
1196     * arguments are presented raw
1197     *
1198     * @return a string representation of the object.
1199     */
1200    @Override
1201    public String toString() {
1202      StringBuilder builder = new StringBuilder();
1203      String[] args = getExecString();
1204      for (String s : args) {
1205        if (s.indexOf(' ') >= 0) {
1206          builder.append('"').append(s).append('"');
1207        } else {
1208          builder.append(s);
1209        }
1210        builder.append(' ');
1211      }
1212      return builder.toString();
1213    }
1214
1215    @Override
1216    public void close() {
1217    }
1218  }
1219
1220  /**
1221   * To check if the passed script to shell command executor timed out or
1222   * not.
1223   *
1224   * @return if the script timed out.
1225   */
1226  public boolean isTimedOut() {
1227    return timedOut.get();
1228  }
1229
1230  /**
1231   * Declare that the command has timed out.
1232   *
1233   */
1234  private void setTimedOut() {
1235    this.timedOut.set(true);
1236  }
1237
1238  /**
1239   * Static method to execute a shell command.
1240   * Covers most of the simple cases without requiring the user to implement
1241   * the <code>Shell</code> interface.
1242   * @param cmd shell command to execute.
1243   * @return the output of the executed command.
1244   */
1245  public static String execCommand(String ... cmd) throws IOException {
1246    return execCommand(null, cmd, 0L);
1247  }
1248
1249  /**
1250   * Static method to execute a shell command.
1251   * Covers most of the simple cases without requiring the user to implement
1252   * the <code>Shell</code> interface.
1253   * @param env the map of environment key=value
1254   * @param cmd shell command to execute.
1255   * @param timeout time in milliseconds after which script should be marked timeout
1256   * @return the output of the executed command.
1257   * @throws IOException on any problem.
1258   */
1259
1260  public static String execCommand(Map<String, String> env, String[] cmd,
1261      long timeout) throws IOException {
1262    ShellCommandExecutor exec = new ShellCommandExecutor(cmd, null, env,
1263                                                          timeout);
1264    exec.execute();
1265    return exec.getOutput();
1266  }
1267
1268  /**
1269   * Static method to execute a shell command.
1270   * Covers most of the simple cases without requiring the user to implement
1271   * the <code>Shell</code> interface.
1272   * @param env the map of environment key=value
1273   * @param cmd shell command to execute.
1274   * @return the output of the executed command.
1275   * @throws IOException on any problem.
1276   */
1277  public static String execCommand(Map<String,String> env, String ... cmd)
1278  throws IOException {
1279    return execCommand(env, cmd, 0L);
1280  }
1281
1282  /**
1283   * Timer which is used to timeout scripts spawned off by shell.
1284   */
1285  private static class ShellTimeoutTimerTask extends TimerTask {
1286
1287    private final Shell shell;
1288
1289    public ShellTimeoutTimerTask(Shell shell) {
1290      this.shell = shell;
1291    }
1292
1293    @Override
1294    public void run() {
1295      Process p = shell.getProcess();
1296      try {
1297        p.exitValue();
1298      } catch (Exception e) {
1299        //Process has not terminated.
1300        //So check if it has completed
1301        //if not just destroy it.
1302        if (p != null && !shell.completed.get()) {
1303          shell.setTimedOut();
1304          p.destroy();
1305        }
1306      }
1307    }
1308  }
1309}