001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.util; 019 020import java.io.BufferedReader; 021import java.io.File; 022import java.io.FileNotFoundException; 023import java.io.IOException; 024import java.io.InputStreamReader; 025import java.io.InputStream; 026import java.io.InterruptedIOException; 027import java.nio.charset.Charset; 028import java.util.Arrays; 029import java.util.Map; 030import java.util.Timer; 031import java.util.TimerTask; 032import java.util.concurrent.atomic.AtomicBoolean; 033 034import com.google.common.annotations.VisibleForTesting; 035import org.apache.hadoop.classification.InterfaceAudience; 036import org.apache.hadoop.classification.InterfaceStability; 037import org.apache.hadoop.security.alias.AbstractJavaKeyStoreProvider; 038import org.slf4j.Logger; 039import org.slf4j.LoggerFactory; 040 041/** 042 * A base class for running a Shell command. 043 * 044 * <code>Shell</code> can be used to run shell commands like <code>du</code> or 045 * <code>df</code>. It also offers facilities to gate commands by 046 * time-intervals. 047 */ 048@InterfaceAudience.Public 049@InterfaceStability.Evolving 050public abstract class Shell { 051 public static final Logger LOG = LoggerFactory.getLogger(Shell.class); 052 053 /** 054 * Text to include when there are windows-specific problems. 055 * {@value} 056 */ 057 private static final String WINDOWS_PROBLEMS = 058 "https://wiki.apache.org/hadoop/WindowsProblems"; 059 060 /** 061 * Name of the windows utils binary: {@value}. 062 */ 063 static final String WINUTILS_EXE = "winutils.exe"; 064 065 /** 066 * System property for the Hadoop home directory: {@value}. 067 */ 068 public static final String SYSPROP_HADOOP_HOME_DIR = "hadoop.home.dir"; 069 070 /** 071 * Environment variable for Hadoop's home dir: {@value}. 072 */ 073 public static final String ENV_HADOOP_HOME = "HADOOP_HOME"; 074 075 /** 076 * query to see if system is Java 7 or later. 077 * Now that Hadoop requires Java 7 or later, this always returns true. 078 * @deprecated This call isn't needed any more: please remove uses of it. 079 * @return true, always. 080 */ 081 @Deprecated 082 public static boolean isJava7OrAbove() { 083 return true; 084 } 085 086 /** 087 * Maximum command line length in Windows 088 * KB830473 documents this as 8191 089 */ 090 public static final int WINDOWS_MAX_SHELL_LENGTH = 8191; 091 092 /** 093 * mis-spelling of {@link #WINDOWS_MAX_SHELL_LENGTH}. 094 * @deprecated use the correctly spelled constant. 095 */ 096 @Deprecated 097 public static final int WINDOWS_MAX_SHELL_LENGHT = WINDOWS_MAX_SHELL_LENGTH; 098 099 /** 100 * Checks if a given command (String[]) fits in the Windows maximum command 101 * line length Note that the input is expected to already include space 102 * delimiters, no extra count will be added for delimiters. 103 * 104 * @param commands command parts, including any space delimiters 105 */ 106 public static void checkWindowsCommandLineLength(String...commands) 107 throws IOException { 108 int len = 0; 109 for (String s: commands) { 110 len += s.length(); 111 } 112 if (len > WINDOWS_MAX_SHELL_LENGTH) { 113 throw new IOException(String.format( 114 "The command line has a length of %d exceeds maximum allowed length" + 115 " of %d. Command starts with: %s", 116 len, WINDOWS_MAX_SHELL_LENGTH, 117 StringUtils.join("", commands).substring(0, 100))); 118 } 119 } 120 121 /** 122 * Quote the given arg so that bash will interpret it as a single value. 123 * Note that this quotes it for one level of bash, if you are passing it 124 * into a badly written shell script, you need to fix your shell script. 125 * @param arg the argument to quote 126 * @return the quoted string 127 */ 128 static String bashQuote(String arg) { 129 StringBuilder buffer = new StringBuilder(arg.length() + 2); 130 buffer.append('\''); 131 buffer.append(arg.replace("'", "'\\''")); 132 buffer.append('\''); 133 return buffer.toString(); 134 } 135 136 /** a Unix command to get the current user's name: {@value}. */ 137 public static final String USER_NAME_COMMAND = "whoami"; 138 139 /** Windows <code>CreateProcess</code> synchronization object. */ 140 public static final Object WindowsProcessLaunchLock = new Object(); 141 142 // OSType detection 143 144 public enum OSType { 145 OS_TYPE_LINUX, 146 OS_TYPE_WIN, 147 OS_TYPE_SOLARIS, 148 OS_TYPE_MAC, 149 OS_TYPE_FREEBSD, 150 OS_TYPE_OTHER 151 } 152 153 /** 154 * Get the type of the operating system, as determined from parsing 155 * the <code>os.name</code> property. 156 */ 157 public static final OSType osType = getOSType(); 158 159 private static OSType getOSType() { 160 String osName = System.getProperty("os.name"); 161 if (osName.startsWith("Windows")) { 162 return OSType.OS_TYPE_WIN; 163 } else if (osName.contains("SunOS") || osName.contains("Solaris")) { 164 return OSType.OS_TYPE_SOLARIS; 165 } else if (osName.contains("Mac")) { 166 return OSType.OS_TYPE_MAC; 167 } else if (osName.contains("FreeBSD")) { 168 return OSType.OS_TYPE_FREEBSD; 169 } else if (osName.startsWith("Linux")) { 170 return OSType.OS_TYPE_LINUX; 171 } else { 172 // Some other form of Unix 173 return OSType.OS_TYPE_OTHER; 174 } 175 } 176 177 // Helper static vars for each platform 178 public static final boolean WINDOWS = (osType == OSType.OS_TYPE_WIN); 179 public static final boolean SOLARIS = (osType == OSType.OS_TYPE_SOLARIS); 180 public static final boolean MAC = (osType == OSType.OS_TYPE_MAC); 181 public static final boolean FREEBSD = (osType == OSType.OS_TYPE_FREEBSD); 182 public static final boolean LINUX = (osType == OSType.OS_TYPE_LINUX); 183 public static final boolean OTHER = (osType == OSType.OS_TYPE_OTHER); 184 185 public static final boolean PPC_64 186 = System.getProperties().getProperty("os.arch").contains("ppc64"); 187 188 /** a Unix command to get the current user's groups list. */ 189 public static String[] getGroupsCommand() { 190 return (WINDOWS)? new String[]{"cmd", "/c", "groups"} 191 : new String[]{"groups"}; 192 } 193 194 /** 195 * A command to get a given user's groups list. 196 * If the OS is not WINDOWS, the command will get the user's primary group 197 * first and finally get the groups list which includes the primary group. 198 * i.e. the user's primary group will be included twice. 199 */ 200 public static String[] getGroupsForUserCommand(final String user) { 201 //'groups username' command return is inconsistent across different unixes 202 if (WINDOWS) { 203 return new String[] 204 {getWinUtilsPath(), "groups", "-F", "\"" + user + "\""}; 205 } else { 206 String quotedUser = bashQuote(user); 207 return new String[] {"bash", "-c", "id -gn " + quotedUser + 208 "; id -Gn " + quotedUser}; 209 } 210 } 211 212 /** 213 * A command to get a given user's group id list. 214 * The command will get the user's primary group 215 * first and finally get the groups list which includes the primary group. 216 * i.e. the user's primary group will be included twice. 217 * This command does not support Windows and will only return group names. 218 */ 219 public static String[] getGroupsIDForUserCommand(final String user) { 220 //'groups username' command return is inconsistent across different unixes 221 if (WINDOWS) { 222 return new String[]{getWinUtilsPath(), "groups", "-F", "\"" + user + 223 "\""}; 224 } else { 225 String quotedUser = bashQuote(user); 226 return new String[] {"bash", "-c", "id -g " + quotedUser + "; id -G " + 227 quotedUser}; 228 } 229 } 230 231 /** A command to get a given netgroup's user list. */ 232 public static String[] getUsersForNetgroupCommand(final String netgroup) { 233 //'groups username' command return is non-consistent across different unixes 234 return new String[] {"getent", "netgroup", netgroup}; 235 } 236 237 /** Return a command to get permission information. */ 238 public static String[] getGetPermissionCommand() { 239 return (WINDOWS) ? new String[] { getWinUtilsPath(), "ls", "-F" } 240 : new String[] { "ls", "-ld" }; 241 } 242 243 /** Return a command to set permission. */ 244 public static String[] getSetPermissionCommand(String perm, boolean recursive) { 245 if (recursive) { 246 return (WINDOWS) ? 247 new String[] { getWinUtilsPath(), "chmod", "-R", perm } 248 : new String[] { "chmod", "-R", perm }; 249 } else { 250 return (WINDOWS) ? 251 new String[] { getWinUtilsPath(), "chmod", perm } 252 : new String[] { "chmod", perm }; 253 } 254 } 255 256 /** 257 * Return a command to set permission for specific file. 258 * 259 * @param perm String permission to set 260 * @param recursive boolean true to apply to all sub-directories recursively 261 * @param file String file to set 262 * @return String[] containing command and arguments 263 */ 264 public static String[] getSetPermissionCommand(String perm, 265 boolean recursive, 266 String file) { 267 String[] baseCmd = getSetPermissionCommand(perm, recursive); 268 String[] cmdWithFile = Arrays.copyOf(baseCmd, baseCmd.length + 1); 269 cmdWithFile[cmdWithFile.length - 1] = file; 270 return cmdWithFile; 271 } 272 273 /** Return a command to set owner. */ 274 public static String[] getSetOwnerCommand(String owner) { 275 return (WINDOWS) ? 276 new String[] { getWinUtilsPath(), "chown", "\"" + owner + "\"" } 277 : new String[] { "chown", owner }; 278 } 279 280 /** Return a command to create symbolic links. */ 281 public static String[] getSymlinkCommand(String target, String link) { 282 return WINDOWS ? 283 new String[] { getWinUtilsPath(), "symlink", link, target } 284 : new String[] { "ln", "-s", target, link }; 285 } 286 287 /** Return a command to read the target of the a symbolic link. */ 288 public static String[] getReadlinkCommand(String link) { 289 return WINDOWS ? 290 new String[] { getWinUtilsPath(), "readlink", link } 291 : new String[] { "readlink", link }; 292 } 293 294 /** 295 * Return a command for determining if process with specified pid is alive. 296 * @param pid process ID 297 * @return a <code>kill -0</code> command or equivalent 298 */ 299 public static String[] getCheckProcessIsAliveCommand(String pid) { 300 return getSignalKillCommand(0, pid); 301 } 302 303 /** Return a command to send a signal to a given pid. */ 304 public static String[] getSignalKillCommand(int code, String pid) { 305 // Code == 0 means check alive 306 if (Shell.WINDOWS) { 307 if (0 == code) { 308 return new String[] {Shell.getWinUtilsPath(), "task", "isAlive", pid }; 309 } else { 310 return new String[] {Shell.getWinUtilsPath(), "task", "kill", pid }; 311 } 312 } 313 314 // Use the bash-builtin instead of the Unix kill command (usually 315 // /bin/kill) as the bash-builtin supports "--" in all Hadoop supported 316 // OSes. 317 final String quotedPid = bashQuote(pid); 318 if (isSetsidAvailable) { 319 return new String[] { "bash", "-c", "kill -" + code + " -- -" + 320 quotedPid }; 321 } else { 322 return new String[] { "bash", "-c", "kill -" + code + " " + 323 quotedPid }; 324 } 325 } 326 327 /** Regular expression for environment variables: {@value}. */ 328 public static final String ENV_NAME_REGEX = "[A-Za-z_][A-Za-z0-9_]*"; 329 330 /** Return a regular expression string that match environment variables. */ 331 public static String getEnvironmentVariableRegex() { 332 return (WINDOWS) 333 ? "%(" + ENV_NAME_REGEX + "?)%" 334 : "\\$(" + ENV_NAME_REGEX + ")"; 335 } 336 337 /** 338 * Returns a File referencing a script with the given basename, inside the 339 * given parent directory. The file extension is inferred by platform: 340 * <code>".cmd"</code> on Windows, or <code>".sh"</code> otherwise. 341 * 342 * @param parent File parent directory 343 * @param basename String script file basename 344 * @return File referencing the script in the directory 345 */ 346 public static File appendScriptExtension(File parent, String basename) { 347 return new File(parent, appendScriptExtension(basename)); 348 } 349 350 /** 351 * Returns a script file name with the given basename. 352 * 353 * The file extension is inferred by platform: 354 * <code>".cmd"</code> on Windows, or <code>".sh"</code> otherwise. 355 * 356 * @param basename String script file basename 357 * @return String script file name 358 */ 359 public static String appendScriptExtension(String basename) { 360 return basename + (WINDOWS ? ".cmd" : ".sh"); 361 } 362 363 /** 364 * Returns a command to run the given script. The script interpreter is 365 * inferred by platform: cmd on Windows or bash otherwise. 366 * 367 * @param script File script to run 368 * @return String[] command to run the script 369 */ 370 public static String[] getRunScriptCommand(File script) { 371 String absolutePath = script.getAbsolutePath(); 372 return WINDOWS ? 373 new String[] {"cmd", "/c", absolutePath } 374 : new String[] {"bash", bashQuote(absolutePath) }; 375 } 376 377 /** a Unix command to set permission: {@value}. */ 378 public static final String SET_PERMISSION_COMMAND = "chmod"; 379 /** a Unix command to set owner: {@value}. */ 380 public static final String SET_OWNER_COMMAND = "chown"; 381 382 /** a Unix command to set the change user's groups list: {@value}. */ 383 public static final String SET_GROUP_COMMAND = "chgrp"; 384 /** a Unix command to create a link: {@value}. */ 385 public static final String LINK_COMMAND = "ln"; 386 /** a Unix command to get a link target: {@value}. */ 387 public static final String READ_LINK_COMMAND = "readlink"; 388 389 /**Time after which the executing script would be timedout. */ 390 protected long timeOutInterval = 0L; 391 /** If or not script timed out*/ 392 private final AtomicBoolean timedOut = new AtomicBoolean(false); 393 394 /** Indicates if the parent env vars should be inherited or not*/ 395 protected boolean inheritParentEnv = true; 396 397 /** 398 * Centralized logic to discover and validate the sanity of the Hadoop 399 * home directory. 400 * 401 * This does a lot of work so it should only be called 402 * privately for initialization once per process. 403 * 404 * @return A directory that exists and via was specified on the command line 405 * via <code>-Dhadoop.home.dir</code> or the <code>HADOOP_HOME</code> 406 * environment variable. 407 * @throws FileNotFoundException if the properties are absent or the specified 408 * path is not a reference to a valid directory. 409 */ 410 private static File checkHadoopHome() throws FileNotFoundException { 411 412 // first check the Dflag hadoop.home.dir with JVM scope 413 String home = System.getProperty(SYSPROP_HADOOP_HOME_DIR); 414 415 // fall back to the system/user-global env variable 416 if (home == null) { 417 home = System.getenv(ENV_HADOOP_HOME); 418 } 419 return checkHadoopHomeInner(home); 420 } 421 422 /* 423 A set of exception strings used to construct error messages; 424 these are referred to in tests 425 */ 426 static final String E_DOES_NOT_EXIST = "does not exist"; 427 static final String E_IS_RELATIVE = "is not an absolute path."; 428 static final String E_NOT_DIRECTORY = "is not a directory."; 429 static final String E_NO_EXECUTABLE = "Could not locate Hadoop executable"; 430 static final String E_NOT_EXECUTABLE_FILE = "Not an executable file"; 431 static final String E_HADOOP_PROPS_UNSET = ENV_HADOOP_HOME + " and " 432 + SYSPROP_HADOOP_HOME_DIR + " are unset."; 433 static final String E_HADOOP_PROPS_EMPTY = ENV_HADOOP_HOME + " or " 434 + SYSPROP_HADOOP_HOME_DIR + " set to an empty string"; 435 static final String E_NOT_A_WINDOWS_SYSTEM = "Not a Windows system"; 436 437 /** 438 * Validate the accessibility of the Hadoop home directory. 439 * 440 * @return A directory that is expected to be the hadoop home directory 441 * @throws FileNotFoundException if the specified 442 * path is not a reference to a valid directory. 443 */ 444 @VisibleForTesting 445 static File checkHadoopHomeInner(String home) throws FileNotFoundException { 446 // couldn't find either setting for hadoop's home directory 447 if (home == null) { 448 throw new FileNotFoundException(E_HADOOP_PROPS_UNSET); 449 } 450 // strip off leading and trailing double quotes 451 while (home.startsWith("\"")) { 452 home = home.substring(1); 453 } 454 while (home.endsWith("\"")) { 455 home = home.substring(0, home.length() - 1); 456 } 457 458 // after stripping any quotes, check for home dir being non-empty 459 if (home.isEmpty()) { 460 throw new FileNotFoundException(E_HADOOP_PROPS_EMPTY); 461 } 462 463 // check that the hadoop home dir value 464 // is an absolute reference to a directory 465 File homedir = new File(home); 466 if (!homedir.isAbsolute()) { 467 throw new FileNotFoundException("Hadoop home directory " + homedir 468 + " " + E_IS_RELATIVE); 469 } 470 if (!homedir.exists()) { 471 throw new FileNotFoundException("Hadoop home directory " + homedir 472 + " " + E_DOES_NOT_EXIST); 473 } 474 if (!homedir.isDirectory()) { 475 throw new FileNotFoundException("Hadoop home directory " + homedir 476 + " "+ E_NOT_DIRECTORY); 477 } 478 return homedir; 479 } 480 481 /** 482 * The Hadoop home directory. 483 */ 484 private static final File HADOOP_HOME_FILE; 485 486 /** 487 * Rethrowable cause for the failure to determine the hadoop 488 * home directory 489 */ 490 private static final IOException HADOOP_HOME_DIR_FAILURE_CAUSE; 491 492 static { 493 File home; 494 IOException ex; 495 try { 496 home = checkHadoopHome(); 497 ex = null; 498 } catch (IOException ioe) { 499 if (LOG.isDebugEnabled()) { 500 LOG.debug("Failed to detect a valid hadoop home directory", ioe); 501 } 502 ex = ioe; 503 home = null; 504 } 505 HADOOP_HOME_FILE = home; 506 HADOOP_HOME_DIR_FAILURE_CAUSE = ex; 507 } 508 509 /** 510 * Optionally extend an error message with some OS-specific text. 511 * @param message core error message 512 * @return error message, possibly with some extra text 513 */ 514 private static String addOsText(String message) { 515 return WINDOWS ? (message + " -see " + WINDOWS_PROBLEMS) : message; 516 } 517 518 /** 519 * Create a {@code FileNotFoundException} with the inner nested cause set 520 * to the given exception. Compensates for the fact that FNFE doesn't 521 * have an initializer that takes an exception. 522 * @param text error text 523 * @param ex inner exception 524 * @return a new exception to throw. 525 */ 526 private static FileNotFoundException fileNotFoundException(String text, 527 Exception ex) { 528 return (FileNotFoundException) new FileNotFoundException(text) 529 .initCause(ex); 530 } 531 532 /** 533 * Get the Hadoop home directory. Raises an exception if not found 534 * @return the home dir 535 * @throws IOException if the home directory cannot be located. 536 */ 537 public static String getHadoopHome() throws IOException { 538 return getHadoopHomeDir().getCanonicalPath(); 539 } 540 541 /** 542 * Get the Hadoop home directory. If it is invalid, 543 * throw an exception. 544 * @return a path referring to hadoop home. 545 * @throws FileNotFoundException if the directory doesn't exist. 546 */ 547 private static File getHadoopHomeDir() throws FileNotFoundException { 548 if (HADOOP_HOME_DIR_FAILURE_CAUSE != null) { 549 throw fileNotFoundException( 550 addOsText(HADOOP_HOME_DIR_FAILURE_CAUSE.toString()), 551 HADOOP_HOME_DIR_FAILURE_CAUSE); 552 } 553 return HADOOP_HOME_FILE; 554 } 555 556 /** 557 * Fully qualify the path to a binary that should be in a known hadoop 558 * bin location. This is primarily useful for disambiguating call-outs 559 * to executable sub-components of Hadoop to avoid clashes with other 560 * executables that may be in the path. Caveat: this call doesn't 561 * just format the path to the bin directory. It also checks for file 562 * existence of the composed path. The output of this call should be 563 * cached by callers. 564 * 565 * @param executable executable 566 * @return executable file reference 567 * @throws FileNotFoundException if the path does not exist 568 */ 569 public static File getQualifiedBin(String executable) 570 throws FileNotFoundException { 571 // construct hadoop bin path to the specified executable 572 return getQualifiedBinInner(getHadoopHomeDir(), executable); 573 } 574 575 /** 576 * Inner logic of {@link #getQualifiedBin(String)}, accessible 577 * for tests. 578 * @param hadoopHomeDir home directory (assumed to be valid) 579 * @param executable executable 580 * @return path to the binary 581 * @throws FileNotFoundException if the executable was not found/valid 582 */ 583 static File getQualifiedBinInner(File hadoopHomeDir, String executable) 584 throws FileNotFoundException { 585 String binDirText = "Hadoop bin directory "; 586 File bin = new File(hadoopHomeDir, "bin"); 587 if (!bin.exists()) { 588 throw new FileNotFoundException(addOsText(binDirText + E_DOES_NOT_EXIST 589 + ": " + bin)); 590 } 591 if (!bin.isDirectory()) { 592 throw new FileNotFoundException(addOsText(binDirText + E_NOT_DIRECTORY 593 + ": " + bin)); 594 } 595 596 File exeFile = new File(bin, executable); 597 if (!exeFile.exists()) { 598 throw new FileNotFoundException( 599 addOsText(E_NO_EXECUTABLE + ": " + exeFile)); 600 } 601 if (!exeFile.isFile()) { 602 throw new FileNotFoundException( 603 addOsText(E_NOT_EXECUTABLE_FILE + ": " + exeFile)); 604 } 605 try { 606 return exeFile.getCanonicalFile(); 607 } catch (IOException e) { 608 // this isn't going to happen, because of all the upfront checks. 609 // so if it does, it gets converted to a FNFE and rethrown 610 throw fileNotFoundException(e.toString(), e); 611 } 612 } 613 614 /** 615 * Fully qualify the path to a binary that should be in a known hadoop 616 * bin location. This is primarily useful for disambiguating call-outs 617 * to executable sub-components of Hadoop to avoid clashes with other 618 * executables that may be in the path. Caveat: this call doesn't 619 * just format the path to the bin directory. It also checks for file 620 * existence of the composed path. The output of this call should be 621 * cached by callers. 622 * 623 * @param executable executable 624 * @return executable file reference 625 * @throws FileNotFoundException if the path does not exist 626 * @throws IOException on path canonicalization failures 627 */ 628 public static String getQualifiedBinPath(String executable) 629 throws IOException { 630 return getQualifiedBin(executable).getCanonicalPath(); 631 } 632 633 /** 634 * Location of winutils as a string; null if not found. 635 * <p> 636 * <i>Important: caller must check for this value being null</i>. 637 * The lack of such checks has led to many support issues being raised. 638 * <p> 639 * @deprecated use one of the exception-raising getter methods, 640 * specifically {@link #getWinUtilsPath()} or {@link #getWinUtilsFile()} 641 */ 642 @Deprecated 643 public static final String WINUTILS; 644 645 /** Canonical path to winutils, private to Shell. */ 646 private static final String WINUTILS_PATH; 647 648 /** file reference to winutils. */ 649 private static final File WINUTILS_FILE; 650 651 /** the exception raised on a failure to init the WINUTILS fields. */ 652 private static final IOException WINUTILS_FAILURE; 653 654 /* 655 * Static WINUTILS_* field initializer. 656 * On non-Windows systems sets the paths to null, and 657 * adds a specific exception to the failure cause, so 658 * that on any attempt to resolve the paths will raise 659 * a meaningful exception. 660 */ 661 static { 662 IOException ioe = null; 663 String path = null; 664 File file = null; 665 // invariant: either there's a valid file and path, 666 // or there is a cached IO exception. 667 if (WINDOWS) { 668 try { 669 file = getQualifiedBin(WINUTILS_EXE); 670 path = file.getCanonicalPath(); 671 ioe = null; 672 } catch (IOException e) { 673 LOG.warn("Did not find {}: {}", WINUTILS_EXE, e); 674 // stack trace comes at debug level 675 LOG.debug("Failed to find " + WINUTILS_EXE, e); 676 file = null; 677 path = null; 678 ioe = e; 679 } 680 } else { 681 // on a non-windows system, the invariant is kept 682 // by adding an explicit exception. 683 ioe = new FileNotFoundException(E_NOT_A_WINDOWS_SYSTEM); 684 } 685 WINUTILS_PATH = path; 686 WINUTILS_FILE = file; 687 688 WINUTILS = path; 689 WINUTILS_FAILURE = ioe; 690 } 691 692 /** 693 * Predicate to indicate whether or not the path to winutils is known. 694 * 695 * If true, then {@link #WINUTILS} is non-null, and both 696 * {@link #getWinUtilsPath()} and {@link #getWinUtilsFile()} 697 * will successfully return this value. Always false on non-windows systems. 698 * @return true if there is a valid path to the binary 699 */ 700 public static boolean hasWinutilsPath() { 701 return WINUTILS_PATH != null; 702 } 703 704 /** 705 * Locate the winutils binary, or fail with a meaningful 706 * exception and stack trace as an RTE. 707 * This method is for use in methods which don't explicitly throw 708 * an <code>IOException</code>. 709 * @return the path to {@link #WINUTILS_EXE} 710 * @throws RuntimeException if the path is not resolvable 711 */ 712 public static String getWinUtilsPath() { 713 if (WINUTILS_FAILURE == null) { 714 return WINUTILS_PATH; 715 } else { 716 throw new RuntimeException(WINUTILS_FAILURE.toString(), 717 WINUTILS_FAILURE); 718 } 719 } 720 721 /** 722 * Get a file reference to winutils. 723 * Always raises an exception if there isn't one 724 * @return the file instance referring to the winutils bin. 725 * @throws FileNotFoundException on any failure to locate that file. 726 */ 727 public static File getWinUtilsFile() throws FileNotFoundException { 728 if (WINUTILS_FAILURE == null) { 729 return WINUTILS_FILE; 730 } else { 731 // raise a new exception to generate a new stack trace 732 throw fileNotFoundException(WINUTILS_FAILURE.toString(), 733 WINUTILS_FAILURE); 734 } 735 } 736 737 public static boolean checkIsBashSupported() throws InterruptedIOException { 738 if (Shell.WINDOWS) { 739 return false; 740 } 741 742 ShellCommandExecutor shexec; 743 boolean supported = true; 744 try { 745 String[] args = {"bash", "-c", "echo 1000"}; 746 shexec = new ShellCommandExecutor(args); 747 shexec.execute(); 748 } catch (InterruptedIOException iioe) { 749 LOG.warn("Interrupted, unable to determine if bash is supported", iioe); 750 throw iioe; 751 } catch (IOException ioe) { 752 LOG.warn("Bash is not supported by the OS", ioe); 753 supported = false; 754 } 755 756 return supported; 757 } 758 759 /** 760 * Flag which is true if setsid exists. 761 */ 762 public static final boolean isSetsidAvailable = isSetsidSupported(); 763 764 /** 765 * Look for <code>setsid</code>. 766 * @return true if <code>setsid</code> was present 767 */ 768 private static boolean isSetsidSupported() { 769 if (Shell.WINDOWS) { 770 return false; 771 } 772 ShellCommandExecutor shexec = null; 773 boolean setsidSupported = true; 774 try { 775 String[] args = {"setsid", "bash", "-c", "echo $$"}; 776 shexec = new ShellCommandExecutor(args); 777 shexec.execute(); 778 } catch (IOException ioe) { 779 LOG.debug("setsid is not available on this machine. So not using it."); 780 setsidSupported = false; 781 } catch (Error err) { 782 if (err.getMessage() != null 783 && err.getMessage().contains("posix_spawn is not " + 784 "a supported process launch mechanism") 785 && (Shell.FREEBSD || Shell.MAC)) { 786 // HADOOP-11924: This is a workaround to avoid failure of class init 787 // by JDK issue on TR locale(JDK-8047340). 788 LOG.info("Avoiding JDK-8047340 on BSD-based systems.", err); 789 setsidSupported = false; 790 } 791 } finally { // handle the exit code 792 if (LOG.isDebugEnabled()) { 793 LOG.debug("setsid exited with exit code " 794 + (shexec != null ? shexec.getExitCode() : "(null executor)")); 795 } 796 } 797 return setsidSupported; 798 } 799 800 /** Token separator regex used to parse Shell tool outputs. */ 801 public static final String TOKEN_SEPARATOR_REGEX 802 = WINDOWS ? "[|\n\r]" : "[ \t\n\r\f]"; 803 804 private long interval; // refresh interval in msec 805 private long lastTime; // last time the command was performed 806 private final boolean redirectErrorStream; // merge stdout and stderr 807 private Map<String, String> environment; // env for the command execution 808 private File dir; 809 private Process process; // sub process used to execute the command 810 private int exitCode; 811 812 /** Flag to indicate whether or not the script has finished executing. */ 813 private final AtomicBoolean completed = new AtomicBoolean(false); 814 815 /** 816 * Create an instance with no minimum interval between runs; stderr is 817 * not merged with stdout. 818 */ 819 protected Shell() { 820 this(0L); 821 } 822 823 /** 824 * Create an instance with a minimum interval between executions; stderr is 825 * not merged with stdout. 826 * @param interval interval in milliseconds between command executions. 827 */ 828 protected Shell(long interval) { 829 this(interval, false); 830 } 831 832 /** 833 * Create a shell instance which can be re-executed when the {@link #run()} 834 * method is invoked with a given elapsed time between calls. 835 * 836 * @param interval the minimum duration in milliseconds to wait before 837 * re-executing the command. If set to 0, there is no minimum. 838 * @param redirectErrorStream should the error stream be merged with 839 * the normal output stream? 840 */ 841 protected Shell(long interval, boolean redirectErrorStream) { 842 this.interval = interval; 843 this.lastTime = (interval < 0) ? 0 : -interval; 844 this.redirectErrorStream = redirectErrorStream; 845 } 846 847 /** 848 * Set the environment for the command. 849 * @param env Mapping of environment variables 850 */ 851 protected void setEnvironment(Map<String, String> env) { 852 this.environment = env; 853 } 854 855 /** 856 * Set the working directory. 857 * @param dir The directory where the command will be executed 858 */ 859 protected void setWorkingDirectory(File dir) { 860 this.dir = dir; 861 } 862 863 /** Check to see if a command needs to be executed and execute if needed. */ 864 protected void run() throws IOException { 865 if (lastTime + interval > Time.monotonicNow()) { 866 return; 867 } 868 exitCode = 0; // reset for next run 869 runCommand(); 870 } 871 872 /** Run the command. */ 873 private void runCommand() throws IOException { 874 ProcessBuilder builder = new ProcessBuilder(getExecString()); 875 Timer timeOutTimer = null; 876 ShellTimeoutTimerTask timeoutTimerTask = null; 877 timedOut.set(false); 878 completed.set(false); 879 880 if (environment != null) { 881 builder.environment().putAll(this.environment); 882 } 883 884 // Remove all env vars from the Builder to prevent leaking of env vars from 885 // the parent process. 886 if (!inheritParentEnv) { 887 // branch-2: Only do this for HADOOP_CREDSTORE_PASSWORD 888 // Sometimes daemons are configured to use the CredentialProvider feature 889 // and given their jceks password via an environment variable. We need to 890 // make sure to remove it so it doesn't leak to child processes, which 891 // might be owned by a different user. For example, the NodeManager 892 // running a User's container. 893 builder.environment().remove( 894 AbstractJavaKeyStoreProvider.CREDENTIAL_PASSWORD_ENV_VAR); 895 } 896 897 if (dir != null) { 898 builder.directory(this.dir); 899 } 900 901 builder.redirectErrorStream(redirectErrorStream); 902 903 if (Shell.WINDOWS) { 904 synchronized (WindowsProcessLaunchLock) { 905 // To workaround the race condition issue with child processes 906 // inheriting unintended handles during process launch that can 907 // lead to hangs on reading output and error streams, we 908 // serialize process creation. More info available at: 909 // http://support.microsoft.com/kb/315939 910 process = builder.start(); 911 } 912 } else { 913 process = builder.start(); 914 } 915 916 if (timeOutInterval > 0) { 917 timeOutTimer = new Timer("Shell command timeout"); 918 timeoutTimerTask = new ShellTimeoutTimerTask( 919 this); 920 //One time scheduling. 921 timeOutTimer.schedule(timeoutTimerTask, timeOutInterval); 922 } 923 final BufferedReader errReader = 924 new BufferedReader(new InputStreamReader( 925 process.getErrorStream(), Charset.defaultCharset())); 926 BufferedReader inReader = 927 new BufferedReader(new InputStreamReader( 928 process.getInputStream(), Charset.defaultCharset())); 929 final StringBuffer errMsg = new StringBuffer(); 930 931 // read error and input streams as this would free up the buffers 932 // free the error stream buffer 933 Thread errThread = new Thread() { 934 @Override 935 public void run() { 936 try { 937 String line = errReader.readLine(); 938 while((line != null) && !isInterrupted()) { 939 errMsg.append(line); 940 errMsg.append(System.getProperty("line.separator")); 941 line = errReader.readLine(); 942 } 943 } catch(IOException ioe) { 944 LOG.warn("Error reading the error stream", ioe); 945 } 946 } 947 }; 948 try { 949 errThread.start(); 950 } catch (IllegalStateException ise) { 951 } catch (OutOfMemoryError oe) { 952 LOG.error("Caught " + oe + ". One possible reason is that ulimit" 953 + " setting of 'max user processes' is too low. If so, do" 954 + " 'ulimit -u <largerNum>' and try again."); 955 throw oe; 956 } 957 try { 958 parseExecResult(inReader); // parse the output 959 // clear the input stream buffer 960 String line = inReader.readLine(); 961 while(line != null) { 962 line = inReader.readLine(); 963 } 964 // wait for the process to finish and check the exit code 965 exitCode = process.waitFor(); 966 // make sure that the error thread exits 967 joinThread(errThread); 968 completed.set(true); 969 //the timeout thread handling 970 //taken care in finally block 971 if (exitCode != 0) { 972 throw new ExitCodeException(exitCode, errMsg.toString()); 973 } 974 } catch (InterruptedException ie) { 975 InterruptedIOException iie = new InterruptedIOException(ie.toString()); 976 iie.initCause(ie); 977 throw iie; 978 } finally { 979 if (timeOutTimer != null) { 980 timeOutTimer.cancel(); 981 } 982 // close the input stream 983 try { 984 // JDK 7 tries to automatically drain the input streams for us 985 // when the process exits, but since close is not synchronized, 986 // it creates a race if we close the stream first and the same 987 // fd is recycled. the stream draining thread will attempt to 988 // drain that fd!! it may block, OOM, or cause bizarre behavior 989 // see: https://bugs.openjdk.java.net/browse/JDK-8024521 990 // issue is fixed in build 7u60 991 InputStream stdout = process.getInputStream(); 992 synchronized (stdout) { 993 inReader.close(); 994 } 995 } catch (IOException ioe) { 996 LOG.warn("Error while closing the input stream", ioe); 997 } 998 if (!completed.get()) { 999 errThread.interrupt(); 1000 joinThread(errThread); 1001 } 1002 try { 1003 InputStream stderr = process.getErrorStream(); 1004 synchronized (stderr) { 1005 errReader.close(); 1006 } 1007 } catch (IOException ioe) { 1008 LOG.warn("Error while closing the error stream", ioe); 1009 } 1010 process.destroy(); 1011 lastTime = Time.monotonicNow(); 1012 } 1013 } 1014 1015 private static void joinThread(Thread t) { 1016 while (t.isAlive()) { 1017 try { 1018 t.join(); 1019 } catch (InterruptedException ie) { 1020 if (LOG.isWarnEnabled()) { 1021 LOG.warn("Interrupted while joining on: " + t, ie); 1022 } 1023 t.interrupt(); // propagate interrupt 1024 } 1025 } 1026 } 1027 1028 /** return an array containing the command name and its parameters. */ 1029 protected abstract String[] getExecString(); 1030 1031 /** Parse the execution result */ 1032 protected abstract void parseExecResult(BufferedReader lines) 1033 throws IOException; 1034 1035 /** 1036 * Get an environment variable. 1037 * @param env the environment var 1038 * @return the value or null if it was unset. 1039 */ 1040 public String getEnvironment(String env) { 1041 return environment.get(env); 1042 } 1043 1044 /** get the current sub-process executing the given command. 1045 * @return process executing the command 1046 */ 1047 public Process getProcess() { 1048 return process; 1049 } 1050 1051 /** get the exit code. 1052 * @return the exit code of the process 1053 */ 1054 public int getExitCode() { 1055 return exitCode; 1056 } 1057 1058 /** 1059 * This is an IOException with exit code added. 1060 */ 1061 public static class ExitCodeException extends IOException { 1062 private final int exitCode; 1063 1064 public ExitCodeException(int exitCode, String message) { 1065 super(message); 1066 this.exitCode = exitCode; 1067 } 1068 1069 public int getExitCode() { 1070 return exitCode; 1071 } 1072 1073 @Override 1074 public String toString() { 1075 final StringBuilder sb = 1076 new StringBuilder("ExitCodeException "); 1077 sb.append("exitCode=").append(exitCode) 1078 .append(": "); 1079 sb.append(super.getMessage()); 1080 return sb.toString(); 1081 } 1082 } 1083 1084 public interface CommandExecutor { 1085 1086 void execute() throws IOException; 1087 1088 int getExitCode() throws IOException; 1089 1090 String getOutput() throws IOException; 1091 1092 void close(); 1093 1094 } 1095 1096 /** 1097 * A simple shell command executor. 1098 * 1099 * <code>ShellCommandExecutor</code>should be used in cases where the output 1100 * of the command needs no explicit parsing and where the command, working 1101 * directory and the environment remains unchanged. The output of the command 1102 * is stored as-is and is expected to be small. 1103 */ 1104 public static class ShellCommandExecutor extends Shell 1105 implements CommandExecutor { 1106 1107 private String[] command; 1108 private StringBuffer output; 1109 1110 1111 public ShellCommandExecutor(String[] execString) { 1112 this(execString, null); 1113 } 1114 1115 public ShellCommandExecutor(String[] execString, File dir) { 1116 this(execString, dir, null); 1117 } 1118 1119 public ShellCommandExecutor(String[] execString, File dir, 1120 Map<String, String> env) { 1121 this(execString, dir, env , 0L); 1122 } 1123 1124 public ShellCommandExecutor(String[] execString, File dir, 1125 Map<String, String> env, long timeout) { 1126 this(execString, dir, env , timeout, true); 1127 } 1128 1129 /** 1130 * Create a new instance of the ShellCommandExecutor to execute a command. 1131 * 1132 * @param execString The command to execute with arguments 1133 * @param dir If not-null, specifies the directory which should be set 1134 * as the current working directory for the command. 1135 * If null, the current working directory is not modified. 1136 * @param env If not-null, environment of the command will include the 1137 * key-value pairs specified in the map. If null, the current 1138 * environment is not modified. 1139 * @param timeout Specifies the time in milliseconds, after which the 1140 * command will be killed and the status marked as timed-out. 1141 * If 0, the command will not be timed out. 1142 * @param inheritParentEnv Indicates if the process should inherit the env 1143 * vars from the parent process or not. 1144 */ 1145 public ShellCommandExecutor(String[] execString, File dir, 1146 Map<String, String> env, long timeout, boolean inheritParentEnv) { 1147 command = execString.clone(); 1148 if (dir != null) { 1149 setWorkingDirectory(dir); 1150 } 1151 if (env != null) { 1152 setEnvironment(env); 1153 } 1154 timeOutInterval = timeout; 1155 this.inheritParentEnv = inheritParentEnv; 1156 } 1157 1158 /** 1159 * Execute the shell command. 1160 * @throws IOException if the command fails, or if the command is 1161 * not well constructed. 1162 */ 1163 public void execute() throws IOException { 1164 for (String s : command) { 1165 if (s == null) { 1166 throw new IOException("(null) entry in command string: " 1167 + StringUtils.join(" ", command)); 1168 } 1169 } 1170 this.run(); 1171 } 1172 1173 @Override 1174 public String[] getExecString() { 1175 return command; 1176 } 1177 1178 @Override 1179 protected void parseExecResult(BufferedReader lines) throws IOException { 1180 output = new StringBuffer(); 1181 char[] buf = new char[512]; 1182 int nRead; 1183 while ( (nRead = lines.read(buf, 0, buf.length)) > 0 ) { 1184 output.append(buf, 0, nRead); 1185 } 1186 } 1187 1188 /** Get the output of the shell command. */ 1189 public String getOutput() { 1190 return (output == null) ? "" : output.toString(); 1191 } 1192 1193 /** 1194 * Returns the commands of this instance. 1195 * Arguments with spaces in are presented with quotes round; other 1196 * arguments are presented raw 1197 * 1198 * @return a string representation of the object. 1199 */ 1200 @Override 1201 public String toString() { 1202 StringBuilder builder = new StringBuilder(); 1203 String[] args = getExecString(); 1204 for (String s : args) { 1205 if (s.indexOf(' ') >= 0) { 1206 builder.append('"').append(s).append('"'); 1207 } else { 1208 builder.append(s); 1209 } 1210 builder.append(' '); 1211 } 1212 return builder.toString(); 1213 } 1214 1215 @Override 1216 public void close() { 1217 } 1218 } 1219 1220 /** 1221 * To check if the passed script to shell command executor timed out or 1222 * not. 1223 * 1224 * @return if the script timed out. 1225 */ 1226 public boolean isTimedOut() { 1227 return timedOut.get(); 1228 } 1229 1230 /** 1231 * Declare that the command has timed out. 1232 * 1233 */ 1234 private void setTimedOut() { 1235 this.timedOut.set(true); 1236 } 1237 1238 /** 1239 * Static method to execute a shell command. 1240 * Covers most of the simple cases without requiring the user to implement 1241 * the <code>Shell</code> interface. 1242 * @param cmd shell command to execute. 1243 * @return the output of the executed command. 1244 */ 1245 public static String execCommand(String ... cmd) throws IOException { 1246 return execCommand(null, cmd, 0L); 1247 } 1248 1249 /** 1250 * Static method to execute a shell command. 1251 * Covers most of the simple cases without requiring the user to implement 1252 * the <code>Shell</code> interface. 1253 * @param env the map of environment key=value 1254 * @param cmd shell command to execute. 1255 * @param timeout time in milliseconds after which script should be marked timeout 1256 * @return the output of the executed command. 1257 * @throws IOException on any problem. 1258 */ 1259 1260 public static String execCommand(Map<String, String> env, String[] cmd, 1261 long timeout) throws IOException { 1262 ShellCommandExecutor exec = new ShellCommandExecutor(cmd, null, env, 1263 timeout); 1264 exec.execute(); 1265 return exec.getOutput(); 1266 } 1267 1268 /** 1269 * Static method to execute a shell command. 1270 * Covers most of the simple cases without requiring the user to implement 1271 * the <code>Shell</code> interface. 1272 * @param env the map of environment key=value 1273 * @param cmd shell command to execute. 1274 * @return the output of the executed command. 1275 * @throws IOException on any problem. 1276 */ 1277 public static String execCommand(Map<String,String> env, String ... cmd) 1278 throws IOException { 1279 return execCommand(env, cmd, 0L); 1280 } 1281 1282 /** 1283 * Timer which is used to timeout scripts spawned off by shell. 1284 */ 1285 private static class ShellTimeoutTimerTask extends TimerTask { 1286 1287 private final Shell shell; 1288 1289 public ShellTimeoutTimerTask(Shell shell) { 1290 this.shell = shell; 1291 } 1292 1293 @Override 1294 public void run() { 1295 Process p = shell.getProcess(); 1296 try { 1297 p.exitValue(); 1298 } catch (Exception e) { 1299 //Process has not terminated. 1300 //So check if it has completed 1301 //if not just destroy it. 1302 if (p != null && !shell.completed.get()) { 1303 shell.setTimedOut(); 1304 p.destroy(); 1305 } 1306 } 1307 } 1308 } 1309}