001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.hadoop.fs; 020 021import java.io.*; 022import java.util.ArrayList; 023import java.util.Arrays; 024import java.util.Enumeration; 025import java.util.List; 026import java.util.Map; 027import java.util.jar.Attributes; 028import java.util.jar.JarOutputStream; 029import java.util.jar.Manifest; 030import java.util.zip.GZIPInputStream; 031import java.util.zip.ZipEntry; 032import java.util.zip.ZipFile; 033 034import org.apache.commons.collections.map.CaseInsensitiveMap; 035import org.apache.commons.compress.archivers.tar.TarArchiveEntry; 036import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; 037import org.apache.hadoop.classification.InterfaceAudience; 038import org.apache.hadoop.classification.InterfaceStability; 039import org.apache.hadoop.conf.Configuration; 040import org.apache.hadoop.fs.permission.FsAction; 041import org.apache.hadoop.fs.permission.FsPermission; 042import org.apache.hadoop.io.IOUtils; 043import org.apache.hadoop.io.nativeio.NativeIO; 044import org.apache.hadoop.util.StringUtils; 045import org.apache.hadoop.util.Shell; 046import org.apache.hadoop.util.Shell.ShellCommandExecutor; 047import org.apache.commons.logging.Log; 048import org.apache.commons.logging.LogFactory; 049 050/** 051 * A collection of file-processing util methods 052 */ 053@InterfaceAudience.Public 054@InterfaceStability.Evolving 055public class FileUtil { 056 057 private static final Log LOG = LogFactory.getLog(FileUtil.class); 058 059 /* The error code is defined in winutils to indicate insufficient 060 * privilege to create symbolic links. This value need to keep in 061 * sync with the constant of the same name in: 062 * "src\winutils\common.h" 063 * */ 064 public static final int SYMLINK_NO_PRIVILEGE = 2; 065 066 /** 067 * convert an array of FileStatus to an array of Path 068 * 069 * @param stats 070 * an array of FileStatus objects 071 * @return an array of paths corresponding to the input 072 */ 073 public static Path[] stat2Paths(FileStatus[] stats) { 074 if (stats == null) 075 return null; 076 Path[] ret = new Path[stats.length]; 077 for (int i = 0; i < stats.length; ++i) { 078 ret[i] = stats[i].getPath(); 079 } 080 return ret; 081 } 082 083 /** 084 * convert an array of FileStatus to an array of Path. 085 * If stats if null, return path 086 * @param stats 087 * an array of FileStatus objects 088 * @param path 089 * default path to return in stats is null 090 * @return an array of paths corresponding to the input 091 */ 092 public static Path[] stat2Paths(FileStatus[] stats, Path path) { 093 if (stats == null) 094 return new Path[]{path}; 095 else 096 return stat2Paths(stats); 097 } 098 099 /** 100 * Delete a directory and all its contents. If 101 * we return false, the directory may be partially-deleted. 102 * (1) If dir is symlink to a file, the symlink is deleted. The file pointed 103 * to by the symlink is not deleted. 104 * (2) If dir is symlink to a directory, symlink is deleted. The directory 105 * pointed to by symlink is not deleted. 106 * (3) If dir is a normal file, it is deleted. 107 * (4) If dir is a normal directory, then dir and all its contents recursively 108 * are deleted. 109 */ 110 public static boolean fullyDelete(final File dir) { 111 return fullyDelete(dir, false); 112 } 113 114 /** 115 * Delete a directory and all its contents. If 116 * we return false, the directory may be partially-deleted. 117 * (1) If dir is symlink to a file, the symlink is deleted. The file pointed 118 * to by the symlink is not deleted. 119 * (2) If dir is symlink to a directory, symlink is deleted. The directory 120 * pointed to by symlink is not deleted. 121 * (3) If dir is a normal file, it is deleted. 122 * (4) If dir is a normal directory, then dir and all its contents recursively 123 * are deleted. 124 * @param dir the file or directory to be deleted 125 * @param tryGrantPermissions true if permissions should be modified to delete a file. 126 * @return true on success false on failure. 127 */ 128 public static boolean fullyDelete(final File dir, boolean tryGrantPermissions) { 129 if (tryGrantPermissions) { 130 // try to chmod +rwx the parent folder of the 'dir': 131 File parent = dir.getParentFile(); 132 grantPermissions(parent); 133 } 134 if (deleteImpl(dir, false)) { 135 // dir is (a) normal file, (b) symlink to a file, (c) empty directory or 136 // (d) symlink to a directory 137 return true; 138 } 139 // handle nonempty directory deletion 140 if (!fullyDeleteContents(dir, tryGrantPermissions)) { 141 return false; 142 } 143 return deleteImpl(dir, true); 144 } 145 146 /** 147 * Returns the target of the given symlink. Returns the empty string if 148 * the given path does not refer to a symlink or there is an error 149 * accessing the symlink. 150 * @param f File representing the symbolic link. 151 * @return The target of the symbolic link, empty string on error or if not 152 * a symlink. 153 */ 154 public static String readLink(File f) { 155 /* NB: Use readSymbolicLink in java.nio.file.Path once available. Could 156 * use getCanonicalPath in File to get the target of the symlink but that 157 * does not indicate if the given path refers to a symlink. 158 */ 159 try { 160 return Shell.execCommand( 161 Shell.getReadlinkCommand(f.toString())).trim(); 162 } catch (IOException x) { 163 return ""; 164 } 165 } 166 167 /* 168 * Pure-Java implementation of "chmod +rwx f". 169 */ 170 private static void grantPermissions(final File f) { 171 FileUtil.setExecutable(f, true); 172 FileUtil.setReadable(f, true); 173 FileUtil.setWritable(f, true); 174 } 175 176 private static boolean deleteImpl(final File f, final boolean doLog) { 177 if (f == null) { 178 LOG.warn("null file argument."); 179 return false; 180 } 181 final boolean wasDeleted = f.delete(); 182 if (wasDeleted) { 183 return true; 184 } 185 final boolean ex = f.exists(); 186 if (doLog && ex) { 187 LOG.warn("Failed to delete file or dir [" 188 + f.getAbsolutePath() + "]: it still exists."); 189 } 190 return !ex; 191 } 192 193 /** 194 * Delete the contents of a directory, not the directory itself. If 195 * we return false, the directory may be partially-deleted. 196 * If dir is a symlink to a directory, all the contents of the actual 197 * directory pointed to by dir will be deleted. 198 */ 199 public static boolean fullyDeleteContents(final File dir) { 200 return fullyDeleteContents(dir, false); 201 } 202 203 /** 204 * Delete the contents of a directory, not the directory itself. If 205 * we return false, the directory may be partially-deleted. 206 * If dir is a symlink to a directory, all the contents of the actual 207 * directory pointed to by dir will be deleted. 208 * @param tryGrantPermissions if 'true', try grant +rwx permissions to this 209 * and all the underlying directories before trying to delete their contents. 210 */ 211 public static boolean fullyDeleteContents(final File dir, final boolean tryGrantPermissions) { 212 if (tryGrantPermissions) { 213 // to be able to list the dir and delete files from it 214 // we must grant the dir rwx permissions: 215 grantPermissions(dir); 216 } 217 boolean deletionSucceeded = true; 218 final File[] contents = dir.listFiles(); 219 if (contents != null) { 220 for (int i = 0; i < contents.length; i++) { 221 if (contents[i].isFile()) { 222 if (!deleteImpl(contents[i], true)) {// normal file or symlink to another file 223 deletionSucceeded = false; 224 continue; // continue deletion of other files/dirs under dir 225 } 226 } else { 227 // Either directory or symlink to another directory. 228 // Try deleting the directory as this might be a symlink 229 boolean b = false; 230 b = deleteImpl(contents[i], false); 231 if (b){ 232 //this was indeed a symlink or an empty directory 233 continue; 234 } 235 // if not an empty directory or symlink let 236 // fullydelete handle it. 237 if (!fullyDelete(contents[i], tryGrantPermissions)) { 238 deletionSucceeded = false; 239 // continue deletion of other files/dirs under dir 240 } 241 } 242 } 243 } 244 return deletionSucceeded; 245 } 246 247 /** 248 * Recursively delete a directory. 249 * 250 * @param fs {@link FileSystem} on which the path is present 251 * @param dir directory to recursively delete 252 * @throws IOException 253 * @deprecated Use {@link FileSystem#delete(Path, boolean)} 254 */ 255 @Deprecated 256 public static void fullyDelete(FileSystem fs, Path dir) 257 throws IOException { 258 fs.delete(dir, true); 259 } 260 261 // 262 // If the destination is a subdirectory of the source, then 263 // generate exception 264 // 265 private static void checkDependencies(FileSystem srcFS, 266 Path src, 267 FileSystem dstFS, 268 Path dst) 269 throws IOException { 270 if (srcFS == dstFS) { 271 String srcq = src.makeQualified(srcFS).toString() + Path.SEPARATOR; 272 String dstq = dst.makeQualified(dstFS).toString() + Path.SEPARATOR; 273 if (dstq.startsWith(srcq)) { 274 if (srcq.length() == dstq.length()) { 275 throw new IOException("Cannot copy " + src + " to itself."); 276 } else { 277 throw new IOException("Cannot copy " + src + " to its subdirectory " + 278 dst); 279 } 280 } 281 } 282 } 283 284 /** Copy files between FileSystems. */ 285 public static boolean copy(FileSystem srcFS, Path src, 286 FileSystem dstFS, Path dst, 287 boolean deleteSource, 288 Configuration conf) throws IOException { 289 return copy(srcFS, src, dstFS, dst, deleteSource, true, conf); 290 } 291 292 public static boolean copy(FileSystem srcFS, Path[] srcs, 293 FileSystem dstFS, Path dst, 294 boolean deleteSource, 295 boolean overwrite, Configuration conf) 296 throws IOException { 297 boolean gotException = false; 298 boolean returnVal = true; 299 StringBuilder exceptions = new StringBuilder(); 300 301 if (srcs.length == 1) 302 return copy(srcFS, srcs[0], dstFS, dst, deleteSource, overwrite, conf); 303 304 // Check if dest is directory 305 if (!dstFS.exists(dst)) { 306 throw new IOException("`" + dst +"': specified destination directory " + 307 "does not exist"); 308 } else { 309 FileStatus sdst = dstFS.getFileStatus(dst); 310 if (!sdst.isDirectory()) 311 throw new IOException("copying multiple files, but last argument `" + 312 dst + "' is not a directory"); 313 } 314 315 for (Path src : srcs) { 316 try { 317 if (!copy(srcFS, src, dstFS, dst, deleteSource, overwrite, conf)) 318 returnVal = false; 319 } catch (IOException e) { 320 gotException = true; 321 exceptions.append(e.getMessage()); 322 exceptions.append("\n"); 323 } 324 } 325 if (gotException) { 326 throw new IOException(exceptions.toString()); 327 } 328 return returnVal; 329 } 330 331 /** Copy files between FileSystems. */ 332 public static boolean copy(FileSystem srcFS, Path src, 333 FileSystem dstFS, Path dst, 334 boolean deleteSource, 335 boolean overwrite, 336 Configuration conf) throws IOException { 337 FileStatus fileStatus = srcFS.getFileStatus(src); 338 return copy(srcFS, fileStatus, dstFS, dst, deleteSource, overwrite, conf); 339 } 340 341 /** Copy files between FileSystems. */ 342 public static boolean copy(FileSystem srcFS, FileStatus srcStatus, 343 FileSystem dstFS, Path dst, 344 boolean deleteSource, 345 boolean overwrite, 346 Configuration conf) throws IOException { 347 Path src = srcStatus.getPath(); 348 dst = checkDest(src.getName(), dstFS, dst, overwrite); 349 if (srcStatus.isDirectory()) { 350 checkDependencies(srcFS, src, dstFS, dst); 351 if (!dstFS.mkdirs(dst)) { 352 return false; 353 } 354 FileStatus contents[] = srcFS.listStatus(src); 355 for (int i = 0; i < contents.length; i++) { 356 copy(srcFS, contents[i], dstFS, 357 new Path(dst, contents[i].getPath().getName()), 358 deleteSource, overwrite, conf); 359 } 360 } else { 361 InputStream in=null; 362 OutputStream out = null; 363 try { 364 in = srcFS.open(src); 365 out = dstFS.create(dst, overwrite); 366 IOUtils.copyBytes(in, out, conf, true); 367 } catch (IOException e) { 368 IOUtils.closeStream(out); 369 IOUtils.closeStream(in); 370 throw e; 371 } 372 } 373 if (deleteSource) { 374 return srcFS.delete(src, true); 375 } else { 376 return true; 377 } 378 379 } 380 381 /** Copy all files in a directory to one output file (merge). */ 382 public static boolean copyMerge(FileSystem srcFS, Path srcDir, 383 FileSystem dstFS, Path dstFile, 384 boolean deleteSource, 385 Configuration conf, String addString) throws IOException { 386 dstFile = checkDest(srcDir.getName(), dstFS, dstFile, false); 387 388 if (!srcFS.getFileStatus(srcDir).isDirectory()) 389 return false; 390 391 OutputStream out = dstFS.create(dstFile); 392 393 try { 394 FileStatus contents[] = srcFS.listStatus(srcDir); 395 Arrays.sort(contents); 396 for (int i = 0; i < contents.length; i++) { 397 if (contents[i].isFile()) { 398 InputStream in = srcFS.open(contents[i].getPath()); 399 try { 400 IOUtils.copyBytes(in, out, conf, false); 401 if (addString!=null) 402 out.write(addString.getBytes("UTF-8")); 403 404 } finally { 405 in.close(); 406 } 407 } 408 } 409 } finally { 410 out.close(); 411 } 412 413 414 if (deleteSource) { 415 return srcFS.delete(srcDir, true); 416 } else { 417 return true; 418 } 419 } 420 421 /** Copy local files to a FileSystem. */ 422 public static boolean copy(File src, 423 FileSystem dstFS, Path dst, 424 boolean deleteSource, 425 Configuration conf) throws IOException { 426 dst = checkDest(src.getName(), dstFS, dst, false); 427 428 if (src.isDirectory()) { 429 if (!dstFS.mkdirs(dst)) { 430 return false; 431 } 432 File contents[] = listFiles(src); 433 for (int i = 0; i < contents.length; i++) { 434 copy(contents[i], dstFS, new Path(dst, contents[i].getName()), 435 deleteSource, conf); 436 } 437 } else if (src.isFile()) { 438 InputStream in = null; 439 OutputStream out =null; 440 try { 441 in = new FileInputStream(src); 442 out = dstFS.create(dst); 443 IOUtils.copyBytes(in, out, conf); 444 } catch (IOException e) { 445 IOUtils.closeStream( out ); 446 IOUtils.closeStream( in ); 447 throw e; 448 } 449 } else { 450 throw new IOException(src.toString() + 451 ": No such file or directory"); 452 } 453 if (deleteSource) { 454 return FileUtil.fullyDelete(src); 455 } else { 456 return true; 457 } 458 } 459 460 /** Copy FileSystem files to local files. */ 461 public static boolean copy(FileSystem srcFS, Path src, 462 File dst, boolean deleteSource, 463 Configuration conf) throws IOException { 464 FileStatus filestatus = srcFS.getFileStatus(src); 465 return copy(srcFS, filestatus, dst, deleteSource, conf); 466 } 467 468 /** Copy FileSystem files to local files. */ 469 private static boolean copy(FileSystem srcFS, FileStatus srcStatus, 470 File dst, boolean deleteSource, 471 Configuration conf) throws IOException { 472 Path src = srcStatus.getPath(); 473 if (srcStatus.isDirectory()) { 474 if (!dst.mkdirs()) { 475 return false; 476 } 477 FileStatus contents[] = srcFS.listStatus(src); 478 for (int i = 0; i < contents.length; i++) { 479 copy(srcFS, contents[i], 480 new File(dst, contents[i].getPath().getName()), 481 deleteSource, conf); 482 } 483 } else { 484 InputStream in = srcFS.open(src); 485 IOUtils.copyBytes(in, new FileOutputStream(dst), conf); 486 } 487 if (deleteSource) { 488 return srcFS.delete(src, true); 489 } else { 490 return true; 491 } 492 } 493 494 private static Path checkDest(String srcName, FileSystem dstFS, Path dst, 495 boolean overwrite) throws IOException { 496 if (dstFS.exists(dst)) { 497 FileStatus sdst = dstFS.getFileStatus(dst); 498 if (sdst.isDirectory()) { 499 if (null == srcName) { 500 throw new IOException("Target " + dst + " is a directory"); 501 } 502 return checkDest(null, dstFS, new Path(dst, srcName), overwrite); 503 } else if (!overwrite) { 504 throw new IOException("Target " + dst + " already exists"); 505 } 506 } 507 return dst; 508 } 509 510 /** 511 * Convert a os-native filename to a path that works for the shell. 512 * @param filename The filename to convert 513 * @return The unix pathname 514 * @throws IOException on windows, there can be problems with the subprocess 515 */ 516 public static String makeShellPath(String filename) throws IOException { 517 return filename; 518 } 519 520 /** 521 * Convert a os-native filename to a path that works for the shell. 522 * @param file The filename to convert 523 * @return The unix pathname 524 * @throws IOException on windows, there can be problems with the subprocess 525 */ 526 public static String makeShellPath(File file) throws IOException { 527 return makeShellPath(file, false); 528 } 529 530 /** 531 * Convert a os-native filename to a path that works for the shell. 532 * @param file The filename to convert 533 * @param makeCanonicalPath 534 * Whether to make canonical path for the file passed 535 * @return The unix pathname 536 * @throws IOException on windows, there can be problems with the subprocess 537 */ 538 public static String makeShellPath(File file, boolean makeCanonicalPath) 539 throws IOException { 540 if (makeCanonicalPath) { 541 return makeShellPath(file.getCanonicalPath()); 542 } else { 543 return makeShellPath(file.toString()); 544 } 545 } 546 547 /** 548 * Takes an input dir and returns the du on that local directory. Very basic 549 * implementation. 550 * 551 * @param dir 552 * The input dir to get the disk space of this local dir 553 * @return The total disk space of the input local directory 554 */ 555 public static long getDU(File dir) { 556 long size = 0; 557 if (!dir.exists()) 558 return 0; 559 if (!dir.isDirectory()) { 560 return dir.length(); 561 } else { 562 File[] allFiles = dir.listFiles(); 563 if(allFiles != null) { 564 for (int i = 0; i < allFiles.length; i++) { 565 boolean isSymLink; 566 try { 567 isSymLink = org.apache.commons.io.FileUtils.isSymlink(allFiles[i]); 568 } catch(IOException ioe) { 569 isSymLink = true; 570 } 571 if(!isSymLink) { 572 size += getDU(allFiles[i]); 573 } 574 } 575 } 576 return size; 577 } 578 } 579 580 /** 581 * Given a File input it will unzip the file in a the unzip directory 582 * passed as the second parameter 583 * @param inFile The zip file as input 584 * @param unzipDir The unzip directory where to unzip the zip file. 585 * @throws IOException 586 */ 587 public static void unZip(File inFile, File unzipDir) throws IOException { 588 Enumeration<? extends ZipEntry> entries; 589 ZipFile zipFile = new ZipFile(inFile); 590 591 try { 592 entries = zipFile.entries(); 593 while (entries.hasMoreElements()) { 594 ZipEntry entry = entries.nextElement(); 595 if (!entry.isDirectory()) { 596 InputStream in = zipFile.getInputStream(entry); 597 try { 598 File file = new File(unzipDir, entry.getName()); 599 if (!file.getParentFile().mkdirs()) { 600 if (!file.getParentFile().isDirectory()) { 601 throw new IOException("Mkdirs failed to create " + 602 file.getParentFile().toString()); 603 } 604 } 605 OutputStream out = new FileOutputStream(file); 606 try { 607 byte[] buffer = new byte[8192]; 608 int i; 609 while ((i = in.read(buffer)) != -1) { 610 out.write(buffer, 0, i); 611 } 612 } finally { 613 out.close(); 614 } 615 } finally { 616 in.close(); 617 } 618 } 619 } 620 } finally { 621 zipFile.close(); 622 } 623 } 624 625 /** 626 * Given a Tar File as input it will untar the file in a the untar directory 627 * passed as the second parameter 628 * 629 * This utility will untar ".tar" files and ".tar.gz","tgz" files. 630 * 631 * @param inFile The tar file as input. 632 * @param untarDir The untar directory where to untar the tar file. 633 * @throws IOException 634 */ 635 public static void unTar(File inFile, File untarDir) throws IOException { 636 if (!untarDir.mkdirs()) { 637 if (!untarDir.isDirectory()) { 638 throw new IOException("Mkdirs failed to create " + untarDir); 639 } 640 } 641 642 boolean gzipped = inFile.toString().endsWith("gz"); 643 if(Shell.WINDOWS) { 644 // Tar is not native to Windows. Use simple Java based implementation for 645 // tests and simple tar archives 646 unTarUsingJava(inFile, untarDir, gzipped); 647 } 648 else { 649 // spawn tar utility to untar archive for full fledged unix behavior such 650 // as resolving symlinks in tar archives 651 unTarUsingTar(inFile, untarDir, gzipped); 652 } 653 } 654 655 private static void unTarUsingTar(File inFile, File untarDir, 656 boolean gzipped) throws IOException { 657 StringBuffer untarCommand = new StringBuffer(); 658 if (gzipped) { 659 untarCommand.append(" gzip -dc '"); 660 untarCommand.append(FileUtil.makeShellPath(inFile)); 661 untarCommand.append("' | ("); 662 } 663 untarCommand.append("cd '"); 664 untarCommand.append(FileUtil.makeShellPath(untarDir)); 665 untarCommand.append("' ; "); 666 untarCommand.append("tar -xf "); 667 668 if (gzipped) { 669 untarCommand.append(" -)"); 670 } else { 671 untarCommand.append(FileUtil.makeShellPath(inFile)); 672 } 673 String[] shellCmd = { "bash", "-c", untarCommand.toString() }; 674 ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd); 675 shexec.execute(); 676 int exitcode = shexec.getExitCode(); 677 if (exitcode != 0) { 678 throw new IOException("Error untarring file " + inFile + 679 ". Tar process exited with exit code " + exitcode); 680 } 681 } 682 683 private static void unTarUsingJava(File inFile, File untarDir, 684 boolean gzipped) throws IOException { 685 InputStream inputStream = null; 686 TarArchiveInputStream tis = null; 687 try { 688 if (gzipped) { 689 inputStream = new BufferedInputStream(new GZIPInputStream( 690 new FileInputStream(inFile))); 691 } else { 692 inputStream = new BufferedInputStream(new FileInputStream(inFile)); 693 } 694 695 tis = new TarArchiveInputStream(inputStream); 696 697 for (TarArchiveEntry entry = tis.getNextTarEntry(); entry != null;) { 698 unpackEntries(tis, entry, untarDir); 699 entry = tis.getNextTarEntry(); 700 } 701 } finally { 702 IOUtils.cleanup(LOG, tis, inputStream); 703 } 704 } 705 706 private static void unpackEntries(TarArchiveInputStream tis, 707 TarArchiveEntry entry, File outputDir) throws IOException { 708 if (entry.isDirectory()) { 709 File subDir = new File(outputDir, entry.getName()); 710 if (!subDir.mkdirs() && !subDir.isDirectory()) { 711 throw new IOException("Mkdirs failed to create tar internal dir " 712 + outputDir); 713 } 714 715 for (TarArchiveEntry e : entry.getDirectoryEntries()) { 716 unpackEntries(tis, e, subDir); 717 } 718 719 return; 720 } 721 722 File outputFile = new File(outputDir, entry.getName()); 723 if (!outputFile.getParentFile().exists()) { 724 if (!outputFile.getParentFile().mkdirs()) { 725 throw new IOException("Mkdirs failed to create tar internal dir " 726 + outputDir); 727 } 728 } 729 730 int count; 731 byte data[] = new byte[2048]; 732 try (BufferedOutputStream outputStream = new BufferedOutputStream( 733 new FileOutputStream(outputFile));) { 734 735 while ((count = tis.read(data)) != -1) { 736 outputStream.write(data, 0, count); 737 } 738 739 outputStream.flush(); 740 } 741 } 742 743 /** 744 * Class for creating hardlinks. 745 * Supports Unix, WindXP. 746 * @deprecated Use {@link org.apache.hadoop.fs.HardLink} 747 */ 748 @Deprecated 749 public static class HardLink extends org.apache.hadoop.fs.HardLink { 750 // This is a stub to assist with coordinated change between 751 // COMMON and HDFS projects. It will be removed after the 752 // corresponding change is committed to HDFS. 753 } 754 755 /** 756 * Create a soft link between a src and destination 757 * only on a local disk. HDFS does not support this. 758 * On Windows, when symlink creation fails due to security 759 * setting, we will log a warning. The return code in this 760 * case is 2. 761 * 762 * @param target the target for symlink 763 * @param linkname the symlink 764 * @return 0 on success 765 */ 766 public static int symLink(String target, String linkname) throws IOException{ 767 // Run the input paths through Java's File so that they are converted to the 768 // native OS form 769 File targetFile = new File( 770 Path.getPathWithoutSchemeAndAuthority(new Path(target)).toString()); 771 File linkFile = new File( 772 Path.getPathWithoutSchemeAndAuthority(new Path(linkname)).toString()); 773 774 // If not on Java7+, copy a file instead of creating a symlink since 775 // Java6 has close to no support for symlinks on Windows. Specifically 776 // File#length and File#renameTo do not work as expected. 777 // (see HADOOP-9061 for additional details) 778 // We still create symlinks for directories, since the scenario in this 779 // case is different. The directory content could change in which 780 // case the symlink loses its purpose (for example task attempt log folder 781 // is symlinked under userlogs and userlogs are generated afterwards). 782 if (Shell.WINDOWS && !Shell.isJava7OrAbove() && targetFile.isFile()) { 783 try { 784 LOG.warn("FileUtil#symlink: On Windows+Java6, copying file instead " + 785 "of creating a symlink. Copying " + target + " -> " + linkname); 786 787 if (!linkFile.getParentFile().exists()) { 788 LOG.warn("Parent directory " + linkFile.getParent() + 789 " does not exist."); 790 return 1; 791 } else { 792 org.apache.commons.io.FileUtils.copyFile(targetFile, linkFile); 793 } 794 } catch (IOException ex) { 795 LOG.warn("FileUtil#symlink failed to copy the file with error: " 796 + ex.getMessage()); 797 // Exit with non-zero exit code 798 return 1; 799 } 800 return 0; 801 } 802 803 String[] cmd = Shell.getSymlinkCommand( 804 targetFile.toString(), 805 linkFile.toString()); 806 807 ShellCommandExecutor shExec; 808 try { 809 if (Shell.WINDOWS && 810 linkFile.getParentFile() != null && 811 !new Path(target).isAbsolute()) { 812 // Relative links on Windows must be resolvable at the time of 813 // creation. To ensure this we run the shell command in the directory 814 // of the link. 815 // 816 shExec = new ShellCommandExecutor(cmd, linkFile.getParentFile()); 817 } else { 818 shExec = new ShellCommandExecutor(cmd); 819 } 820 shExec.execute(); 821 } catch (Shell.ExitCodeException ec) { 822 int returnVal = ec.getExitCode(); 823 if (Shell.WINDOWS && returnVal == SYMLINK_NO_PRIVILEGE) { 824 LOG.warn("Fail to create symbolic links on Windows. " 825 + "The default security settings in Windows disallow non-elevated " 826 + "administrators and all non-administrators from creating symbolic links. " 827 + "This behavior can be changed in the Local Security Policy management console"); 828 } else if (returnVal != 0) { 829 LOG.warn("Command '" + StringUtils.join(" ", cmd) + "' failed " 830 + returnVal + " with: " + ec.getMessage()); 831 } 832 return returnVal; 833 } catch (IOException e) { 834 if (LOG.isDebugEnabled()) { 835 LOG.debug("Error while create symlink " + linkname + " to " + target 836 + "." + " Exception: " + StringUtils.stringifyException(e)); 837 } 838 throw e; 839 } 840 return shExec.getExitCode(); 841 } 842 843 /** 844 * Change the permissions on a filename. 845 * @param filename the name of the file to change 846 * @param perm the permission string 847 * @return the exit code from the command 848 * @throws IOException 849 * @throws InterruptedException 850 */ 851 public static int chmod(String filename, String perm 852 ) throws IOException, InterruptedException { 853 return chmod(filename, perm, false); 854 } 855 856 /** 857 * Change the permissions on a file / directory, recursively, if 858 * needed. 859 * @param filename name of the file whose permissions are to change 860 * @param perm permission string 861 * @param recursive true, if permissions should be changed recursively 862 * @return the exit code from the command. 863 * @throws IOException 864 */ 865 public static int chmod(String filename, String perm, boolean recursive) 866 throws IOException { 867 String [] cmd = Shell.getSetPermissionCommand(perm, recursive); 868 String[] args = new String[cmd.length + 1]; 869 System.arraycopy(cmd, 0, args, 0, cmd.length); 870 args[cmd.length] = new File(filename).getPath(); 871 ShellCommandExecutor shExec = new ShellCommandExecutor(args); 872 try { 873 shExec.execute(); 874 }catch(IOException e) { 875 if(LOG.isDebugEnabled()) { 876 LOG.debug("Error while changing permission : " + filename 877 +" Exception: " + StringUtils.stringifyException(e)); 878 } 879 } 880 return shExec.getExitCode(); 881 } 882 883 /** 884 * Set the ownership on a file / directory. User name and group name 885 * cannot both be null. 886 * @param file the file to change 887 * @param username the new user owner name 888 * @param groupname the new group owner name 889 * @throws IOException 890 */ 891 public static void setOwner(File file, String username, 892 String groupname) throws IOException { 893 if (username == null && groupname == null) { 894 throw new IOException("username == null && groupname == null"); 895 } 896 String arg = (username == null ? "" : username) 897 + (groupname == null ? "" : ":" + groupname); 898 String [] cmd = Shell.getSetOwnerCommand(arg); 899 execCommand(file, cmd); 900 } 901 902 /** 903 * Platform independent implementation for {@link File#setReadable(boolean)} 904 * File#setReadable does not work as expected on Windows. 905 * @param f input file 906 * @param readable 907 * @return true on success, false otherwise 908 */ 909 public static boolean setReadable(File f, boolean readable) { 910 if (Shell.WINDOWS) { 911 try { 912 String permission = readable ? "u+r" : "u-r"; 913 FileUtil.chmod(f.getCanonicalPath(), permission, false); 914 return true; 915 } catch (IOException ex) { 916 return false; 917 } 918 } else { 919 return f.setReadable(readable); 920 } 921 } 922 923 /** 924 * Platform independent implementation for {@link File#setWritable(boolean)} 925 * File#setWritable does not work as expected on Windows. 926 * @param f input file 927 * @param writable 928 * @return true on success, false otherwise 929 */ 930 public static boolean setWritable(File f, boolean writable) { 931 if (Shell.WINDOWS) { 932 try { 933 String permission = writable ? "u+w" : "u-w"; 934 FileUtil.chmod(f.getCanonicalPath(), permission, false); 935 return true; 936 } catch (IOException ex) { 937 return false; 938 } 939 } else { 940 return f.setWritable(writable); 941 } 942 } 943 944 /** 945 * Platform independent implementation for {@link File#setExecutable(boolean)} 946 * File#setExecutable does not work as expected on Windows. 947 * Note: revoking execute permission on folders does not have the same 948 * behavior on Windows as on Unix platforms. Creating, deleting or renaming 949 * a file within that folder will still succeed on Windows. 950 * @param f input file 951 * @param executable 952 * @return true on success, false otherwise 953 */ 954 public static boolean setExecutable(File f, boolean executable) { 955 if (Shell.WINDOWS) { 956 try { 957 String permission = executable ? "u+x" : "u-x"; 958 FileUtil.chmod(f.getCanonicalPath(), permission, false); 959 return true; 960 } catch (IOException ex) { 961 return false; 962 } 963 } else { 964 return f.setExecutable(executable); 965 } 966 } 967 968 /** 969 * Platform independent implementation for {@link File#canRead()} 970 * @param f input file 971 * @return On Unix, same as {@link File#canRead()} 972 * On Windows, true if process has read access on the path 973 */ 974 public static boolean canRead(File f) { 975 if (Shell.WINDOWS) { 976 try { 977 return NativeIO.Windows.access(f.getCanonicalPath(), 978 NativeIO.Windows.AccessRight.ACCESS_READ); 979 } catch (IOException e) { 980 return false; 981 } 982 } else { 983 return f.canRead(); 984 } 985 } 986 987 /** 988 * Platform independent implementation for {@link File#canWrite()} 989 * @param f input file 990 * @return On Unix, same as {@link File#canWrite()} 991 * On Windows, true if process has write access on the path 992 */ 993 public static boolean canWrite(File f) { 994 if (Shell.WINDOWS) { 995 try { 996 return NativeIO.Windows.access(f.getCanonicalPath(), 997 NativeIO.Windows.AccessRight.ACCESS_WRITE); 998 } catch (IOException e) { 999 return false; 1000 } 1001 } else { 1002 return f.canWrite(); 1003 } 1004 } 1005 1006 /** 1007 * Platform independent implementation for {@link File#canExecute()} 1008 * @param f input file 1009 * @return On Unix, same as {@link File#canExecute()} 1010 * On Windows, true if process has execute access on the path 1011 */ 1012 public static boolean canExecute(File f) { 1013 if (Shell.WINDOWS) { 1014 try { 1015 return NativeIO.Windows.access(f.getCanonicalPath(), 1016 NativeIO.Windows.AccessRight.ACCESS_EXECUTE); 1017 } catch (IOException e) { 1018 return false; 1019 } 1020 } else { 1021 return f.canExecute(); 1022 } 1023 } 1024 1025 /** 1026 * Set permissions to the required value. Uses the java primitives instead 1027 * of forking if group == other. 1028 * @param f the file to change 1029 * @param permission the new permissions 1030 * @throws IOException 1031 */ 1032 public static void setPermission(File f, FsPermission permission 1033 ) throws IOException { 1034 FsAction user = permission.getUserAction(); 1035 FsAction group = permission.getGroupAction(); 1036 FsAction other = permission.getOtherAction(); 1037 1038 // use the native/fork if the group/other permissions are different 1039 // or if the native is available or on Windows 1040 if (group != other || NativeIO.isAvailable() || Shell.WINDOWS) { 1041 execSetPermission(f, permission); 1042 return; 1043 } 1044 1045 boolean rv = true; 1046 1047 // read perms 1048 rv = f.setReadable(group.implies(FsAction.READ), false); 1049 checkReturnValue(rv, f, permission); 1050 if (group.implies(FsAction.READ) != user.implies(FsAction.READ)) { 1051 rv = f.setReadable(user.implies(FsAction.READ), true); 1052 checkReturnValue(rv, f, permission); 1053 } 1054 1055 // write perms 1056 rv = f.setWritable(group.implies(FsAction.WRITE), false); 1057 checkReturnValue(rv, f, permission); 1058 if (group.implies(FsAction.WRITE) != user.implies(FsAction.WRITE)) { 1059 rv = f.setWritable(user.implies(FsAction.WRITE), true); 1060 checkReturnValue(rv, f, permission); 1061 } 1062 1063 // exec perms 1064 rv = f.setExecutable(group.implies(FsAction.EXECUTE), false); 1065 checkReturnValue(rv, f, permission); 1066 if (group.implies(FsAction.EXECUTE) != user.implies(FsAction.EXECUTE)) { 1067 rv = f.setExecutable(user.implies(FsAction.EXECUTE), true); 1068 checkReturnValue(rv, f, permission); 1069 } 1070 } 1071 1072 private static void checkReturnValue(boolean rv, File p, 1073 FsPermission permission 1074 ) throws IOException { 1075 if (!rv) { 1076 throw new IOException("Failed to set permissions of path: " + p + 1077 " to " + 1078 String.format("%04o", permission.toShort())); 1079 } 1080 } 1081 1082 private static void execSetPermission(File f, 1083 FsPermission permission 1084 ) throws IOException { 1085 if (NativeIO.isAvailable()) { 1086 NativeIO.POSIX.chmod(f.getCanonicalPath(), permission.toShort()); 1087 } else { 1088 execCommand(f, Shell.getSetPermissionCommand( 1089 String.format("%04o", permission.toShort()), false)); 1090 } 1091 } 1092 1093 static String execCommand(File f, String... cmd) throws IOException { 1094 String[] args = new String[cmd.length + 1]; 1095 System.arraycopy(cmd, 0, args, 0, cmd.length); 1096 args[cmd.length] = f.getCanonicalPath(); 1097 String output = Shell.execCommand(args); 1098 return output; 1099 } 1100 1101 /** 1102 * Create a tmp file for a base file. 1103 * @param basefile the base file of the tmp 1104 * @param prefix file name prefix of tmp 1105 * @param isDeleteOnExit if true, the tmp will be deleted when the VM exits 1106 * @return a newly created tmp file 1107 * @exception IOException If a tmp file cannot created 1108 * @see java.io.File#createTempFile(String, String, File) 1109 * @see java.io.File#deleteOnExit() 1110 */ 1111 public static final File createLocalTempFile(final File basefile, 1112 final String prefix, 1113 final boolean isDeleteOnExit) 1114 throws IOException { 1115 File tmp = File.createTempFile(prefix + basefile.getName(), 1116 "", basefile.getParentFile()); 1117 if (isDeleteOnExit) { 1118 tmp.deleteOnExit(); 1119 } 1120 return tmp; 1121 } 1122 1123 /** 1124 * Move the src file to the name specified by target. 1125 * @param src the source file 1126 * @param target the target file 1127 * @exception IOException If this operation fails 1128 */ 1129 public static void replaceFile(File src, File target) throws IOException { 1130 /* renameTo() has two limitations on Windows platform. 1131 * src.renameTo(target) fails if 1132 * 1) If target already exists OR 1133 * 2) If target is already open for reading/writing. 1134 */ 1135 if (!src.renameTo(target)) { 1136 int retries = 5; 1137 while (target.exists() && !target.delete() && retries-- >= 0) { 1138 try { 1139 Thread.sleep(1000); 1140 } catch (InterruptedException e) { 1141 throw new IOException("replaceFile interrupted."); 1142 } 1143 } 1144 if (!src.renameTo(target)) { 1145 throw new IOException("Unable to rename " + src + 1146 " to " + target); 1147 } 1148 } 1149 } 1150 1151 /** 1152 * A wrapper for {@link File#listFiles()}. This java.io API returns null 1153 * when a dir is not a directory or for any I/O error. Instead of having 1154 * null check everywhere File#listFiles() is used, we will add utility API 1155 * to get around this problem. For the majority of cases where we prefer 1156 * an IOException to be thrown. 1157 * @param dir directory for which listing should be performed 1158 * @return list of files or empty list 1159 * @exception IOException for invalid directory or for a bad disk. 1160 */ 1161 public static File[] listFiles(File dir) throws IOException { 1162 File[] files = dir.listFiles(); 1163 if(files == null) { 1164 throw new IOException("Invalid directory or I/O error occurred for dir: " 1165 + dir.toString()); 1166 } 1167 return files; 1168 } 1169 1170 /** 1171 * A wrapper for {@link File#list()}. This java.io API returns null 1172 * when a dir is not a directory or for any I/O error. Instead of having 1173 * null check everywhere File#list() is used, we will add utility API 1174 * to get around this problem. For the majority of cases where we prefer 1175 * an IOException to be thrown. 1176 * @param dir directory for which listing should be performed 1177 * @return list of file names or empty string list 1178 * @exception IOException for invalid directory or for a bad disk. 1179 */ 1180 public static String[] list(File dir) throws IOException { 1181 String[] fileNames = dir.list(); 1182 if(fileNames == null) { 1183 throw new IOException("Invalid directory or I/O error occurred for dir: " 1184 + dir.toString()); 1185 } 1186 return fileNames; 1187 } 1188 1189 public static String[] createJarWithClassPath(String inputClassPath, Path pwd, 1190 Map<String, String> callerEnv) throws IOException { 1191 return createJarWithClassPath(inputClassPath, pwd, pwd, callerEnv); 1192 } 1193 1194 /** 1195 * Create a jar file at the given path, containing a manifest with a classpath 1196 * that references all specified entries. 1197 * 1198 * Some platforms may have an upper limit on command line length. For example, 1199 * the maximum command line length on Windows is 8191 characters, but the 1200 * length of the classpath may exceed this. To work around this limitation, 1201 * use this method to create a small intermediate jar with a manifest that 1202 * contains the full classpath. It returns the absolute path to the new jar, 1203 * which the caller may set as the classpath for a new process. 1204 * 1205 * Environment variable evaluation is not supported within a jar manifest, so 1206 * this method expands environment variables before inserting classpath entries 1207 * to the manifest. The method parses environment variables according to 1208 * platform-specific syntax (%VAR% on Windows, or $VAR otherwise). On Windows, 1209 * environment variables are case-insensitive. For example, %VAR% and %var% 1210 * evaluate to the same value. 1211 * 1212 * Specifying the classpath in a jar manifest does not support wildcards, so 1213 * this method expands wildcards internally. Any classpath entry that ends 1214 * with * is translated to all files at that path with extension .jar or .JAR. 1215 * 1216 * @param inputClassPath String input classpath to bundle into the jar manifest 1217 * @param pwd Path to working directory to save jar 1218 * @param targetDir path to where the jar execution will have its working dir 1219 * @param callerEnv Map<String, String> caller's environment variables to use 1220 * for expansion 1221 * @return String[] with absolute path to new jar in position 0 and 1222 * unexpanded wild card entry path in position 1 1223 * @throws IOException if there is an I/O error while writing the jar file 1224 */ 1225 public static String[] createJarWithClassPath(String inputClassPath, Path pwd, 1226 Path targetDir, 1227 Map<String, String> callerEnv) throws IOException { 1228 // Replace environment variables, case-insensitive on Windows 1229 @SuppressWarnings("unchecked") 1230 Map<String, String> env = Shell.WINDOWS ? new CaseInsensitiveMap(callerEnv) : 1231 callerEnv; 1232 String[] classPathEntries = inputClassPath.split(File.pathSeparator); 1233 for (int i = 0; i < classPathEntries.length; ++i) { 1234 classPathEntries[i] = StringUtils.replaceTokens(classPathEntries[i], 1235 StringUtils.ENV_VAR_PATTERN, env); 1236 } 1237 File workingDir = new File(pwd.toString()); 1238 if (!workingDir.mkdirs()) { 1239 // If mkdirs returns false because the working directory already exists, 1240 // then this is acceptable. If it returns false due to some other I/O 1241 // error, then this method will fail later with an IOException while saving 1242 // the jar. 1243 LOG.debug("mkdirs false for " + workingDir + ", execution will continue"); 1244 } 1245 1246 StringBuilder unexpandedWildcardClasspath = new StringBuilder(); 1247 // Append all entries 1248 List<String> classPathEntryList = new ArrayList<String>( 1249 classPathEntries.length); 1250 for (String classPathEntry: classPathEntries) { 1251 if (classPathEntry.length() == 0) { 1252 continue; 1253 } 1254 if (classPathEntry.endsWith("*")) { 1255 boolean foundWildCardJar = false; 1256 // Append all jars that match the wildcard 1257 Path globPath = new Path(classPathEntry).suffix("{.jar,.JAR}"); 1258 FileStatus[] wildcardJars = FileContext.getLocalFSFileContext().util() 1259 .globStatus(globPath); 1260 if (wildcardJars != null) { 1261 for (FileStatus wildcardJar: wildcardJars) { 1262 foundWildCardJar = true; 1263 classPathEntryList.add(wildcardJar.getPath().toUri().toURL() 1264 .toExternalForm()); 1265 } 1266 } 1267 if (!foundWildCardJar) { 1268 unexpandedWildcardClasspath.append(File.pathSeparator); 1269 unexpandedWildcardClasspath.append(classPathEntry); 1270 } 1271 } else { 1272 // Append just this entry 1273 File fileCpEntry = null; 1274 if(!new Path(classPathEntry).isAbsolute()) { 1275 fileCpEntry = new File(targetDir.toString(), classPathEntry); 1276 } 1277 else { 1278 fileCpEntry = new File(classPathEntry); 1279 } 1280 String classPathEntryUrl = fileCpEntry.toURI().toURL() 1281 .toExternalForm(); 1282 1283 // File.toURI only appends trailing '/' if it can determine that it is a 1284 // directory that already exists. (See JavaDocs.) If this entry had a 1285 // trailing '/' specified by the caller, then guarantee that the 1286 // classpath entry in the manifest has a trailing '/', and thus refers to 1287 // a directory instead of a file. This can happen if the caller is 1288 // creating a classpath jar referencing a directory that hasn't been 1289 // created yet, but will definitely be created before running. 1290 if (classPathEntry.endsWith(Path.SEPARATOR) && 1291 !classPathEntryUrl.endsWith(Path.SEPARATOR)) { 1292 classPathEntryUrl = classPathEntryUrl + Path.SEPARATOR; 1293 } 1294 classPathEntryList.add(classPathEntryUrl); 1295 } 1296 } 1297 String jarClassPath = StringUtils.join(" ", classPathEntryList); 1298 1299 // Create the manifest 1300 Manifest jarManifest = new Manifest(); 1301 jarManifest.getMainAttributes().putValue( 1302 Attributes.Name.MANIFEST_VERSION.toString(), "1.0"); 1303 jarManifest.getMainAttributes().putValue( 1304 Attributes.Name.CLASS_PATH.toString(), jarClassPath); 1305 1306 // Write the manifest to output JAR file 1307 File classPathJar = File.createTempFile("classpath-", ".jar", workingDir); 1308 FileOutputStream fos = null; 1309 BufferedOutputStream bos = null; 1310 JarOutputStream jos = null; 1311 try { 1312 fos = new FileOutputStream(classPathJar); 1313 bos = new BufferedOutputStream(fos); 1314 jos = new JarOutputStream(bos, jarManifest); 1315 } finally { 1316 IOUtils.cleanup(LOG, jos, bos, fos); 1317 } 1318 String[] jarCp = {classPathJar.getCanonicalPath(), 1319 unexpandedWildcardClasspath.toString()}; 1320 return jarCp; 1321 } 1322}