001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.hadoop.fs; 020 021import java.io.*; 022import java.util.ArrayList; 023import java.util.Arrays; 024import java.util.Enumeration; 025import java.util.List; 026import java.util.Map; 027import java.util.jar.Attributes; 028import java.util.jar.JarOutputStream; 029import java.util.jar.Manifest; 030import java.util.zip.GZIPInputStream; 031import java.util.zip.ZipEntry; 032import java.util.zip.ZipFile; 033 034import org.apache.commons.collections.map.CaseInsensitiveMap; 035import org.apache.commons.compress.archivers.tar.TarArchiveEntry; 036import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; 037import org.apache.hadoop.classification.InterfaceAudience; 038import org.apache.hadoop.classification.InterfaceStability; 039import org.apache.hadoop.conf.Configuration; 040import org.apache.hadoop.fs.permission.FsAction; 041import org.apache.hadoop.fs.permission.FsPermission; 042import org.apache.hadoop.io.IOUtils; 043import org.apache.hadoop.io.nativeio.NativeIO; 044import org.apache.hadoop.util.StringUtils; 045import org.apache.hadoop.util.Shell; 046import org.apache.hadoop.util.Shell.ShellCommandExecutor; 047import org.apache.commons.logging.Log; 048import org.apache.commons.logging.LogFactory; 049 050/** 051 * A collection of file-processing util methods 052 */ 053@InterfaceAudience.Public 054@InterfaceStability.Evolving 055public class FileUtil { 056 057 private static final Log LOG = LogFactory.getLog(FileUtil.class); 058 059 /* The error code is defined in winutils to indicate insufficient 060 * privilege to create symbolic links. This value need to keep in 061 * sync with the constant of the same name in: 062 * "src\winutils\common.h" 063 * */ 064 public static final int SYMLINK_NO_PRIVILEGE = 2; 065 066 /** 067 * convert an array of FileStatus to an array of Path 068 * 069 * @param stats 070 * an array of FileStatus objects 071 * @return an array of paths corresponding to the input 072 */ 073 public static Path[] stat2Paths(FileStatus[] stats) { 074 if (stats == null) 075 return null; 076 Path[] ret = new Path[stats.length]; 077 for (int i = 0; i < stats.length; ++i) { 078 ret[i] = stats[i].getPath(); 079 } 080 return ret; 081 } 082 083 /** 084 * convert an array of FileStatus to an array of Path. 085 * If stats if null, return path 086 * @param stats 087 * an array of FileStatus objects 088 * @param path 089 * default path to return in stats is null 090 * @return an array of paths corresponding to the input 091 */ 092 public static Path[] stat2Paths(FileStatus[] stats, Path path) { 093 if (stats == null) 094 return new Path[]{path}; 095 else 096 return stat2Paths(stats); 097 } 098 099 /** 100 * Delete a directory and all its contents. If 101 * we return false, the directory may be partially-deleted. 102 * (1) If dir is symlink to a file, the symlink is deleted. The file pointed 103 * to by the symlink is not deleted. 104 * (2) If dir is symlink to a directory, symlink is deleted. The directory 105 * pointed to by symlink is not deleted. 106 * (3) If dir is a normal file, it is deleted. 107 * (4) If dir is a normal directory, then dir and all its contents recursively 108 * are deleted. 109 */ 110 public static boolean fullyDelete(final File dir) { 111 return fullyDelete(dir, false); 112 } 113 114 /** 115 * Delete a directory and all its contents. If 116 * we return false, the directory may be partially-deleted. 117 * (1) If dir is symlink to a file, the symlink is deleted. The file pointed 118 * to by the symlink is not deleted. 119 * (2) If dir is symlink to a directory, symlink is deleted. The directory 120 * pointed to by symlink is not deleted. 121 * (3) If dir is a normal file, it is deleted. 122 * (4) If dir is a normal directory, then dir and all its contents recursively 123 * are deleted. 124 * @param dir the file or directory to be deleted 125 * @param tryGrantPermissions true if permissions should be modified to delete a file. 126 * @return true on success false on failure. 127 */ 128 public static boolean fullyDelete(final File dir, boolean tryGrantPermissions) { 129 if (tryGrantPermissions) { 130 // try to chmod +rwx the parent folder of the 'dir': 131 File parent = dir.getParentFile(); 132 grantPermissions(parent); 133 } 134 if (deleteImpl(dir, false)) { 135 // dir is (a) normal file, (b) symlink to a file, (c) empty directory or 136 // (d) symlink to a directory 137 return true; 138 } 139 // handle nonempty directory deletion 140 if (!fullyDeleteContents(dir, tryGrantPermissions)) { 141 return false; 142 } 143 return deleteImpl(dir, true); 144 } 145 146 /** 147 * Returns the target of the given symlink. Returns the empty string if 148 * the given path does not refer to a symlink or there is an error 149 * accessing the symlink. 150 * @param f File representing the symbolic link. 151 * @return The target of the symbolic link, empty string on error or if not 152 * a symlink. 153 */ 154 public static String readLink(File f) { 155 /* NB: Use readSymbolicLink in java.nio.file.Path once available. Could 156 * use getCanonicalPath in File to get the target of the symlink but that 157 * does not indicate if the given path refers to a symlink. 158 */ 159 try { 160 return Shell.execCommand( 161 Shell.getReadlinkCommand(f.toString())).trim(); 162 } catch (IOException x) { 163 return ""; 164 } 165 } 166 167 /* 168 * Pure-Java implementation of "chmod +rwx f". 169 */ 170 private static void grantPermissions(final File f) { 171 FileUtil.setExecutable(f, true); 172 FileUtil.setReadable(f, true); 173 FileUtil.setWritable(f, true); 174 } 175 176 private static boolean deleteImpl(final File f, final boolean doLog) { 177 if (f == null) { 178 LOG.warn("null file argument."); 179 return false; 180 } 181 final boolean wasDeleted = f.delete(); 182 if (wasDeleted) { 183 return true; 184 } 185 final boolean ex = f.exists(); 186 if (doLog && ex) { 187 LOG.warn("Failed to delete file or dir [" 188 + f.getAbsolutePath() + "]: it still exists."); 189 } 190 return !ex; 191 } 192 193 /** 194 * Delete the contents of a directory, not the directory itself. If 195 * we return false, the directory may be partially-deleted. 196 * If dir is a symlink to a directory, all the contents of the actual 197 * directory pointed to by dir will be deleted. 198 */ 199 public static boolean fullyDeleteContents(final File dir) { 200 return fullyDeleteContents(dir, false); 201 } 202 203 /** 204 * Delete the contents of a directory, not the directory itself. If 205 * we return false, the directory may be partially-deleted. 206 * If dir is a symlink to a directory, all the contents of the actual 207 * directory pointed to by dir will be deleted. 208 * @param tryGrantPermissions if 'true', try grant +rwx permissions to this 209 * and all the underlying directories before trying to delete their contents. 210 */ 211 public static boolean fullyDeleteContents(final File dir, final boolean tryGrantPermissions) { 212 if (tryGrantPermissions) { 213 // to be able to list the dir and delete files from it 214 // we must grant the dir rwx permissions: 215 grantPermissions(dir); 216 } 217 boolean deletionSucceeded = true; 218 final File[] contents = dir.listFiles(); 219 if (contents != null) { 220 for (int i = 0; i < contents.length; i++) { 221 if (contents[i].isFile()) { 222 if (!deleteImpl(contents[i], true)) {// normal file or symlink to another file 223 deletionSucceeded = false; 224 continue; // continue deletion of other files/dirs under dir 225 } 226 } else { 227 // Either directory or symlink to another directory. 228 // Try deleting the directory as this might be a symlink 229 boolean b = false; 230 b = deleteImpl(contents[i], false); 231 if (b){ 232 //this was indeed a symlink or an empty directory 233 continue; 234 } 235 // if not an empty directory or symlink let 236 // fullydelete handle it. 237 if (!fullyDelete(contents[i], tryGrantPermissions)) { 238 deletionSucceeded = false; 239 // continue deletion of other files/dirs under dir 240 } 241 } 242 } 243 } 244 return deletionSucceeded; 245 } 246 247 /** 248 * Recursively delete a directory. 249 * 250 * @param fs {@link FileSystem} on which the path is present 251 * @param dir directory to recursively delete 252 * @throws IOException 253 * @deprecated Use {@link FileSystem#delete(Path, boolean)} 254 */ 255 @Deprecated 256 public static void fullyDelete(FileSystem fs, Path dir) 257 throws IOException { 258 fs.delete(dir, true); 259 } 260 261 // 262 // If the destination is a subdirectory of the source, then 263 // generate exception 264 // 265 private static void checkDependencies(FileSystem srcFS, 266 Path src, 267 FileSystem dstFS, 268 Path dst) 269 throws IOException { 270 if (srcFS == dstFS) { 271 String srcq = src.makeQualified(srcFS).toString() + Path.SEPARATOR; 272 String dstq = dst.makeQualified(dstFS).toString() + Path.SEPARATOR; 273 if (dstq.startsWith(srcq)) { 274 if (srcq.length() == dstq.length()) { 275 throw new IOException("Cannot copy " + src + " to itself."); 276 } else { 277 throw new IOException("Cannot copy " + src + " to its subdirectory " + 278 dst); 279 } 280 } 281 } 282 } 283 284 /** Copy files between FileSystems. */ 285 public static boolean copy(FileSystem srcFS, Path src, 286 FileSystem dstFS, Path dst, 287 boolean deleteSource, 288 Configuration conf) throws IOException { 289 return copy(srcFS, src, dstFS, dst, deleteSource, true, conf); 290 } 291 292 public static boolean copy(FileSystem srcFS, Path[] srcs, 293 FileSystem dstFS, Path dst, 294 boolean deleteSource, 295 boolean overwrite, Configuration conf) 296 throws IOException { 297 boolean gotException = false; 298 boolean returnVal = true; 299 StringBuilder exceptions = new StringBuilder(); 300 301 if (srcs.length == 1) 302 return copy(srcFS, srcs[0], dstFS, dst, deleteSource, overwrite, conf); 303 304 // Check if dest is directory 305 if (!dstFS.exists(dst)) { 306 throw new IOException("`" + dst +"': specified destination directory " + 307 "does not exist"); 308 } else { 309 FileStatus sdst = dstFS.getFileStatus(dst); 310 if (!sdst.isDirectory()) 311 throw new IOException("copying multiple files, but last argument `" + 312 dst + "' is not a directory"); 313 } 314 315 for (Path src : srcs) { 316 try { 317 if (!copy(srcFS, src, dstFS, dst, deleteSource, overwrite, conf)) 318 returnVal = false; 319 } catch (IOException e) { 320 gotException = true; 321 exceptions.append(e.getMessage()); 322 exceptions.append("\n"); 323 } 324 } 325 if (gotException) { 326 throw new IOException(exceptions.toString()); 327 } 328 return returnVal; 329 } 330 331 /** Copy files between FileSystems. */ 332 public static boolean copy(FileSystem srcFS, Path src, 333 FileSystem dstFS, Path dst, 334 boolean deleteSource, 335 boolean overwrite, 336 Configuration conf) throws IOException { 337 FileStatus fileStatus = srcFS.getFileStatus(src); 338 return copy(srcFS, fileStatus, dstFS, dst, deleteSource, overwrite, conf); 339 } 340 341 /** Copy files between FileSystems. */ 342 public static boolean copy(FileSystem srcFS, FileStatus srcStatus, 343 FileSystem dstFS, Path dst, 344 boolean deleteSource, 345 boolean overwrite, 346 Configuration conf) throws IOException { 347 Path src = srcStatus.getPath(); 348 dst = checkDest(src.getName(), dstFS, dst, overwrite); 349 if (srcStatus.isDirectory()) { 350 checkDependencies(srcFS, src, dstFS, dst); 351 if (!dstFS.mkdirs(dst)) { 352 return false; 353 } 354 FileStatus contents[] = srcFS.listStatus(src); 355 for (int i = 0; i < contents.length; i++) { 356 copy(srcFS, contents[i], dstFS, 357 new Path(dst, contents[i].getPath().getName()), 358 deleteSource, overwrite, conf); 359 } 360 } else { 361 InputStream in=null; 362 OutputStream out = null; 363 try { 364 in = srcFS.open(src); 365 out = dstFS.create(dst, overwrite); 366 IOUtils.copyBytes(in, out, conf, true); 367 } catch (IOException e) { 368 IOUtils.closeStream(out); 369 IOUtils.closeStream(in); 370 throw e; 371 } 372 } 373 if (deleteSource) { 374 return srcFS.delete(src, true); 375 } else { 376 return true; 377 } 378 379 } 380 381 /** Copy all files in a directory to one output file (merge). */ 382 public static boolean copyMerge(FileSystem srcFS, Path srcDir, 383 FileSystem dstFS, Path dstFile, 384 boolean deleteSource, 385 Configuration conf, String addString) throws IOException { 386 dstFile = checkDest(srcDir.getName(), dstFS, dstFile, false); 387 388 if (!srcFS.getFileStatus(srcDir).isDirectory()) 389 return false; 390 391 OutputStream out = dstFS.create(dstFile); 392 393 try { 394 FileStatus contents[] = srcFS.listStatus(srcDir); 395 Arrays.sort(contents); 396 for (int i = 0; i < contents.length; i++) { 397 if (contents[i].isFile()) { 398 InputStream in = srcFS.open(contents[i].getPath()); 399 try { 400 IOUtils.copyBytes(in, out, conf, false); 401 if (addString!=null) 402 out.write(addString.getBytes("UTF-8")); 403 404 } finally { 405 in.close(); 406 } 407 } 408 } 409 } finally { 410 out.close(); 411 } 412 413 414 if (deleteSource) { 415 return srcFS.delete(srcDir, true); 416 } else { 417 return true; 418 } 419 } 420 421 /** Copy local files to a FileSystem. */ 422 public static boolean copy(File src, 423 FileSystem dstFS, Path dst, 424 boolean deleteSource, 425 Configuration conf) throws IOException { 426 dst = checkDest(src.getName(), dstFS, dst, false); 427 428 if (src.isDirectory()) { 429 if (!dstFS.mkdirs(dst)) { 430 return false; 431 } 432 File contents[] = listFiles(src); 433 for (int i = 0; i < contents.length; i++) { 434 copy(contents[i], dstFS, new Path(dst, contents[i].getName()), 435 deleteSource, conf); 436 } 437 } else if (src.isFile()) { 438 InputStream in = null; 439 OutputStream out =null; 440 try { 441 in = new FileInputStream(src); 442 out = dstFS.create(dst); 443 IOUtils.copyBytes(in, out, conf); 444 } catch (IOException e) { 445 IOUtils.closeStream( out ); 446 IOUtils.closeStream( in ); 447 throw e; 448 } 449 } else { 450 throw new IOException(src.toString() + 451 ": No such file or directory"); 452 } 453 if (deleteSource) { 454 return FileUtil.fullyDelete(src); 455 } else { 456 return true; 457 } 458 } 459 460 /** Copy FileSystem files to local files. */ 461 public static boolean copy(FileSystem srcFS, Path src, 462 File dst, boolean deleteSource, 463 Configuration conf) throws IOException { 464 FileStatus filestatus = srcFS.getFileStatus(src); 465 return copy(srcFS, filestatus, dst, deleteSource, conf); 466 } 467 468 /** Copy FileSystem files to local files. */ 469 private static boolean copy(FileSystem srcFS, FileStatus srcStatus, 470 File dst, boolean deleteSource, 471 Configuration conf) throws IOException { 472 Path src = srcStatus.getPath(); 473 if (srcStatus.isDirectory()) { 474 if (!dst.mkdirs()) { 475 return false; 476 } 477 FileStatus contents[] = srcFS.listStatus(src); 478 for (int i = 0; i < contents.length; i++) { 479 copy(srcFS, contents[i], 480 new File(dst, contents[i].getPath().getName()), 481 deleteSource, conf); 482 } 483 } else { 484 InputStream in = srcFS.open(src); 485 IOUtils.copyBytes(in, new FileOutputStream(dst), conf); 486 } 487 if (deleteSource) { 488 return srcFS.delete(src, true); 489 } else { 490 return true; 491 } 492 } 493 494 private static Path checkDest(String srcName, FileSystem dstFS, Path dst, 495 boolean overwrite) throws IOException { 496 if (dstFS.exists(dst)) { 497 FileStatus sdst = dstFS.getFileStatus(dst); 498 if (sdst.isDirectory()) { 499 if (null == srcName) { 500 throw new IOException("Target " + dst + " is a directory"); 501 } 502 return checkDest(null, dstFS, new Path(dst, srcName), overwrite); 503 } else if (!overwrite) { 504 throw new IOException("Target " + dst + " already exists"); 505 } 506 } 507 return dst; 508 } 509 510 /** 511 * Convert a os-native filename to a path that works for the shell. 512 * @param filename The filename to convert 513 * @return The unix pathname 514 * @throws IOException on windows, there can be problems with the subprocess 515 */ 516 public static String makeShellPath(String filename) throws IOException { 517 return filename; 518 } 519 520 /** 521 * Convert a os-native filename to a path that works for the shell. 522 * @param file The filename to convert 523 * @return The unix pathname 524 * @throws IOException on windows, there can be problems with the subprocess 525 */ 526 public static String makeShellPath(File file) throws IOException { 527 return makeShellPath(file, false); 528 } 529 530 /** 531 * Convert a os-native filename to a path that works for the shell. 532 * @param file The filename to convert 533 * @param makeCanonicalPath 534 * Whether to make canonical path for the file passed 535 * @return The unix pathname 536 * @throws IOException on windows, there can be problems with the subprocess 537 */ 538 public static String makeShellPath(File file, boolean makeCanonicalPath) 539 throws IOException { 540 if (makeCanonicalPath) { 541 return makeShellPath(file.getCanonicalPath()); 542 } else { 543 return makeShellPath(file.toString()); 544 } 545 } 546 547 /** 548 * Takes an input dir and returns the du on that local directory. Very basic 549 * implementation. 550 * 551 * @param dir 552 * The input dir to get the disk space of this local dir 553 * @return The total disk space of the input local directory 554 */ 555 public static long getDU(File dir) { 556 long size = 0; 557 if (!dir.exists()) 558 return 0; 559 if (!dir.isDirectory()) { 560 return dir.length(); 561 } else { 562 File[] allFiles = dir.listFiles(); 563 if(allFiles != null) { 564 for (int i = 0; i < allFiles.length; i++) { 565 boolean isSymLink; 566 try { 567 isSymLink = org.apache.commons.io.FileUtils.isSymlink(allFiles[i]); 568 } catch(IOException ioe) { 569 isSymLink = true; 570 } 571 if(!isSymLink) { 572 size += getDU(allFiles[i]); 573 } 574 } 575 } 576 return size; 577 } 578 } 579 580 /** 581 * Given a File input it will unzip the file in a the unzip directory 582 * passed as the second parameter 583 * @param inFile The zip file as input 584 * @param unzipDir The unzip directory where to unzip the zip file. 585 * @throws IOException 586 */ 587 public static void unZip(File inFile, File unzipDir) throws IOException { 588 Enumeration<? extends ZipEntry> entries; 589 ZipFile zipFile = new ZipFile(inFile); 590 String targetDirPath = unzipDir.getCanonicalPath() + File.separator; 591 592 try { 593 entries = zipFile.entries(); 594 while (entries.hasMoreElements()) { 595 ZipEntry entry = entries.nextElement(); 596 if (!entry.isDirectory()) { 597 File file = new File(unzipDir, entry.getName()); 598 if (!file.getCanonicalPath().startsWith(targetDirPath)) { 599 throw new IOException("expanding " + entry.getName() 600 + " would create file outside of " + unzipDir); 601 } 602 InputStream in = zipFile.getInputStream(entry); 603 try { 604 if (!file.getParentFile().mkdirs()) { 605 if (!file.getParentFile().isDirectory()) { 606 throw new IOException("Mkdirs failed to create " + 607 file.getParentFile().toString()); 608 } 609 } 610 OutputStream out = new FileOutputStream(file); 611 try { 612 byte[] buffer = new byte[8192]; 613 int i; 614 while ((i = in.read(buffer)) != -1) { 615 out.write(buffer, 0, i); 616 } 617 } finally { 618 out.close(); 619 } 620 } finally { 621 in.close(); 622 } 623 } 624 } 625 } finally { 626 zipFile.close(); 627 } 628 } 629 630 /** 631 * Given a Tar File as input it will untar the file in a the untar directory 632 * passed as the second parameter 633 * 634 * This utility will untar ".tar" files and ".tar.gz","tgz" files. 635 * 636 * @param inFile The tar file as input. 637 * @param untarDir The untar directory where to untar the tar file. 638 * @throws IOException 639 */ 640 public static void unTar(File inFile, File untarDir) throws IOException { 641 if (!untarDir.mkdirs()) { 642 if (!untarDir.isDirectory()) { 643 throw new IOException("Mkdirs failed to create " + untarDir); 644 } 645 } 646 647 boolean gzipped = inFile.toString().endsWith("gz"); 648 if(Shell.WINDOWS) { 649 // Tar is not native to Windows. Use simple Java based implementation for 650 // tests and simple tar archives 651 unTarUsingJava(inFile, untarDir, gzipped); 652 } 653 else { 654 // spawn tar utility to untar archive for full fledged unix behavior such 655 // as resolving symlinks in tar archives 656 unTarUsingTar(inFile, untarDir, gzipped); 657 } 658 } 659 660 private static void unTarUsingTar(File inFile, File untarDir, 661 boolean gzipped) throws IOException { 662 StringBuffer untarCommand = new StringBuffer(); 663 if (gzipped) { 664 untarCommand.append(" gzip -dc '"); 665 untarCommand.append(FileUtil.makeShellPath(inFile)); 666 untarCommand.append("' | ("); 667 } 668 untarCommand.append("cd '"); 669 untarCommand.append(FileUtil.makeShellPath(untarDir)); 670 untarCommand.append("' ; "); 671 untarCommand.append("tar -xf "); 672 673 if (gzipped) { 674 untarCommand.append(" -)"); 675 } else { 676 untarCommand.append(FileUtil.makeShellPath(inFile)); 677 } 678 String[] shellCmd = { "bash", "-c", untarCommand.toString() }; 679 ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd); 680 shexec.execute(); 681 int exitcode = shexec.getExitCode(); 682 if (exitcode != 0) { 683 throw new IOException("Error untarring file " + inFile + 684 ". Tar process exited with exit code " + exitcode); 685 } 686 } 687 688 private static void unTarUsingJava(File inFile, File untarDir, 689 boolean gzipped) throws IOException { 690 InputStream inputStream = null; 691 TarArchiveInputStream tis = null; 692 try { 693 if (gzipped) { 694 inputStream = new BufferedInputStream(new GZIPInputStream( 695 new FileInputStream(inFile))); 696 } else { 697 inputStream = new BufferedInputStream(new FileInputStream(inFile)); 698 } 699 700 tis = new TarArchiveInputStream(inputStream); 701 702 for (TarArchiveEntry entry = tis.getNextTarEntry(); entry != null;) { 703 unpackEntries(tis, entry, untarDir); 704 entry = tis.getNextTarEntry(); 705 } 706 } finally { 707 IOUtils.cleanup(LOG, tis, inputStream); 708 } 709 } 710 711 private static void unpackEntries(TarArchiveInputStream tis, 712 TarArchiveEntry entry, File outputDir) throws IOException { 713 String targetDirPath = outputDir.getCanonicalPath() + File.separator; 714 File outputFile = new File(outputDir, entry.getName()); 715 if (!outputFile.getCanonicalPath().startsWith(targetDirPath)) { 716 throw new IOException("expanding " + entry.getName() 717 + " would create entry outside of " + outputDir); 718 } 719 720 if (entry.isDirectory()) { 721 File subDir = new File(outputDir, entry.getName()); 722 if (!subDir.mkdirs() && !subDir.isDirectory()) { 723 throw new IOException("Mkdirs failed to create tar internal dir " 724 + outputDir); 725 } 726 727 for (TarArchiveEntry e : entry.getDirectoryEntries()) { 728 unpackEntries(tis, e, subDir); 729 } 730 731 return; 732 } 733 734 if (!outputFile.getParentFile().exists()) { 735 if (!outputFile.getParentFile().mkdirs()) { 736 throw new IOException("Mkdirs failed to create tar internal dir " 737 + outputDir); 738 } 739 } 740 741 int count; 742 byte data[] = new byte[2048]; 743 try (BufferedOutputStream outputStream = new BufferedOutputStream( 744 new FileOutputStream(outputFile));) { 745 746 while ((count = tis.read(data)) != -1) { 747 outputStream.write(data, 0, count); 748 } 749 750 outputStream.flush(); 751 } 752 } 753 754 /** 755 * Class for creating hardlinks. 756 * Supports Unix, WindXP. 757 * @deprecated Use {@link org.apache.hadoop.fs.HardLink} 758 */ 759 @Deprecated 760 public static class HardLink extends org.apache.hadoop.fs.HardLink { 761 // This is a stub to assist with coordinated change between 762 // COMMON and HDFS projects. It will be removed after the 763 // corresponding change is committed to HDFS. 764 } 765 766 /** 767 * Create a soft link between a src and destination 768 * only on a local disk. HDFS does not support this. 769 * On Windows, when symlink creation fails due to security 770 * setting, we will log a warning. The return code in this 771 * case is 2. 772 * 773 * @param target the target for symlink 774 * @param linkname the symlink 775 * @return 0 on success 776 */ 777 public static int symLink(String target, String linkname) throws IOException{ 778 // Run the input paths through Java's File so that they are converted to the 779 // native OS form 780 File targetFile = new File( 781 Path.getPathWithoutSchemeAndAuthority(new Path(target)).toString()); 782 File linkFile = new File( 783 Path.getPathWithoutSchemeAndAuthority(new Path(linkname)).toString()); 784 785 // If not on Java7+, copy a file instead of creating a symlink since 786 // Java6 has close to no support for symlinks on Windows. Specifically 787 // File#length and File#renameTo do not work as expected. 788 // (see HADOOP-9061 for additional details) 789 // We still create symlinks for directories, since the scenario in this 790 // case is different. The directory content could change in which 791 // case the symlink loses its purpose (for example task attempt log folder 792 // is symlinked under userlogs and userlogs are generated afterwards). 793 if (Shell.WINDOWS && !Shell.isJava7OrAbove() && targetFile.isFile()) { 794 try { 795 LOG.warn("FileUtil#symlink: On Windows+Java6, copying file instead " + 796 "of creating a symlink. Copying " + target + " -> " + linkname); 797 798 if (!linkFile.getParentFile().exists()) { 799 LOG.warn("Parent directory " + linkFile.getParent() + 800 " does not exist."); 801 return 1; 802 } else { 803 org.apache.commons.io.FileUtils.copyFile(targetFile, linkFile); 804 } 805 } catch (IOException ex) { 806 LOG.warn("FileUtil#symlink failed to copy the file with error: " 807 + ex.getMessage()); 808 // Exit with non-zero exit code 809 return 1; 810 } 811 return 0; 812 } 813 814 String[] cmd = Shell.getSymlinkCommand( 815 targetFile.toString(), 816 linkFile.toString()); 817 818 ShellCommandExecutor shExec; 819 try { 820 if (Shell.WINDOWS && 821 linkFile.getParentFile() != null && 822 !new Path(target).isAbsolute()) { 823 // Relative links on Windows must be resolvable at the time of 824 // creation. To ensure this we run the shell command in the directory 825 // of the link. 826 // 827 shExec = new ShellCommandExecutor(cmd, linkFile.getParentFile()); 828 } else { 829 shExec = new ShellCommandExecutor(cmd); 830 } 831 shExec.execute(); 832 } catch (Shell.ExitCodeException ec) { 833 int returnVal = ec.getExitCode(); 834 if (Shell.WINDOWS && returnVal == SYMLINK_NO_PRIVILEGE) { 835 LOG.warn("Fail to create symbolic links on Windows. " 836 + "The default security settings in Windows disallow non-elevated " 837 + "administrators and all non-administrators from creating symbolic links. " 838 + "This behavior can be changed in the Local Security Policy management console"); 839 } else if (returnVal != 0) { 840 LOG.warn("Command '" + StringUtils.join(" ", cmd) + "' failed " 841 + returnVal + " with: " + ec.getMessage()); 842 } 843 return returnVal; 844 } catch (IOException e) { 845 if (LOG.isDebugEnabled()) { 846 LOG.debug("Error while create symlink " + linkname + " to " + target 847 + "." + " Exception: " + StringUtils.stringifyException(e)); 848 } 849 throw e; 850 } 851 return shExec.getExitCode(); 852 } 853 854 /** 855 * Change the permissions on a filename. 856 * @param filename the name of the file to change 857 * @param perm the permission string 858 * @return the exit code from the command 859 * @throws IOException 860 * @throws InterruptedException 861 */ 862 public static int chmod(String filename, String perm 863 ) throws IOException, InterruptedException { 864 return chmod(filename, perm, false); 865 } 866 867 /** 868 * Change the permissions on a file / directory, recursively, if 869 * needed. 870 * @param filename name of the file whose permissions are to change 871 * @param perm permission string 872 * @param recursive true, if permissions should be changed recursively 873 * @return the exit code from the command. 874 * @throws IOException 875 */ 876 public static int chmod(String filename, String perm, boolean recursive) 877 throws IOException { 878 String [] cmd = Shell.getSetPermissionCommand(perm, recursive); 879 String[] args = new String[cmd.length + 1]; 880 System.arraycopy(cmd, 0, args, 0, cmd.length); 881 args[cmd.length] = new File(filename).getPath(); 882 ShellCommandExecutor shExec = new ShellCommandExecutor(args); 883 try { 884 shExec.execute(); 885 }catch(IOException e) { 886 if(LOG.isDebugEnabled()) { 887 LOG.debug("Error while changing permission : " + filename 888 +" Exception: " + StringUtils.stringifyException(e)); 889 } 890 } 891 return shExec.getExitCode(); 892 } 893 894 /** 895 * Set the ownership on a file / directory. User name and group name 896 * cannot both be null. 897 * @param file the file to change 898 * @param username the new user owner name 899 * @param groupname the new group owner name 900 * @throws IOException 901 */ 902 public static void setOwner(File file, String username, 903 String groupname) throws IOException { 904 if (username == null && groupname == null) { 905 throw new IOException("username == null && groupname == null"); 906 } 907 String arg = (username == null ? "" : username) 908 + (groupname == null ? "" : ":" + groupname); 909 String [] cmd = Shell.getSetOwnerCommand(arg); 910 execCommand(file, cmd); 911 } 912 913 /** 914 * Platform independent implementation for {@link File#setReadable(boolean)} 915 * File#setReadable does not work as expected on Windows. 916 * @param f input file 917 * @param readable 918 * @return true on success, false otherwise 919 */ 920 public static boolean setReadable(File f, boolean readable) { 921 if (Shell.WINDOWS) { 922 try { 923 String permission = readable ? "u+r" : "u-r"; 924 FileUtil.chmod(f.getCanonicalPath(), permission, false); 925 return true; 926 } catch (IOException ex) { 927 return false; 928 } 929 } else { 930 return f.setReadable(readable); 931 } 932 } 933 934 /** 935 * Platform independent implementation for {@link File#setWritable(boolean)} 936 * File#setWritable does not work as expected on Windows. 937 * @param f input file 938 * @param writable 939 * @return true on success, false otherwise 940 */ 941 public static boolean setWritable(File f, boolean writable) { 942 if (Shell.WINDOWS) { 943 try { 944 String permission = writable ? "u+w" : "u-w"; 945 FileUtil.chmod(f.getCanonicalPath(), permission, false); 946 return true; 947 } catch (IOException ex) { 948 return false; 949 } 950 } else { 951 return f.setWritable(writable); 952 } 953 } 954 955 /** 956 * Platform independent implementation for {@link File#setExecutable(boolean)} 957 * File#setExecutable does not work as expected on Windows. 958 * Note: revoking execute permission on folders does not have the same 959 * behavior on Windows as on Unix platforms. Creating, deleting or renaming 960 * a file within that folder will still succeed on Windows. 961 * @param f input file 962 * @param executable 963 * @return true on success, false otherwise 964 */ 965 public static boolean setExecutable(File f, boolean executable) { 966 if (Shell.WINDOWS) { 967 try { 968 String permission = executable ? "u+x" : "u-x"; 969 FileUtil.chmod(f.getCanonicalPath(), permission, false); 970 return true; 971 } catch (IOException ex) { 972 return false; 973 } 974 } else { 975 return f.setExecutable(executable); 976 } 977 } 978 979 /** 980 * Platform independent implementation for {@link File#canRead()} 981 * @param f input file 982 * @return On Unix, same as {@link File#canRead()} 983 * On Windows, true if process has read access on the path 984 */ 985 public static boolean canRead(File f) { 986 if (Shell.WINDOWS) { 987 try { 988 return NativeIO.Windows.access(f.getCanonicalPath(), 989 NativeIO.Windows.AccessRight.ACCESS_READ); 990 } catch (IOException e) { 991 return false; 992 } 993 } else { 994 return f.canRead(); 995 } 996 } 997 998 /** 999 * Platform independent implementation for {@link File#canWrite()} 1000 * @param f input file 1001 * @return On Unix, same as {@link File#canWrite()} 1002 * On Windows, true if process has write access on the path 1003 */ 1004 public static boolean canWrite(File f) { 1005 if (Shell.WINDOWS) { 1006 try { 1007 return NativeIO.Windows.access(f.getCanonicalPath(), 1008 NativeIO.Windows.AccessRight.ACCESS_WRITE); 1009 } catch (IOException e) { 1010 return false; 1011 } 1012 } else { 1013 return f.canWrite(); 1014 } 1015 } 1016 1017 /** 1018 * Platform independent implementation for {@link File#canExecute()} 1019 * @param f input file 1020 * @return On Unix, same as {@link File#canExecute()} 1021 * On Windows, true if process has execute access on the path 1022 */ 1023 public static boolean canExecute(File f) { 1024 if (Shell.WINDOWS) { 1025 try { 1026 return NativeIO.Windows.access(f.getCanonicalPath(), 1027 NativeIO.Windows.AccessRight.ACCESS_EXECUTE); 1028 } catch (IOException e) { 1029 return false; 1030 } 1031 } else { 1032 return f.canExecute(); 1033 } 1034 } 1035 1036 /** 1037 * Set permissions to the required value. Uses the java primitives instead 1038 * of forking if group == other. 1039 * @param f the file to change 1040 * @param permission the new permissions 1041 * @throws IOException 1042 */ 1043 public static void setPermission(File f, FsPermission permission 1044 ) throws IOException { 1045 FsAction user = permission.getUserAction(); 1046 FsAction group = permission.getGroupAction(); 1047 FsAction other = permission.getOtherAction(); 1048 1049 // use the native/fork if the group/other permissions are different 1050 // or if the native is available or on Windows 1051 if (group != other || NativeIO.isAvailable() || Shell.WINDOWS) { 1052 execSetPermission(f, permission); 1053 return; 1054 } 1055 1056 boolean rv = true; 1057 1058 // read perms 1059 rv = f.setReadable(group.implies(FsAction.READ), false); 1060 checkReturnValue(rv, f, permission); 1061 if (group.implies(FsAction.READ) != user.implies(FsAction.READ)) { 1062 rv = f.setReadable(user.implies(FsAction.READ), true); 1063 checkReturnValue(rv, f, permission); 1064 } 1065 1066 // write perms 1067 rv = f.setWritable(group.implies(FsAction.WRITE), false); 1068 checkReturnValue(rv, f, permission); 1069 if (group.implies(FsAction.WRITE) != user.implies(FsAction.WRITE)) { 1070 rv = f.setWritable(user.implies(FsAction.WRITE), true); 1071 checkReturnValue(rv, f, permission); 1072 } 1073 1074 // exec perms 1075 rv = f.setExecutable(group.implies(FsAction.EXECUTE), false); 1076 checkReturnValue(rv, f, permission); 1077 if (group.implies(FsAction.EXECUTE) != user.implies(FsAction.EXECUTE)) { 1078 rv = f.setExecutable(user.implies(FsAction.EXECUTE), true); 1079 checkReturnValue(rv, f, permission); 1080 } 1081 } 1082 1083 private static void checkReturnValue(boolean rv, File p, 1084 FsPermission permission 1085 ) throws IOException { 1086 if (!rv) { 1087 throw new IOException("Failed to set permissions of path: " + p + 1088 " to " + 1089 String.format("%04o", permission.toShort())); 1090 } 1091 } 1092 1093 private static void execSetPermission(File f, 1094 FsPermission permission 1095 ) throws IOException { 1096 if (NativeIO.isAvailable()) { 1097 NativeIO.POSIX.chmod(f.getCanonicalPath(), permission.toShort()); 1098 } else { 1099 execCommand(f, Shell.getSetPermissionCommand( 1100 String.format("%04o", permission.toShort()), false)); 1101 } 1102 } 1103 1104 static String execCommand(File f, String... cmd) throws IOException { 1105 String[] args = new String[cmd.length + 1]; 1106 System.arraycopy(cmd, 0, args, 0, cmd.length); 1107 args[cmd.length] = f.getCanonicalPath(); 1108 String output = Shell.execCommand(args); 1109 return output; 1110 } 1111 1112 /** 1113 * Create a tmp file for a base file. 1114 * @param basefile the base file of the tmp 1115 * @param prefix file name prefix of tmp 1116 * @param isDeleteOnExit if true, the tmp will be deleted when the VM exits 1117 * @return a newly created tmp file 1118 * @exception IOException If a tmp file cannot created 1119 * @see java.io.File#createTempFile(String, String, File) 1120 * @see java.io.File#deleteOnExit() 1121 */ 1122 public static final File createLocalTempFile(final File basefile, 1123 final String prefix, 1124 final boolean isDeleteOnExit) 1125 throws IOException { 1126 File tmp = File.createTempFile(prefix + basefile.getName(), 1127 "", basefile.getParentFile()); 1128 if (isDeleteOnExit) { 1129 tmp.deleteOnExit(); 1130 } 1131 return tmp; 1132 } 1133 1134 /** 1135 * Move the src file to the name specified by target. 1136 * @param src the source file 1137 * @param target the target file 1138 * @exception IOException If this operation fails 1139 */ 1140 public static void replaceFile(File src, File target) throws IOException { 1141 /* renameTo() has two limitations on Windows platform. 1142 * src.renameTo(target) fails if 1143 * 1) If target already exists OR 1144 * 2) If target is already open for reading/writing. 1145 */ 1146 if (!src.renameTo(target)) { 1147 int retries = 5; 1148 while (target.exists() && !target.delete() && retries-- >= 0) { 1149 try { 1150 Thread.sleep(1000); 1151 } catch (InterruptedException e) { 1152 throw new IOException("replaceFile interrupted."); 1153 } 1154 } 1155 if (!src.renameTo(target)) { 1156 throw new IOException("Unable to rename " + src + 1157 " to " + target); 1158 } 1159 } 1160 } 1161 1162 /** 1163 * A wrapper for {@link File#listFiles()}. This java.io API returns null 1164 * when a dir is not a directory or for any I/O error. Instead of having 1165 * null check everywhere File#listFiles() is used, we will add utility API 1166 * to get around this problem. For the majority of cases where we prefer 1167 * an IOException to be thrown. 1168 * @param dir directory for which listing should be performed 1169 * @return list of files or empty list 1170 * @exception IOException for invalid directory or for a bad disk. 1171 */ 1172 public static File[] listFiles(File dir) throws IOException { 1173 File[] files = dir.listFiles(); 1174 if(files == null) { 1175 throw new IOException("Invalid directory or I/O error occurred for dir: " 1176 + dir.toString()); 1177 } 1178 return files; 1179 } 1180 1181 /** 1182 * A wrapper for {@link File#list()}. This java.io API returns null 1183 * when a dir is not a directory or for any I/O error. Instead of having 1184 * null check everywhere File#list() is used, we will add utility API 1185 * to get around this problem. For the majority of cases where we prefer 1186 * an IOException to be thrown. 1187 * @param dir directory for which listing should be performed 1188 * @return list of file names or empty string list 1189 * @exception IOException for invalid directory or for a bad disk. 1190 */ 1191 public static String[] list(File dir) throws IOException { 1192 String[] fileNames = dir.list(); 1193 if(fileNames == null) { 1194 throw new IOException("Invalid directory or I/O error occurred for dir: " 1195 + dir.toString()); 1196 } 1197 return fileNames; 1198 } 1199 1200 public static String[] createJarWithClassPath(String inputClassPath, Path pwd, 1201 Map<String, String> callerEnv) throws IOException { 1202 return createJarWithClassPath(inputClassPath, pwd, pwd, callerEnv); 1203 } 1204 1205 /** 1206 * Create a jar file at the given path, containing a manifest with a classpath 1207 * that references all specified entries. 1208 * 1209 * Some platforms may have an upper limit on command line length. For example, 1210 * the maximum command line length on Windows is 8191 characters, but the 1211 * length of the classpath may exceed this. To work around this limitation, 1212 * use this method to create a small intermediate jar with a manifest that 1213 * contains the full classpath. It returns the absolute path to the new jar, 1214 * which the caller may set as the classpath for a new process. 1215 * 1216 * Environment variable evaluation is not supported within a jar manifest, so 1217 * this method expands environment variables before inserting classpath entries 1218 * to the manifest. The method parses environment variables according to 1219 * platform-specific syntax (%VAR% on Windows, or $VAR otherwise). On Windows, 1220 * environment variables are case-insensitive. For example, %VAR% and %var% 1221 * evaluate to the same value. 1222 * 1223 * Specifying the classpath in a jar manifest does not support wildcards, so 1224 * this method expands wildcards internally. Any classpath entry that ends 1225 * with * is translated to all files at that path with extension .jar or .JAR. 1226 * 1227 * @param inputClassPath String input classpath to bundle into the jar manifest 1228 * @param pwd Path to working directory to save jar 1229 * @param targetDir path to where the jar execution will have its working dir 1230 * @param callerEnv Map<String, String> caller's environment variables to use 1231 * for expansion 1232 * @return String[] with absolute path to new jar in position 0 and 1233 * unexpanded wild card entry path in position 1 1234 * @throws IOException if there is an I/O error while writing the jar file 1235 */ 1236 public static String[] createJarWithClassPath(String inputClassPath, Path pwd, 1237 Path targetDir, 1238 Map<String, String> callerEnv) throws IOException { 1239 // Replace environment variables, case-insensitive on Windows 1240 @SuppressWarnings("unchecked") 1241 Map<String, String> env = Shell.WINDOWS ? new CaseInsensitiveMap(callerEnv) : 1242 callerEnv; 1243 String[] classPathEntries = inputClassPath.split(File.pathSeparator); 1244 for (int i = 0; i < classPathEntries.length; ++i) { 1245 classPathEntries[i] = StringUtils.replaceTokens(classPathEntries[i], 1246 StringUtils.ENV_VAR_PATTERN, env); 1247 } 1248 File workingDir = new File(pwd.toString()); 1249 if (!workingDir.mkdirs()) { 1250 // If mkdirs returns false because the working directory already exists, 1251 // then this is acceptable. If it returns false due to some other I/O 1252 // error, then this method will fail later with an IOException while saving 1253 // the jar. 1254 LOG.debug("mkdirs false for " + workingDir + ", execution will continue"); 1255 } 1256 1257 StringBuilder unexpandedWildcardClasspath = new StringBuilder(); 1258 // Append all entries 1259 List<String> classPathEntryList = new ArrayList<String>( 1260 classPathEntries.length); 1261 for (String classPathEntry: classPathEntries) { 1262 if (classPathEntry.length() == 0) { 1263 continue; 1264 } 1265 if (classPathEntry.endsWith("*")) { 1266 boolean foundWildCardJar = false; 1267 // Append all jars that match the wildcard 1268 Path globPath = new Path(classPathEntry).suffix("{.jar,.JAR}"); 1269 FileStatus[] wildcardJars = FileContext.getLocalFSFileContext().util() 1270 .globStatus(globPath); 1271 if (wildcardJars != null) { 1272 for (FileStatus wildcardJar: wildcardJars) { 1273 foundWildCardJar = true; 1274 classPathEntryList.add(wildcardJar.getPath().toUri().toURL() 1275 .toExternalForm()); 1276 } 1277 } 1278 if (!foundWildCardJar) { 1279 unexpandedWildcardClasspath.append(File.pathSeparator); 1280 unexpandedWildcardClasspath.append(classPathEntry); 1281 } 1282 } else { 1283 // Append just this entry 1284 File fileCpEntry = null; 1285 if(!new Path(classPathEntry).isAbsolute()) { 1286 fileCpEntry = new File(targetDir.toString(), classPathEntry); 1287 } 1288 else { 1289 fileCpEntry = new File(classPathEntry); 1290 } 1291 String classPathEntryUrl = fileCpEntry.toURI().toURL() 1292 .toExternalForm(); 1293 1294 // File.toURI only appends trailing '/' if it can determine that it is a 1295 // directory that already exists. (See JavaDocs.) If this entry had a 1296 // trailing '/' specified by the caller, then guarantee that the 1297 // classpath entry in the manifest has a trailing '/', and thus refers to 1298 // a directory instead of a file. This can happen if the caller is 1299 // creating a classpath jar referencing a directory that hasn't been 1300 // created yet, but will definitely be created before running. 1301 if (classPathEntry.endsWith(Path.SEPARATOR) && 1302 !classPathEntryUrl.endsWith(Path.SEPARATOR)) { 1303 classPathEntryUrl = classPathEntryUrl + Path.SEPARATOR; 1304 } 1305 classPathEntryList.add(classPathEntryUrl); 1306 } 1307 } 1308 String jarClassPath = StringUtils.join(" ", classPathEntryList); 1309 1310 // Create the manifest 1311 Manifest jarManifest = new Manifest(); 1312 jarManifest.getMainAttributes().putValue( 1313 Attributes.Name.MANIFEST_VERSION.toString(), "1.0"); 1314 jarManifest.getMainAttributes().putValue( 1315 Attributes.Name.CLASS_PATH.toString(), jarClassPath); 1316 1317 // Write the manifest to output JAR file 1318 File classPathJar = File.createTempFile("classpath-", ".jar", workingDir); 1319 FileOutputStream fos = null; 1320 BufferedOutputStream bos = null; 1321 JarOutputStream jos = null; 1322 try { 1323 fos = new FileOutputStream(classPathJar); 1324 bos = new BufferedOutputStream(fos); 1325 jos = new JarOutputStream(bos, jarManifest); 1326 } finally { 1327 IOUtils.cleanup(LOG, jos, bos, fos); 1328 } 1329 String[] jarCp = {classPathJar.getCanonicalPath(), 1330 unexpandedWildcardClasspath.toString()}; 1331 return jarCp; 1332 } 1333}