001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.fs;
020
021import java.io.*;
022import java.util.ArrayList;
023import java.util.Arrays;
024import java.util.Enumeration;
025import java.util.List;
026import java.util.Map;
027import java.util.jar.Attributes;
028import java.util.jar.JarOutputStream;
029import java.util.jar.Manifest;
030import java.util.zip.GZIPInputStream;
031import java.util.zip.ZipEntry;
032import java.util.zip.ZipFile;
033
034import org.apache.commons.collections.map.CaseInsensitiveMap;
035import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
036import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
037import org.apache.hadoop.classification.InterfaceAudience;
038import org.apache.hadoop.classification.InterfaceStability;
039import org.apache.hadoop.conf.Configuration;
040import org.apache.hadoop.fs.permission.FsAction;
041import org.apache.hadoop.fs.permission.FsPermission;
042import org.apache.hadoop.io.IOUtils;
043import org.apache.hadoop.io.nativeio.NativeIO;
044import org.apache.hadoop.util.StringUtils;
045import org.apache.hadoop.util.Shell;
046import org.apache.hadoop.util.Shell.ShellCommandExecutor;
047import org.apache.commons.logging.Log;
048import org.apache.commons.logging.LogFactory;
049
050/**
051 * A collection of file-processing util methods
052 */
053@InterfaceAudience.Public
054@InterfaceStability.Evolving
055public class FileUtil {
056
057  private static final Log LOG = LogFactory.getLog(FileUtil.class);
058
059  /* The error code is defined in winutils to indicate insufficient
060   * privilege to create symbolic links. This value need to keep in
061   * sync with the constant of the same name in:
062   * "src\winutils\common.h"
063   * */
064  public static final int SYMLINK_NO_PRIVILEGE = 2;
065
066  /**
067   * convert an array of FileStatus to an array of Path
068   * 
069   * @param stats
070   *          an array of FileStatus objects
071   * @return an array of paths corresponding to the input
072   */
073  public static Path[] stat2Paths(FileStatus[] stats) {
074    if (stats == null)
075      return null;
076    Path[] ret = new Path[stats.length];
077    for (int i = 0; i < stats.length; ++i) {
078      ret[i] = stats[i].getPath();
079    }
080    return ret;
081  }
082
083  /**
084   * convert an array of FileStatus to an array of Path.
085   * If stats if null, return path
086   * @param stats
087   *          an array of FileStatus objects
088   * @param path
089   *          default path to return in stats is null
090   * @return an array of paths corresponding to the input
091   */
092  public static Path[] stat2Paths(FileStatus[] stats, Path path) {
093    if (stats == null)
094      return new Path[]{path};
095    else
096      return stat2Paths(stats);
097  }
098  
099  /**
100   * Delete a directory and all its contents.  If
101   * we return false, the directory may be partially-deleted.
102   * (1) If dir is symlink to a file, the symlink is deleted. The file pointed
103   *     to by the symlink is not deleted.
104   * (2) If dir is symlink to a directory, symlink is deleted. The directory
105   *     pointed to by symlink is not deleted.
106   * (3) If dir is a normal file, it is deleted.
107   * (4) If dir is a normal directory, then dir and all its contents recursively
108   *     are deleted.
109   */
110  public static boolean fullyDelete(final File dir) {
111    return fullyDelete(dir, false);
112  }
113  
114  /**
115   * Delete a directory and all its contents.  If
116   * we return false, the directory may be partially-deleted.
117   * (1) If dir is symlink to a file, the symlink is deleted. The file pointed
118   *     to by the symlink is not deleted.
119   * (2) If dir is symlink to a directory, symlink is deleted. The directory
120   *     pointed to by symlink is not deleted.
121   * (3) If dir is a normal file, it is deleted.
122   * (4) If dir is a normal directory, then dir and all its contents recursively
123   *     are deleted.
124   * @param dir the file or directory to be deleted
125   * @param tryGrantPermissions true if permissions should be modified to delete a file.
126   * @return true on success false on failure.
127   */
128  public static boolean fullyDelete(final File dir, boolean tryGrantPermissions) {
129    if (tryGrantPermissions) {
130      // try to chmod +rwx the parent folder of the 'dir': 
131      File parent = dir.getParentFile();
132      grantPermissions(parent);
133    }
134    if (deleteImpl(dir, false)) {
135      // dir is (a) normal file, (b) symlink to a file, (c) empty directory or
136      // (d) symlink to a directory
137      return true;
138    }
139    // handle nonempty directory deletion
140    if (!fullyDeleteContents(dir, tryGrantPermissions)) {
141      return false;
142    }
143    return deleteImpl(dir, true);
144  }
145
146  /**
147   * Returns the target of the given symlink. Returns the empty string if
148   * the given path does not refer to a symlink or there is an error
149   * accessing the symlink.
150   * @param f File representing the symbolic link.
151   * @return The target of the symbolic link, empty string on error or if not
152   *         a symlink.
153   */
154  public static String readLink(File f) {
155    /* NB: Use readSymbolicLink in java.nio.file.Path once available. Could
156     * use getCanonicalPath in File to get the target of the symlink but that
157     * does not indicate if the given path refers to a symlink.
158     */
159    try {
160      return Shell.execCommand(
161          Shell.getReadlinkCommand(f.toString())).trim();
162    } catch (IOException x) {
163      return "";
164    }
165  }
166
167  /*
168   * Pure-Java implementation of "chmod +rwx f".
169   */
170  private static void grantPermissions(final File f) {
171      FileUtil.setExecutable(f, true);
172      FileUtil.setReadable(f, true);
173      FileUtil.setWritable(f, true);
174  }
175
176  private static boolean deleteImpl(final File f, final boolean doLog) {
177    if (f == null) {
178      LOG.warn("null file argument.");
179      return false;
180    }
181    final boolean wasDeleted = f.delete();
182    if (wasDeleted) {
183      return true;
184    }
185    final boolean ex = f.exists();
186    if (doLog && ex) {
187      LOG.warn("Failed to delete file or dir ["
188          + f.getAbsolutePath() + "]: it still exists.");
189    }
190    return !ex;
191  }
192  
193  /**
194   * Delete the contents of a directory, not the directory itself.  If
195   * we return false, the directory may be partially-deleted.
196   * If dir is a symlink to a directory, all the contents of the actual
197   * directory pointed to by dir will be deleted.
198   */
199  public static boolean fullyDeleteContents(final File dir) {
200    return fullyDeleteContents(dir, false);
201  }
202  
203  /**
204   * Delete the contents of a directory, not the directory itself.  If
205   * we return false, the directory may be partially-deleted.
206   * If dir is a symlink to a directory, all the contents of the actual
207   * directory pointed to by dir will be deleted.
208   * @param tryGrantPermissions if 'true', try grant +rwx permissions to this 
209   * and all the underlying directories before trying to delete their contents.
210   */
211  public static boolean fullyDeleteContents(final File dir, final boolean tryGrantPermissions) {
212    if (tryGrantPermissions) {
213      // to be able to list the dir and delete files from it
214      // we must grant the dir rwx permissions: 
215      grantPermissions(dir);
216    }
217    boolean deletionSucceeded = true;
218    final File[] contents = dir.listFiles();
219    if (contents != null) {
220      for (int i = 0; i < contents.length; i++) {
221        if (contents[i].isFile()) {
222          if (!deleteImpl(contents[i], true)) {// normal file or symlink to another file
223            deletionSucceeded = false;
224            continue; // continue deletion of other files/dirs under dir
225          }
226        } else {
227          // Either directory or symlink to another directory.
228          // Try deleting the directory as this might be a symlink
229          boolean b = false;
230          b = deleteImpl(contents[i], false);
231          if (b){
232            //this was indeed a symlink or an empty directory
233            continue;
234          }
235          // if not an empty directory or symlink let
236          // fullydelete handle it.
237          if (!fullyDelete(contents[i], tryGrantPermissions)) {
238            deletionSucceeded = false;
239            // continue deletion of other files/dirs under dir
240          }
241        }
242      }
243    }
244    return deletionSucceeded;
245  }
246
247  /**
248   * Recursively delete a directory.
249   * 
250   * @param fs {@link FileSystem} on which the path is present
251   * @param dir directory to recursively delete 
252   * @throws IOException
253   * @deprecated Use {@link FileSystem#delete(Path, boolean)}
254   */
255  @Deprecated
256  public static void fullyDelete(FileSystem fs, Path dir) 
257  throws IOException {
258    fs.delete(dir, true);
259  }
260
261  //
262  // If the destination is a subdirectory of the source, then
263  // generate exception
264  //
265  private static void checkDependencies(FileSystem srcFS, 
266                                        Path src, 
267                                        FileSystem dstFS, 
268                                        Path dst)
269                                        throws IOException {
270    if (srcFS == dstFS) {
271      String srcq = src.makeQualified(srcFS).toString() + Path.SEPARATOR;
272      String dstq = dst.makeQualified(dstFS).toString() + Path.SEPARATOR;
273      if (dstq.startsWith(srcq)) {
274        if (srcq.length() == dstq.length()) {
275          throw new IOException("Cannot copy " + src + " to itself.");
276        } else {
277          throw new IOException("Cannot copy " + src + " to its subdirectory " +
278                                dst);
279        }
280      }
281    }
282  }
283
284  /** Copy files between FileSystems. */
285  public static boolean copy(FileSystem srcFS, Path src, 
286                             FileSystem dstFS, Path dst, 
287                             boolean deleteSource,
288                             Configuration conf) throws IOException {
289    return copy(srcFS, src, dstFS, dst, deleteSource, true, conf);
290  }
291
292  public static boolean copy(FileSystem srcFS, Path[] srcs, 
293                             FileSystem dstFS, Path dst,
294                             boolean deleteSource, 
295                             boolean overwrite, Configuration conf)
296                             throws IOException {
297    boolean gotException = false;
298    boolean returnVal = true;
299    StringBuilder exceptions = new StringBuilder();
300
301    if (srcs.length == 1)
302      return copy(srcFS, srcs[0], dstFS, dst, deleteSource, overwrite, conf);
303
304    // Check if dest is directory
305    if (!dstFS.exists(dst)) {
306      throw new IOException("`" + dst +"': specified destination directory " +
307                            "does not exist");
308    } else {
309      FileStatus sdst = dstFS.getFileStatus(dst);
310      if (!sdst.isDirectory()) 
311        throw new IOException("copying multiple files, but last argument `" +
312                              dst + "' is not a directory");
313    }
314
315    for (Path src : srcs) {
316      try {
317        if (!copy(srcFS, src, dstFS, dst, deleteSource, overwrite, conf))
318          returnVal = false;
319      } catch (IOException e) {
320        gotException = true;
321        exceptions.append(e.getMessage());
322        exceptions.append("\n");
323      }
324    }
325    if (gotException) {
326      throw new IOException(exceptions.toString());
327    }
328    return returnVal;
329  }
330
331  /** Copy files between FileSystems. */
332  public static boolean copy(FileSystem srcFS, Path src, 
333                             FileSystem dstFS, Path dst, 
334                             boolean deleteSource,
335                             boolean overwrite,
336                             Configuration conf) throws IOException {
337    FileStatus fileStatus = srcFS.getFileStatus(src);
338    return copy(srcFS, fileStatus, dstFS, dst, deleteSource, overwrite, conf);
339  }
340
341  /** Copy files between FileSystems. */
342  public static boolean copy(FileSystem srcFS, FileStatus srcStatus,
343                             FileSystem dstFS, Path dst,
344                             boolean deleteSource,
345                             boolean overwrite,
346                             Configuration conf) throws IOException {
347    Path src = srcStatus.getPath();
348    dst = checkDest(src.getName(), dstFS, dst, overwrite);
349    if (srcStatus.isDirectory()) {
350      checkDependencies(srcFS, src, dstFS, dst);
351      if (!dstFS.mkdirs(dst)) {
352        return false;
353      }
354      FileStatus contents[] = srcFS.listStatus(src);
355      for (int i = 0; i < contents.length; i++) {
356        copy(srcFS, contents[i], dstFS,
357             new Path(dst, contents[i].getPath().getName()),
358             deleteSource, overwrite, conf);
359      }
360    } else {
361      InputStream in=null;
362      OutputStream out = null;
363      try {
364        in = srcFS.open(src);
365        out = dstFS.create(dst, overwrite);
366        IOUtils.copyBytes(in, out, conf, true);
367      } catch (IOException e) {
368        IOUtils.closeStream(out);
369        IOUtils.closeStream(in);
370        throw e;
371      }
372    }
373    if (deleteSource) {
374      return srcFS.delete(src, true);
375    } else {
376      return true;
377    }
378  
379  }
380
381  /** Copy all files in a directory to one output file (merge). */
382  public static boolean copyMerge(FileSystem srcFS, Path srcDir, 
383                                  FileSystem dstFS, Path dstFile, 
384                                  boolean deleteSource,
385                                  Configuration conf, String addString) throws IOException {
386    dstFile = checkDest(srcDir.getName(), dstFS, dstFile, false);
387
388    if (!srcFS.getFileStatus(srcDir).isDirectory())
389      return false;
390   
391    OutputStream out = dstFS.create(dstFile);
392    
393    try {
394      FileStatus contents[] = srcFS.listStatus(srcDir);
395      Arrays.sort(contents);
396      for (int i = 0; i < contents.length; i++) {
397        if (contents[i].isFile()) {
398          InputStream in = srcFS.open(contents[i].getPath());
399          try {
400            IOUtils.copyBytes(in, out, conf, false);
401            if (addString!=null)
402              out.write(addString.getBytes("UTF-8"));
403                
404          } finally {
405            in.close();
406          } 
407        }
408      }
409    } finally {
410      out.close();
411    }
412    
413
414    if (deleteSource) {
415      return srcFS.delete(srcDir, true);
416    } else {
417      return true;
418    }
419  }  
420  
421  /** Copy local files to a FileSystem. */
422  public static boolean copy(File src,
423                             FileSystem dstFS, Path dst,
424                             boolean deleteSource,
425                             Configuration conf) throws IOException {
426    dst = checkDest(src.getName(), dstFS, dst, false);
427
428    if (src.isDirectory()) {
429      if (!dstFS.mkdirs(dst)) {
430        return false;
431      }
432      File contents[] = listFiles(src);
433      for (int i = 0; i < contents.length; i++) {
434        copy(contents[i], dstFS, new Path(dst, contents[i].getName()),
435             deleteSource, conf);
436      }
437    } else if (src.isFile()) {
438      InputStream in = null;
439      OutputStream out =null;
440      try {
441        in = new FileInputStream(src);
442        out = dstFS.create(dst);
443        IOUtils.copyBytes(in, out, conf);
444      } catch (IOException e) {
445        IOUtils.closeStream( out );
446        IOUtils.closeStream( in );
447        throw e;
448      }
449    } else {
450      throw new IOException(src.toString() + 
451                            ": No such file or directory");
452    }
453    if (deleteSource) {
454      return FileUtil.fullyDelete(src);
455    } else {
456      return true;
457    }
458  }
459
460  /** Copy FileSystem files to local files. */
461  public static boolean copy(FileSystem srcFS, Path src, 
462                             File dst, boolean deleteSource,
463                             Configuration conf) throws IOException {
464    FileStatus filestatus = srcFS.getFileStatus(src);
465    return copy(srcFS, filestatus, dst, deleteSource, conf);
466  }
467
468  /** Copy FileSystem files to local files. */
469  private static boolean copy(FileSystem srcFS, FileStatus srcStatus,
470                              File dst, boolean deleteSource,
471                              Configuration conf) throws IOException {
472    Path src = srcStatus.getPath();
473    if (srcStatus.isDirectory()) {
474      if (!dst.mkdirs()) {
475        return false;
476      }
477      FileStatus contents[] = srcFS.listStatus(src);
478      for (int i = 0; i < contents.length; i++) {
479        copy(srcFS, contents[i],
480             new File(dst, contents[i].getPath().getName()),
481             deleteSource, conf);
482      }
483    } else {
484      InputStream in = srcFS.open(src);
485      IOUtils.copyBytes(in, new FileOutputStream(dst), conf);
486    }
487    if (deleteSource) {
488      return srcFS.delete(src, true);
489    } else {
490      return true;
491    }
492  }
493
494  private static Path checkDest(String srcName, FileSystem dstFS, Path dst,
495      boolean overwrite) throws IOException {
496    if (dstFS.exists(dst)) {
497      FileStatus sdst = dstFS.getFileStatus(dst);
498      if (sdst.isDirectory()) {
499        if (null == srcName) {
500          throw new IOException("Target " + dst + " is a directory");
501        }
502        return checkDest(null, dstFS, new Path(dst, srcName), overwrite);
503      } else if (!overwrite) {
504        throw new IOException("Target " + dst + " already exists");
505      }
506    }
507    return dst;
508  }
509
510  /**
511   * Convert a os-native filename to a path that works for the shell.
512   * @param filename The filename to convert
513   * @return The unix pathname
514   * @throws IOException on windows, there can be problems with the subprocess
515   */
516  public static String makeShellPath(String filename) throws IOException {
517    return filename;
518  }
519  
520  /**
521   * Convert a os-native filename to a path that works for the shell.
522   * @param file The filename to convert
523   * @return The unix pathname
524   * @throws IOException on windows, there can be problems with the subprocess
525   */
526  public static String makeShellPath(File file) throws IOException {
527    return makeShellPath(file, false);
528  }
529
530  /**
531   * Convert a os-native filename to a path that works for the shell.
532   * @param file The filename to convert
533   * @param makeCanonicalPath 
534   *          Whether to make canonical path for the file passed
535   * @return The unix pathname
536   * @throws IOException on windows, there can be problems with the subprocess
537   */
538  public static String makeShellPath(File file, boolean makeCanonicalPath) 
539  throws IOException {
540    if (makeCanonicalPath) {
541      return makeShellPath(file.getCanonicalPath());
542    } else {
543      return makeShellPath(file.toString());
544    }
545  }
546
547  /**
548   * Takes an input dir and returns the du on that local directory. Very basic
549   * implementation.
550   * 
551   * @param dir
552   *          The input dir to get the disk space of this local dir
553   * @return The total disk space of the input local directory
554   */
555  public static long getDU(File dir) {
556    long size = 0;
557    if (!dir.exists())
558      return 0;
559    if (!dir.isDirectory()) {
560      return dir.length();
561    } else {
562      File[] allFiles = dir.listFiles();
563      if(allFiles != null) {
564         for (int i = 0; i < allFiles.length; i++) {
565           boolean isSymLink;
566           try {
567             isSymLink = org.apache.commons.io.FileUtils.isSymlink(allFiles[i]);
568           } catch(IOException ioe) {
569             isSymLink = true;
570           }
571           if(!isSymLink) {
572             size += getDU(allFiles[i]);
573           }
574         }
575      }
576      return size;
577    }
578  }
579    
580  /**
581   * Given a File input it will unzip the file in a the unzip directory
582   * passed as the second parameter
583   * @param inFile The zip file as input
584   * @param unzipDir The unzip directory where to unzip the zip file.
585   * @throws IOException
586   */
587  public static void unZip(File inFile, File unzipDir) throws IOException {
588    Enumeration<? extends ZipEntry> entries;
589    ZipFile zipFile = new ZipFile(inFile);
590
591    try {
592      entries = zipFile.entries();
593      while (entries.hasMoreElements()) {
594        ZipEntry entry = entries.nextElement();
595        if (!entry.isDirectory()) {
596          InputStream in = zipFile.getInputStream(entry);
597          try {
598            File file = new File(unzipDir, entry.getName());
599            if (!file.getParentFile().mkdirs()) {           
600              if (!file.getParentFile().isDirectory()) {
601                throw new IOException("Mkdirs failed to create " + 
602                                      file.getParentFile().toString());
603              }
604            }
605            OutputStream out = new FileOutputStream(file);
606            try {
607              byte[] buffer = new byte[8192];
608              int i;
609              while ((i = in.read(buffer)) != -1) {
610                out.write(buffer, 0, i);
611              }
612            } finally {
613              out.close();
614            }
615          } finally {
616            in.close();
617          }
618        }
619      }
620    } finally {
621      zipFile.close();
622    }
623  }
624
625  /**
626   * Given a Tar File as input it will untar the file in a the untar directory
627   * passed as the second parameter
628   * 
629   * This utility will untar ".tar" files and ".tar.gz","tgz" files.
630   *  
631   * @param inFile The tar file as input. 
632   * @param untarDir The untar directory where to untar the tar file.
633   * @throws IOException
634   */
635  public static void unTar(File inFile, File untarDir) throws IOException {
636    if (!untarDir.mkdirs()) {
637      if (!untarDir.isDirectory()) {
638        throw new IOException("Mkdirs failed to create " + untarDir);
639      }
640    }
641
642    boolean gzipped = inFile.toString().endsWith("gz");
643    if(Shell.WINDOWS) {
644      // Tar is not native to Windows. Use simple Java based implementation for 
645      // tests and simple tar archives
646      unTarUsingJava(inFile, untarDir, gzipped);
647    }
648    else {
649      // spawn tar utility to untar archive for full fledged unix behavior such 
650      // as resolving symlinks in tar archives
651      unTarUsingTar(inFile, untarDir, gzipped);
652    }
653  }
654  
655  private static void unTarUsingTar(File inFile, File untarDir,
656      boolean gzipped) throws IOException {
657    StringBuffer untarCommand = new StringBuffer();
658    if (gzipped) {
659      untarCommand.append(" gzip -dc '");
660      untarCommand.append(FileUtil.makeShellPath(inFile));
661      untarCommand.append("' | (");
662    } 
663    untarCommand.append("cd '");
664    untarCommand.append(FileUtil.makeShellPath(untarDir)); 
665    untarCommand.append("' ; ");
666    untarCommand.append("tar -xf ");
667
668    if (gzipped) {
669      untarCommand.append(" -)");
670    } else {
671      untarCommand.append(FileUtil.makeShellPath(inFile));
672    }
673    String[] shellCmd = { "bash", "-c", untarCommand.toString() };
674    ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd);
675    shexec.execute();
676    int exitcode = shexec.getExitCode();
677    if (exitcode != 0) {
678      throw new IOException("Error untarring file " + inFile + 
679                  ". Tar process exited with exit code " + exitcode);
680    }
681  }
682  
683  private static void unTarUsingJava(File inFile, File untarDir,
684      boolean gzipped) throws IOException {
685    InputStream inputStream = null;
686    TarArchiveInputStream tis = null;
687    try {
688      if (gzipped) {
689        inputStream = new BufferedInputStream(new GZIPInputStream(
690            new FileInputStream(inFile)));
691      } else {
692        inputStream = new BufferedInputStream(new FileInputStream(inFile));
693      }
694
695      tis = new TarArchiveInputStream(inputStream);
696
697      for (TarArchiveEntry entry = tis.getNextTarEntry(); entry != null;) {
698        unpackEntries(tis, entry, untarDir);
699        entry = tis.getNextTarEntry();
700      }
701    } finally {
702      IOUtils.cleanup(LOG, tis, inputStream);
703    }
704  }
705  
706  private static void unpackEntries(TarArchiveInputStream tis,
707      TarArchiveEntry entry, File outputDir) throws IOException {
708    if (entry.isDirectory()) {
709      File subDir = new File(outputDir, entry.getName());
710      if (!subDir.mkdirs() && !subDir.isDirectory()) {
711        throw new IOException("Mkdirs failed to create tar internal dir "
712            + outputDir);
713      }
714
715      for (TarArchiveEntry e : entry.getDirectoryEntries()) {
716        unpackEntries(tis, e, subDir);
717      }
718
719      return;
720    }
721
722    File outputFile = new File(outputDir, entry.getName());
723    if (!outputFile.getParentFile().exists()) {
724      if (!outputFile.getParentFile().mkdirs()) {
725        throw new IOException("Mkdirs failed to create tar internal dir "
726            + outputDir);
727      }
728    }
729
730    int count;
731    byte data[] = new byte[2048];
732    try (BufferedOutputStream outputStream = new BufferedOutputStream(
733        new FileOutputStream(outputFile));) {
734
735      while ((count = tis.read(data)) != -1) {
736        outputStream.write(data, 0, count);
737      }
738
739      outputStream.flush();
740    }
741  }
742  
743  /**
744   * Class for creating hardlinks.
745   * Supports Unix, WindXP.
746   * @deprecated Use {@link org.apache.hadoop.fs.HardLink}
747   */
748  @Deprecated
749  public static class HardLink extends org.apache.hadoop.fs.HardLink { 
750    // This is a stub to assist with coordinated change between
751    // COMMON and HDFS projects.  It will be removed after the
752    // corresponding change is committed to HDFS.
753  }
754
755  /**
756   * Create a soft link between a src and destination
757   * only on a local disk. HDFS does not support this.
758   * On Windows, when symlink creation fails due to security
759   * setting, we will log a warning. The return code in this
760   * case is 2.
761   *
762   * @param target the target for symlink 
763   * @param linkname the symlink
764   * @return 0 on success
765   */
766  public static int symLink(String target, String linkname) throws IOException{
767    // Run the input paths through Java's File so that they are converted to the
768    // native OS form
769    File targetFile = new File(
770        Path.getPathWithoutSchemeAndAuthority(new Path(target)).toString());
771    File linkFile = new File(
772        Path.getPathWithoutSchemeAndAuthority(new Path(linkname)).toString());
773
774    // If not on Java7+, copy a file instead of creating a symlink since
775    // Java6 has close to no support for symlinks on Windows. Specifically
776    // File#length and File#renameTo do not work as expected.
777    // (see HADOOP-9061 for additional details)
778    // We still create symlinks for directories, since the scenario in this
779    // case is different. The directory content could change in which
780    // case the symlink loses its purpose (for example task attempt log folder
781    // is symlinked under userlogs and userlogs are generated afterwards).
782    if (Shell.WINDOWS && !Shell.isJava7OrAbove() && targetFile.isFile()) {
783      try {
784        LOG.warn("FileUtil#symlink: On Windows+Java6, copying file instead " +
785            "of creating a symlink. Copying " + target + " -> " + linkname);
786
787        if (!linkFile.getParentFile().exists()) {
788          LOG.warn("Parent directory " + linkFile.getParent() +
789              " does not exist.");
790          return 1;
791        } else {
792          org.apache.commons.io.FileUtils.copyFile(targetFile, linkFile);
793        }
794      } catch (IOException ex) {
795        LOG.warn("FileUtil#symlink failed to copy the file with error: "
796            + ex.getMessage());
797        // Exit with non-zero exit code
798        return 1;
799      }
800      return 0;
801    }
802
803    String[] cmd = Shell.getSymlinkCommand(
804        targetFile.toString(),
805        linkFile.toString());
806
807    ShellCommandExecutor shExec;
808    try {
809      if (Shell.WINDOWS &&
810          linkFile.getParentFile() != null &&
811          !new Path(target).isAbsolute()) {
812        // Relative links on Windows must be resolvable at the time of
813        // creation. To ensure this we run the shell command in the directory
814        // of the link.
815        //
816        shExec = new ShellCommandExecutor(cmd, linkFile.getParentFile());
817      } else {
818        shExec = new ShellCommandExecutor(cmd);
819      }
820      shExec.execute();
821    } catch (Shell.ExitCodeException ec) {
822      int returnVal = ec.getExitCode();
823      if (Shell.WINDOWS && returnVal == SYMLINK_NO_PRIVILEGE) {
824        LOG.warn("Fail to create symbolic links on Windows. "
825            + "The default security settings in Windows disallow non-elevated "
826            + "administrators and all non-administrators from creating symbolic links. "
827            + "This behavior can be changed in the Local Security Policy management console");
828      } else if (returnVal != 0) {
829        LOG.warn("Command '" + StringUtils.join(" ", cmd) + "' failed "
830            + returnVal + " with: " + ec.getMessage());
831      }
832      return returnVal;
833    } catch (IOException e) {
834      if (LOG.isDebugEnabled()) {
835        LOG.debug("Error while create symlink " + linkname + " to " + target
836            + "." + " Exception: " + StringUtils.stringifyException(e));
837      }
838      throw e;
839    }
840    return shExec.getExitCode();
841  }
842
843  /**
844   * Change the permissions on a filename.
845   * @param filename the name of the file to change
846   * @param perm the permission string
847   * @return the exit code from the command
848   * @throws IOException
849   * @throws InterruptedException
850   */
851  public static int chmod(String filename, String perm
852                          ) throws IOException, InterruptedException {
853    return chmod(filename, perm, false);
854  }
855
856  /**
857   * Change the permissions on a file / directory, recursively, if
858   * needed.
859   * @param filename name of the file whose permissions are to change
860   * @param perm permission string
861   * @param recursive true, if permissions should be changed recursively
862   * @return the exit code from the command.
863   * @throws IOException
864   */
865  public static int chmod(String filename, String perm, boolean recursive)
866                            throws IOException {
867    String [] cmd = Shell.getSetPermissionCommand(perm, recursive);
868    String[] args = new String[cmd.length + 1];
869    System.arraycopy(cmd, 0, args, 0, cmd.length);
870    args[cmd.length] = new File(filename).getPath();
871    ShellCommandExecutor shExec = new ShellCommandExecutor(args);
872    try {
873      shExec.execute();
874    }catch(IOException e) {
875      if(LOG.isDebugEnabled()) {
876        LOG.debug("Error while changing permission : " + filename 
877                  +" Exception: " + StringUtils.stringifyException(e));
878      }
879    }
880    return shExec.getExitCode();
881  }
882
883  /**
884   * Set the ownership on a file / directory. User name and group name
885   * cannot both be null.
886   * @param file the file to change
887   * @param username the new user owner name
888   * @param groupname the new group owner name
889   * @throws IOException
890   */
891  public static void setOwner(File file, String username,
892      String groupname) throws IOException {
893    if (username == null && groupname == null) {
894      throw new IOException("username == null && groupname == null");
895    }
896    String arg = (username == null ? "" : username)
897        + (groupname == null ? "" : ":" + groupname);
898    String [] cmd = Shell.getSetOwnerCommand(arg);
899    execCommand(file, cmd);
900  }
901
902  /**
903   * Platform independent implementation for {@link File#setReadable(boolean)}
904   * File#setReadable does not work as expected on Windows.
905   * @param f input file
906   * @param readable
907   * @return true on success, false otherwise
908   */
909  public static boolean setReadable(File f, boolean readable) {
910    if (Shell.WINDOWS) {
911      try {
912        String permission = readable ? "u+r" : "u-r";
913        FileUtil.chmod(f.getCanonicalPath(), permission, false);
914        return true;
915      } catch (IOException ex) {
916        return false;
917      }
918    } else {
919      return f.setReadable(readable);
920    }
921  }
922
923  /**
924   * Platform independent implementation for {@link File#setWritable(boolean)}
925   * File#setWritable does not work as expected on Windows.
926   * @param f input file
927   * @param writable
928   * @return true on success, false otherwise
929   */
930  public static boolean setWritable(File f, boolean writable) {
931    if (Shell.WINDOWS) {
932      try {
933        String permission = writable ? "u+w" : "u-w";
934        FileUtil.chmod(f.getCanonicalPath(), permission, false);
935        return true;
936      } catch (IOException ex) {
937        return false;
938      }
939    } else {
940      return f.setWritable(writable);
941    }
942  }
943
944  /**
945   * Platform independent implementation for {@link File#setExecutable(boolean)}
946   * File#setExecutable does not work as expected on Windows.
947   * Note: revoking execute permission on folders does not have the same
948   * behavior on Windows as on Unix platforms. Creating, deleting or renaming
949   * a file within that folder will still succeed on Windows.
950   * @param f input file
951   * @param executable
952   * @return true on success, false otherwise
953   */
954  public static boolean setExecutable(File f, boolean executable) {
955    if (Shell.WINDOWS) {
956      try {
957        String permission = executable ? "u+x" : "u-x";
958        FileUtil.chmod(f.getCanonicalPath(), permission, false);
959        return true;
960      } catch (IOException ex) {
961        return false;
962      }
963    } else {
964      return f.setExecutable(executable);
965    }
966  }
967
968  /**
969   * Platform independent implementation for {@link File#canRead()}
970   * @param f input file
971   * @return On Unix, same as {@link File#canRead()}
972   *         On Windows, true if process has read access on the path
973   */
974  public static boolean canRead(File f) {
975    if (Shell.WINDOWS) {
976      try {
977        return NativeIO.Windows.access(f.getCanonicalPath(),
978            NativeIO.Windows.AccessRight.ACCESS_READ);
979      } catch (IOException e) {
980        return false;
981      }
982    } else {
983      return f.canRead();
984    }
985  }
986
987  /**
988   * Platform independent implementation for {@link File#canWrite()}
989   * @param f input file
990   * @return On Unix, same as {@link File#canWrite()}
991   *         On Windows, true if process has write access on the path
992   */
993  public static boolean canWrite(File f) {
994    if (Shell.WINDOWS) {
995      try {
996        return NativeIO.Windows.access(f.getCanonicalPath(),
997            NativeIO.Windows.AccessRight.ACCESS_WRITE);
998      } catch (IOException e) {
999        return false;
1000      }
1001    } else {
1002      return f.canWrite();
1003    }
1004  }
1005
1006  /**
1007   * Platform independent implementation for {@link File#canExecute()}
1008   * @param f input file
1009   * @return On Unix, same as {@link File#canExecute()}
1010   *         On Windows, true if process has execute access on the path
1011   */
1012  public static boolean canExecute(File f) {
1013    if (Shell.WINDOWS) {
1014      try {
1015        return NativeIO.Windows.access(f.getCanonicalPath(),
1016            NativeIO.Windows.AccessRight.ACCESS_EXECUTE);
1017      } catch (IOException e) {
1018        return false;
1019      }
1020    } else {
1021      return f.canExecute();
1022    }
1023  }
1024
1025  /**
1026   * Set permissions to the required value. Uses the java primitives instead
1027   * of forking if group == other.
1028   * @param f the file to change
1029   * @param permission the new permissions
1030   * @throws IOException
1031   */
1032  public static void setPermission(File f, FsPermission permission
1033                                   ) throws IOException {
1034    FsAction user = permission.getUserAction();
1035    FsAction group = permission.getGroupAction();
1036    FsAction other = permission.getOtherAction();
1037
1038    // use the native/fork if the group/other permissions are different
1039    // or if the native is available or on Windows
1040    if (group != other || NativeIO.isAvailable() || Shell.WINDOWS) {
1041      execSetPermission(f, permission);
1042      return;
1043    }
1044    
1045    boolean rv = true;
1046    
1047    // read perms
1048    rv = f.setReadable(group.implies(FsAction.READ), false);
1049    checkReturnValue(rv, f, permission);
1050    if (group.implies(FsAction.READ) != user.implies(FsAction.READ)) {
1051      rv = f.setReadable(user.implies(FsAction.READ), true);
1052      checkReturnValue(rv, f, permission);
1053    }
1054
1055    // write perms
1056    rv = f.setWritable(group.implies(FsAction.WRITE), false);
1057    checkReturnValue(rv, f, permission);
1058    if (group.implies(FsAction.WRITE) != user.implies(FsAction.WRITE)) {
1059      rv = f.setWritable(user.implies(FsAction.WRITE), true);
1060      checkReturnValue(rv, f, permission);
1061    }
1062
1063    // exec perms
1064    rv = f.setExecutable(group.implies(FsAction.EXECUTE), false);
1065    checkReturnValue(rv, f, permission);
1066    if (group.implies(FsAction.EXECUTE) != user.implies(FsAction.EXECUTE)) {
1067      rv = f.setExecutable(user.implies(FsAction.EXECUTE), true);
1068      checkReturnValue(rv, f, permission);
1069    }
1070  }
1071
1072  private static void checkReturnValue(boolean rv, File p, 
1073                                       FsPermission permission
1074                                       ) throws IOException {
1075    if (!rv) {
1076      throw new IOException("Failed to set permissions of path: " + p + 
1077                            " to " + 
1078                            String.format("%04o", permission.toShort()));
1079    }
1080  }
1081  
1082  private static void execSetPermission(File f, 
1083                                        FsPermission permission
1084                                       )  throws IOException {
1085    if (NativeIO.isAvailable()) {
1086      NativeIO.POSIX.chmod(f.getCanonicalPath(), permission.toShort());
1087    } else {
1088      execCommand(f, Shell.getSetPermissionCommand(
1089                  String.format("%04o", permission.toShort()), false));
1090    }
1091  }
1092  
1093  static String execCommand(File f, String... cmd) throws IOException {
1094    String[] args = new String[cmd.length + 1];
1095    System.arraycopy(cmd, 0, args, 0, cmd.length);
1096    args[cmd.length] = f.getCanonicalPath();
1097    String output = Shell.execCommand(args);
1098    return output;
1099  }
1100
1101  /**
1102   * Create a tmp file for a base file.
1103   * @param basefile the base file of the tmp
1104   * @param prefix file name prefix of tmp
1105   * @param isDeleteOnExit if true, the tmp will be deleted when the VM exits
1106   * @return a newly created tmp file
1107   * @exception IOException If a tmp file cannot created
1108   * @see java.io.File#createTempFile(String, String, File)
1109   * @see java.io.File#deleteOnExit()
1110   */
1111  public static final File createLocalTempFile(final File basefile,
1112                                               final String prefix,
1113                                               final boolean isDeleteOnExit)
1114    throws IOException {
1115    File tmp = File.createTempFile(prefix + basefile.getName(),
1116                                   "", basefile.getParentFile());
1117    if (isDeleteOnExit) {
1118      tmp.deleteOnExit();
1119    }
1120    return tmp;
1121  }
1122
1123  /**
1124   * Move the src file to the name specified by target.
1125   * @param src the source file
1126   * @param target the target file
1127   * @exception IOException If this operation fails
1128   */
1129  public static void replaceFile(File src, File target) throws IOException {
1130    /* renameTo() has two limitations on Windows platform.
1131     * src.renameTo(target) fails if
1132     * 1) If target already exists OR
1133     * 2) If target is already open for reading/writing.
1134     */
1135    if (!src.renameTo(target)) {
1136      int retries = 5;
1137      while (target.exists() && !target.delete() && retries-- >= 0) {
1138        try {
1139          Thread.sleep(1000);
1140        } catch (InterruptedException e) {
1141          throw new IOException("replaceFile interrupted.");
1142        }
1143      }
1144      if (!src.renameTo(target)) {
1145        throw new IOException("Unable to rename " + src +
1146                              " to " + target);
1147      }
1148    }
1149  }
1150  
1151  /**
1152   * A wrapper for {@link File#listFiles()}. This java.io API returns null 
1153   * when a dir is not a directory or for any I/O error. Instead of having
1154   * null check everywhere File#listFiles() is used, we will add utility API
1155   * to get around this problem. For the majority of cases where we prefer 
1156   * an IOException to be thrown.
1157   * @param dir directory for which listing should be performed
1158   * @return list of files or empty list
1159   * @exception IOException for invalid directory or for a bad disk.
1160   */
1161  public static File[] listFiles(File dir) throws IOException {
1162    File[] files = dir.listFiles();
1163    if(files == null) {
1164      throw new IOException("Invalid directory or I/O error occurred for dir: "
1165                + dir.toString());
1166    }
1167    return files;
1168  }  
1169  
1170  /**
1171   * A wrapper for {@link File#list()}. This java.io API returns null 
1172   * when a dir is not a directory or for any I/O error. Instead of having
1173   * null check everywhere File#list() is used, we will add utility API
1174   * to get around this problem. For the majority of cases where we prefer 
1175   * an IOException to be thrown.
1176   * @param dir directory for which listing should be performed
1177   * @return list of file names or empty string list
1178   * @exception IOException for invalid directory or for a bad disk.
1179   */
1180  public static String[] list(File dir) throws IOException {
1181    String[] fileNames = dir.list();
1182    if(fileNames == null) {
1183      throw new IOException("Invalid directory or I/O error occurred for dir: "
1184                + dir.toString());
1185    }
1186    return fileNames;
1187  }  
1188  
1189  public static String[] createJarWithClassPath(String inputClassPath, Path pwd,
1190      Map<String, String> callerEnv) throws IOException {
1191    return createJarWithClassPath(inputClassPath, pwd, pwd, callerEnv);
1192  }
1193  
1194  /**
1195   * Create a jar file at the given path, containing a manifest with a classpath
1196   * that references all specified entries.
1197   * 
1198   * Some platforms may have an upper limit on command line length.  For example,
1199   * the maximum command line length on Windows is 8191 characters, but the
1200   * length of the classpath may exceed this.  To work around this limitation,
1201   * use this method to create a small intermediate jar with a manifest that
1202   * contains the full classpath.  It returns the absolute path to the new jar,
1203   * which the caller may set as the classpath for a new process.
1204   * 
1205   * Environment variable evaluation is not supported within a jar manifest, so
1206   * this method expands environment variables before inserting classpath entries
1207   * to the manifest.  The method parses environment variables according to
1208   * platform-specific syntax (%VAR% on Windows, or $VAR otherwise).  On Windows,
1209   * environment variables are case-insensitive.  For example, %VAR% and %var%
1210   * evaluate to the same value.
1211   * 
1212   * Specifying the classpath in a jar manifest does not support wildcards, so
1213   * this method expands wildcards internally.  Any classpath entry that ends
1214   * with * is translated to all files at that path with extension .jar or .JAR.
1215   * 
1216   * @param inputClassPath String input classpath to bundle into the jar manifest
1217   * @param pwd Path to working directory to save jar
1218   * @param targetDir path to where the jar execution will have its working dir
1219   * @param callerEnv Map<String, String> caller's environment variables to use
1220   *   for expansion
1221   * @return String[] with absolute path to new jar in position 0 and
1222   *   unexpanded wild card entry path in position 1
1223   * @throws IOException if there is an I/O error while writing the jar file
1224   */
1225  public static String[] createJarWithClassPath(String inputClassPath, Path pwd,
1226      Path targetDir,
1227      Map<String, String> callerEnv) throws IOException {
1228    // Replace environment variables, case-insensitive on Windows
1229    @SuppressWarnings("unchecked")
1230    Map<String, String> env = Shell.WINDOWS ? new CaseInsensitiveMap(callerEnv) :
1231      callerEnv;
1232    String[] classPathEntries = inputClassPath.split(File.pathSeparator);
1233    for (int i = 0; i < classPathEntries.length; ++i) {
1234      classPathEntries[i] = StringUtils.replaceTokens(classPathEntries[i],
1235        StringUtils.ENV_VAR_PATTERN, env);
1236    }
1237    File workingDir = new File(pwd.toString());
1238    if (!workingDir.mkdirs()) {
1239      // If mkdirs returns false because the working directory already exists,
1240      // then this is acceptable.  If it returns false due to some other I/O
1241      // error, then this method will fail later with an IOException while saving
1242      // the jar.
1243      LOG.debug("mkdirs false for " + workingDir + ", execution will continue");
1244    }
1245
1246    StringBuilder unexpandedWildcardClasspath = new StringBuilder();
1247    // Append all entries
1248    List<String> classPathEntryList = new ArrayList<String>(
1249      classPathEntries.length);
1250    for (String classPathEntry: classPathEntries) {
1251      if (classPathEntry.length() == 0) {
1252        continue;
1253      }
1254      if (classPathEntry.endsWith("*")) {
1255        boolean foundWildCardJar = false;
1256        // Append all jars that match the wildcard
1257        Path globPath = new Path(classPathEntry).suffix("{.jar,.JAR}");
1258        FileStatus[] wildcardJars = FileContext.getLocalFSFileContext().util()
1259          .globStatus(globPath);
1260        if (wildcardJars != null) {
1261          for (FileStatus wildcardJar: wildcardJars) {
1262            foundWildCardJar = true;
1263            classPathEntryList.add(wildcardJar.getPath().toUri().toURL()
1264              .toExternalForm());
1265          }
1266        }
1267        if (!foundWildCardJar) {
1268          unexpandedWildcardClasspath.append(File.pathSeparator);
1269          unexpandedWildcardClasspath.append(classPathEntry);
1270        }
1271      } else {
1272        // Append just this entry
1273        File fileCpEntry = null;
1274        if(!new Path(classPathEntry).isAbsolute()) {
1275          fileCpEntry = new File(targetDir.toString(), classPathEntry);
1276        }
1277        else {
1278          fileCpEntry = new File(classPathEntry);
1279        }
1280        String classPathEntryUrl = fileCpEntry.toURI().toURL()
1281          .toExternalForm();
1282
1283        // File.toURI only appends trailing '/' if it can determine that it is a
1284        // directory that already exists.  (See JavaDocs.)  If this entry had a
1285        // trailing '/' specified by the caller, then guarantee that the
1286        // classpath entry in the manifest has a trailing '/', and thus refers to
1287        // a directory instead of a file.  This can happen if the caller is
1288        // creating a classpath jar referencing a directory that hasn't been
1289        // created yet, but will definitely be created before running.
1290        if (classPathEntry.endsWith(Path.SEPARATOR) &&
1291            !classPathEntryUrl.endsWith(Path.SEPARATOR)) {
1292          classPathEntryUrl = classPathEntryUrl + Path.SEPARATOR;
1293        }
1294        classPathEntryList.add(classPathEntryUrl);
1295      }
1296    }
1297    String jarClassPath = StringUtils.join(" ", classPathEntryList);
1298
1299    // Create the manifest
1300    Manifest jarManifest = new Manifest();
1301    jarManifest.getMainAttributes().putValue(
1302        Attributes.Name.MANIFEST_VERSION.toString(), "1.0");
1303    jarManifest.getMainAttributes().putValue(
1304        Attributes.Name.CLASS_PATH.toString(), jarClassPath);
1305
1306    // Write the manifest to output JAR file
1307    File classPathJar = File.createTempFile("classpath-", ".jar", workingDir);
1308    FileOutputStream fos = null;
1309    BufferedOutputStream bos = null;
1310    JarOutputStream jos = null;
1311    try {
1312      fos = new FileOutputStream(classPathJar);
1313      bos = new BufferedOutputStream(fos);
1314      jos = new JarOutputStream(bos, jarManifest);
1315    } finally {
1316      IOUtils.cleanup(LOG, jos, bos, fos);
1317    }
1318    String[] jarCp = {classPathJar.getCanonicalPath(),
1319                        unexpandedWildcardClasspath.toString()};
1320    return jarCp;
1321  }
1322}