001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    
019    package org.apache.hadoop.fs;
020    
021    import java.io.*;
022    import java.util.ArrayList;
023    import java.util.Arrays;
024    import java.util.Enumeration;
025    import java.util.List;
026    import java.util.Map;
027    import java.util.jar.Attributes;
028    import java.util.jar.JarOutputStream;
029    import java.util.jar.Manifest;
030    import java.util.zip.GZIPInputStream;
031    import java.util.zip.ZipEntry;
032    import java.util.zip.ZipFile;
033    
034    import org.apache.commons.collections.map.CaseInsensitiveMap;
035    import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
036    import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
037    import org.apache.hadoop.classification.InterfaceAudience;
038    import org.apache.hadoop.classification.InterfaceStability;
039    import org.apache.hadoop.conf.Configuration;
040    import org.apache.hadoop.fs.permission.FsAction;
041    import org.apache.hadoop.fs.permission.FsPermission;
042    import org.apache.hadoop.io.IOUtils;
043    import org.apache.hadoop.io.nativeio.NativeIO;
044    import org.apache.hadoop.util.StringUtils;
045    import org.apache.hadoop.util.Shell;
046    import org.apache.hadoop.util.Shell.ShellCommandExecutor;
047    import org.apache.commons.logging.Log;
048    import org.apache.commons.logging.LogFactory;
049    
050    /**
051     * A collection of file-processing util methods
052     */
053    @InterfaceAudience.Public
054    @InterfaceStability.Evolving
055    public class FileUtil {
056    
057      private static final Log LOG = LogFactory.getLog(FileUtil.class);
058    
059      /* The error code is defined in winutils to indicate insufficient
060       * privilege to create symbolic links. This value need to keep in
061       * sync with the constant of the same name in:
062       * "src\winutils\common.h"
063       * */
064      public static final int SYMLINK_NO_PRIVILEGE = 2;
065    
066      /**
067       * convert an array of FileStatus to an array of Path
068       * 
069       * @param stats
070       *          an array of FileStatus objects
071       * @return an array of paths corresponding to the input
072       */
073      public static Path[] stat2Paths(FileStatus[] stats) {
074        if (stats == null)
075          return null;
076        Path[] ret = new Path[stats.length];
077        for (int i = 0; i < stats.length; ++i) {
078          ret[i] = stats[i].getPath();
079        }
080        return ret;
081      }
082    
083      /**
084       * convert an array of FileStatus to an array of Path.
085       * If stats if null, return path
086       * @param stats
087       *          an array of FileStatus objects
088       * @param path
089       *          default path to return in stats is null
090       * @return an array of paths corresponding to the input
091       */
092      public static Path[] stat2Paths(FileStatus[] stats, Path path) {
093        if (stats == null)
094          return new Path[]{path};
095        else
096          return stat2Paths(stats);
097      }
098      
099      /**
100       * Delete a directory and all its contents.  If
101       * we return false, the directory may be partially-deleted.
102       * (1) If dir is symlink to a file, the symlink is deleted. The file pointed
103       *     to by the symlink is not deleted.
104       * (2) If dir is symlink to a directory, symlink is deleted. The directory
105       *     pointed to by symlink is not deleted.
106       * (3) If dir is a normal file, it is deleted.
107       * (4) If dir is a normal directory, then dir and all its contents recursively
108       *     are deleted.
109       */
110      public static boolean fullyDelete(final File dir) {
111        return fullyDelete(dir, false);
112      }
113      
114      /**
115       * Delete a directory and all its contents.  If
116       * we return false, the directory may be partially-deleted.
117       * (1) If dir is symlink to a file, the symlink is deleted. The file pointed
118       *     to by the symlink is not deleted.
119       * (2) If dir is symlink to a directory, symlink is deleted. The directory
120       *     pointed to by symlink is not deleted.
121       * (3) If dir is a normal file, it is deleted.
122       * (4) If dir is a normal directory, then dir and all its contents recursively
123       *     are deleted.
124       * @param dir the file or directory to be deleted
125       * @param tryGrantPermissions true if permissions should be modified to delete a file.
126       * @return true on success false on failure.
127       */
128      public static boolean fullyDelete(final File dir, boolean tryGrantPermissions) {
129        if (tryGrantPermissions) {
130          // try to chmod +rwx the parent folder of the 'dir': 
131          File parent = dir.getParentFile();
132          grantPermissions(parent);
133        }
134        if (deleteImpl(dir, false)) {
135          // dir is (a) normal file, (b) symlink to a file, (c) empty directory or
136          // (d) symlink to a directory
137          return true;
138        }
139        // handle nonempty directory deletion
140        if (!fullyDeleteContents(dir, tryGrantPermissions)) {
141          return false;
142        }
143        return deleteImpl(dir, true);
144      }
145    
146      /**
147       * Returns the target of the given symlink. Returns the empty string if
148       * the given path does not refer to a symlink or there is an error
149       * accessing the symlink.
150       * @param f File representing the symbolic link.
151       * @return The target of the symbolic link, empty string on error or if not
152       *         a symlink.
153       */
154      public static String readLink(File f) {
155        /* NB: Use readSymbolicLink in java.nio.file.Path once available. Could
156         * use getCanonicalPath in File to get the target of the symlink but that
157         * does not indicate if the given path refers to a symlink.
158         */
159        try {
160          return Shell.execCommand(
161              Shell.getReadlinkCommand(f.toString())).trim();
162        } catch (IOException x) {
163          return "";
164        }
165      }
166    
167      /*
168       * Pure-Java implementation of "chmod +rwx f".
169       */
170      private static void grantPermissions(final File f) {
171          FileUtil.setExecutable(f, true);
172          FileUtil.setReadable(f, true);
173          FileUtil.setWritable(f, true);
174      }
175    
176      private static boolean deleteImpl(final File f, final boolean doLog) {
177        if (f == null) {
178          LOG.warn("null file argument.");
179          return false;
180        }
181        final boolean wasDeleted = f.delete();
182        if (wasDeleted) {
183          return true;
184        }
185        final boolean ex = f.exists();
186        if (doLog && ex) {
187          LOG.warn("Failed to delete file or dir ["
188              + f.getAbsolutePath() + "]: it still exists.");
189        }
190        return !ex;
191      }
192      
193      /**
194       * Delete the contents of a directory, not the directory itself.  If
195       * we return false, the directory may be partially-deleted.
196       * If dir is a symlink to a directory, all the contents of the actual
197       * directory pointed to by dir will be deleted.
198       */
199      public static boolean fullyDeleteContents(final File dir) {
200        return fullyDeleteContents(dir, false);
201      }
202      
203      /**
204       * Delete the contents of a directory, not the directory itself.  If
205       * we return false, the directory may be partially-deleted.
206       * If dir is a symlink to a directory, all the contents of the actual
207       * directory pointed to by dir will be deleted.
208       * @param tryGrantPermissions if 'true', try grant +rwx permissions to this 
209       * and all the underlying directories before trying to delete their contents.
210       */
211      public static boolean fullyDeleteContents(final File dir, final boolean tryGrantPermissions) {
212        if (tryGrantPermissions) {
213          // to be able to list the dir and delete files from it
214          // we must grant the dir rwx permissions: 
215          grantPermissions(dir);
216        }
217        boolean deletionSucceeded = true;
218        final File[] contents = dir.listFiles();
219        if (contents != null) {
220          for (int i = 0; i < contents.length; i++) {
221            if (contents[i].isFile()) {
222              if (!deleteImpl(contents[i], true)) {// normal file or symlink to another file
223                deletionSucceeded = false;
224                continue; // continue deletion of other files/dirs under dir
225              }
226            } else {
227              // Either directory or symlink to another directory.
228              // Try deleting the directory as this might be a symlink
229              boolean b = false;
230              b = deleteImpl(contents[i], false);
231              if (b){
232                //this was indeed a symlink or an empty directory
233                continue;
234              }
235              // if not an empty directory or symlink let
236              // fullydelete handle it.
237              if (!fullyDelete(contents[i], tryGrantPermissions)) {
238                deletionSucceeded = false;
239                // continue deletion of other files/dirs under dir
240              }
241            }
242          }
243        }
244        return deletionSucceeded;
245      }
246    
247      /**
248       * Recursively delete a directory.
249       * 
250       * @param fs {@link FileSystem} on which the path is present
251       * @param dir directory to recursively delete 
252       * @throws IOException
253       * @deprecated Use {@link FileSystem#delete(Path, boolean)}
254       */
255      @Deprecated
256      public static void fullyDelete(FileSystem fs, Path dir) 
257      throws IOException {
258        fs.delete(dir, true);
259      }
260    
261      //
262      // If the destination is a subdirectory of the source, then
263      // generate exception
264      //
265      private static void checkDependencies(FileSystem srcFS, 
266                                            Path src, 
267                                            FileSystem dstFS, 
268                                            Path dst)
269                                            throws IOException {
270        if (srcFS == dstFS) {
271          String srcq = src.makeQualified(srcFS).toString() + Path.SEPARATOR;
272          String dstq = dst.makeQualified(dstFS).toString() + Path.SEPARATOR;
273          if (dstq.startsWith(srcq)) {
274            if (srcq.length() == dstq.length()) {
275              throw new IOException("Cannot copy " + src + " to itself.");
276            } else {
277              throw new IOException("Cannot copy " + src + " to its subdirectory " +
278                                    dst);
279            }
280          }
281        }
282      }
283    
284      /** Copy files between FileSystems. */
285      public static boolean copy(FileSystem srcFS, Path src, 
286                                 FileSystem dstFS, Path dst, 
287                                 boolean deleteSource,
288                                 Configuration conf) throws IOException {
289        return copy(srcFS, src, dstFS, dst, deleteSource, true, conf);
290      }
291    
292      public static boolean copy(FileSystem srcFS, Path[] srcs, 
293                                 FileSystem dstFS, Path dst,
294                                 boolean deleteSource, 
295                                 boolean overwrite, Configuration conf)
296                                 throws IOException {
297        boolean gotException = false;
298        boolean returnVal = true;
299        StringBuilder exceptions = new StringBuilder();
300    
301        if (srcs.length == 1)
302          return copy(srcFS, srcs[0], dstFS, dst, deleteSource, overwrite, conf);
303    
304        // Check if dest is directory
305        if (!dstFS.exists(dst)) {
306          throw new IOException("`" + dst +"': specified destination directory " +
307                                "does not exist");
308        } else {
309          FileStatus sdst = dstFS.getFileStatus(dst);
310          if (!sdst.isDirectory()) 
311            throw new IOException("copying multiple files, but last argument `" +
312                                  dst + "' is not a directory");
313        }
314    
315        for (Path src : srcs) {
316          try {
317            if (!copy(srcFS, src, dstFS, dst, deleteSource, overwrite, conf))
318              returnVal = false;
319          } catch (IOException e) {
320            gotException = true;
321            exceptions.append(e.getMessage());
322            exceptions.append("\n");
323          }
324        }
325        if (gotException) {
326          throw new IOException(exceptions.toString());
327        }
328        return returnVal;
329      }
330    
331      /** Copy files between FileSystems. */
332      public static boolean copy(FileSystem srcFS, Path src, 
333                                 FileSystem dstFS, Path dst, 
334                                 boolean deleteSource,
335                                 boolean overwrite,
336                                 Configuration conf) throws IOException {
337        FileStatus fileStatus = srcFS.getFileStatus(src);
338        return copy(srcFS, fileStatus, dstFS, dst, deleteSource, overwrite, conf);
339      }
340    
341      /** Copy files between FileSystems. */
342      public static boolean copy(FileSystem srcFS, FileStatus srcStatus,
343                                 FileSystem dstFS, Path dst,
344                                 boolean deleteSource,
345                                 boolean overwrite,
346                                 Configuration conf) throws IOException {
347        Path src = srcStatus.getPath();
348        dst = checkDest(src.getName(), dstFS, dst, overwrite);
349        if (srcStatus.isDirectory()) {
350          checkDependencies(srcFS, src, dstFS, dst);
351          if (!dstFS.mkdirs(dst)) {
352            return false;
353          }
354          FileStatus contents[] = srcFS.listStatus(src);
355          for (int i = 0; i < contents.length; i++) {
356            copy(srcFS, contents[i], dstFS,
357                 new Path(dst, contents[i].getPath().getName()),
358                 deleteSource, overwrite, conf);
359          }
360        } else {
361          InputStream in=null;
362          OutputStream out = null;
363          try {
364            in = srcFS.open(src);
365            out = dstFS.create(dst, overwrite);
366            IOUtils.copyBytes(in, out, conf, true);
367          } catch (IOException e) {
368            IOUtils.closeStream(out);
369            IOUtils.closeStream(in);
370            throw e;
371          }
372        }
373        if (deleteSource) {
374          return srcFS.delete(src, true);
375        } else {
376          return true;
377        }
378      
379      }
380    
381      /** Copy all files in a directory to one output file (merge). */
382      public static boolean copyMerge(FileSystem srcFS, Path srcDir, 
383                                      FileSystem dstFS, Path dstFile, 
384                                      boolean deleteSource,
385                                      Configuration conf, String addString) throws IOException {
386        dstFile = checkDest(srcDir.getName(), dstFS, dstFile, false);
387    
388        if (!srcFS.getFileStatus(srcDir).isDirectory())
389          return false;
390       
391        OutputStream out = dstFS.create(dstFile);
392        
393        try {
394          FileStatus contents[] = srcFS.listStatus(srcDir);
395          Arrays.sort(contents);
396          for (int i = 0; i < contents.length; i++) {
397            if (contents[i].isFile()) {
398              InputStream in = srcFS.open(contents[i].getPath());
399              try {
400                IOUtils.copyBytes(in, out, conf, false);
401                if (addString!=null)
402                  out.write(addString.getBytes("UTF-8"));
403                    
404              } finally {
405                in.close();
406              } 
407            }
408          }
409        } finally {
410          out.close();
411        }
412        
413    
414        if (deleteSource) {
415          return srcFS.delete(srcDir, true);
416        } else {
417          return true;
418        }
419      }  
420      
421      /** Copy local files to a FileSystem. */
422      public static boolean copy(File src,
423                                 FileSystem dstFS, Path dst,
424                                 boolean deleteSource,
425                                 Configuration conf) throws IOException {
426        dst = checkDest(src.getName(), dstFS, dst, false);
427    
428        if (src.isDirectory()) {
429          if (!dstFS.mkdirs(dst)) {
430            return false;
431          }
432          File contents[] = listFiles(src);
433          for (int i = 0; i < contents.length; i++) {
434            copy(contents[i], dstFS, new Path(dst, contents[i].getName()),
435                 deleteSource, conf);
436          }
437        } else if (src.isFile()) {
438          InputStream in = null;
439          OutputStream out =null;
440          try {
441            in = new FileInputStream(src);
442            out = dstFS.create(dst);
443            IOUtils.copyBytes(in, out, conf);
444          } catch (IOException e) {
445            IOUtils.closeStream( out );
446            IOUtils.closeStream( in );
447            throw e;
448          }
449        } else {
450          throw new IOException(src.toString() + 
451                                ": No such file or directory");
452        }
453        if (deleteSource) {
454          return FileUtil.fullyDelete(src);
455        } else {
456          return true;
457        }
458      }
459    
460      /** Copy FileSystem files to local files. */
461      public static boolean copy(FileSystem srcFS, Path src, 
462                                 File dst, boolean deleteSource,
463                                 Configuration conf) throws IOException {
464        FileStatus filestatus = srcFS.getFileStatus(src);
465        return copy(srcFS, filestatus, dst, deleteSource, conf);
466      }
467    
468      /** Copy FileSystem files to local files. */
469      private static boolean copy(FileSystem srcFS, FileStatus srcStatus,
470                                  File dst, boolean deleteSource,
471                                  Configuration conf) throws IOException {
472        Path src = srcStatus.getPath();
473        if (srcStatus.isDirectory()) {
474          if (!dst.mkdirs()) {
475            return false;
476          }
477          FileStatus contents[] = srcFS.listStatus(src);
478          for (int i = 0; i < contents.length; i++) {
479            copy(srcFS, contents[i],
480                 new File(dst, contents[i].getPath().getName()),
481                 deleteSource, conf);
482          }
483        } else {
484          InputStream in = srcFS.open(src);
485          IOUtils.copyBytes(in, new FileOutputStream(dst), conf);
486        }
487        if (deleteSource) {
488          return srcFS.delete(src, true);
489        } else {
490          return true;
491        }
492      }
493    
494      private static Path checkDest(String srcName, FileSystem dstFS, Path dst,
495          boolean overwrite) throws IOException {
496        if (dstFS.exists(dst)) {
497          FileStatus sdst = dstFS.getFileStatus(dst);
498          if (sdst.isDirectory()) {
499            if (null == srcName) {
500              throw new IOException("Target " + dst + " is a directory");
501            }
502            return checkDest(null, dstFS, new Path(dst, srcName), overwrite);
503          } else if (!overwrite) {
504            throw new IOException("Target " + dst + " already exists");
505          }
506        }
507        return dst;
508      }
509    
510      /**
511       * Convert a os-native filename to a path that works for the shell.
512       * @param filename The filename to convert
513       * @return The unix pathname
514       * @throws IOException on windows, there can be problems with the subprocess
515       */
516      public static String makeShellPath(String filename) throws IOException {
517        return filename;
518      }
519      
520      /**
521       * Convert a os-native filename to a path that works for the shell.
522       * @param file The filename to convert
523       * @return The unix pathname
524       * @throws IOException on windows, there can be problems with the subprocess
525       */
526      public static String makeShellPath(File file) throws IOException {
527        return makeShellPath(file, false);
528      }
529    
530      /**
531       * Convert a os-native filename to a path that works for the shell.
532       * @param file The filename to convert
533       * @param makeCanonicalPath 
534       *          Whether to make canonical path for the file passed
535       * @return The unix pathname
536       * @throws IOException on windows, there can be problems with the subprocess
537       */
538      public static String makeShellPath(File file, boolean makeCanonicalPath) 
539      throws IOException {
540        if (makeCanonicalPath) {
541          return makeShellPath(file.getCanonicalPath());
542        } else {
543          return makeShellPath(file.toString());
544        }
545      }
546    
547      /**
548       * Takes an input dir and returns the du on that local directory. Very basic
549       * implementation.
550       * 
551       * @param dir
552       *          The input dir to get the disk space of this local dir
553       * @return The total disk space of the input local directory
554       */
555      public static long getDU(File dir) {
556        long size = 0;
557        if (!dir.exists())
558          return 0;
559        if (!dir.isDirectory()) {
560          return dir.length();
561        } else {
562          File[] allFiles = dir.listFiles();
563          if(allFiles != null) {
564             for (int i = 0; i < allFiles.length; i++) {
565               boolean isSymLink;
566               try {
567                 isSymLink = org.apache.commons.io.FileUtils.isSymlink(allFiles[i]);
568               } catch(IOException ioe) {
569                 isSymLink = true;
570               }
571               if(!isSymLink) {
572                 size += getDU(allFiles[i]);
573               }
574             }
575          }
576          return size;
577        }
578      }
579        
580      /**
581       * Given a File input it will unzip the file in a the unzip directory
582       * passed as the second parameter
583       * @param inFile The zip file as input
584       * @param unzipDir The unzip directory where to unzip the zip file.
585       * @throws IOException
586       */
587      public static void unZip(File inFile, File unzipDir) throws IOException {
588        Enumeration<? extends ZipEntry> entries;
589        ZipFile zipFile = new ZipFile(inFile);
590    
591        try {
592          entries = zipFile.entries();
593          while (entries.hasMoreElements()) {
594            ZipEntry entry = entries.nextElement();
595            if (!entry.isDirectory()) {
596              InputStream in = zipFile.getInputStream(entry);
597              try {
598                File file = new File(unzipDir, entry.getName());
599                if (!file.getParentFile().mkdirs()) {           
600                  if (!file.getParentFile().isDirectory()) {
601                    throw new IOException("Mkdirs failed to create " + 
602                                          file.getParentFile().toString());
603                  }
604                }
605                OutputStream out = new FileOutputStream(file);
606                try {
607                  byte[] buffer = new byte[8192];
608                  int i;
609                  while ((i = in.read(buffer)) != -1) {
610                    out.write(buffer, 0, i);
611                  }
612                } finally {
613                  out.close();
614                }
615              } finally {
616                in.close();
617              }
618            }
619          }
620        } finally {
621          zipFile.close();
622        }
623      }
624    
625      /**
626       * Given a Tar File as input it will untar the file in a the untar directory
627       * passed as the second parameter
628       * 
629       * This utility will untar ".tar" files and ".tar.gz","tgz" files.
630       *  
631       * @param inFile The tar file as input. 
632       * @param untarDir The untar directory where to untar the tar file.
633       * @throws IOException
634       */
635      public static void unTar(File inFile, File untarDir) throws IOException {
636        if (!untarDir.mkdirs()) {
637          if (!untarDir.isDirectory()) {
638            throw new IOException("Mkdirs failed to create " + untarDir);
639          }
640        }
641    
642        boolean gzipped = inFile.toString().endsWith("gz");
643        if(Shell.WINDOWS) {
644          // Tar is not native to Windows. Use simple Java based implementation for 
645          // tests and simple tar archives
646          unTarUsingJava(inFile, untarDir, gzipped);
647        }
648        else {
649          // spawn tar utility to untar archive for full fledged unix behavior such 
650          // as resolving symlinks in tar archives
651          unTarUsingTar(inFile, untarDir, gzipped);
652        }
653      }
654      
655      private static void unTarUsingTar(File inFile, File untarDir,
656          boolean gzipped) throws IOException {
657        StringBuffer untarCommand = new StringBuffer();
658        if (gzipped) {
659          untarCommand.append(" gzip -dc '");
660          untarCommand.append(FileUtil.makeShellPath(inFile));
661          untarCommand.append("' | (");
662        } 
663        untarCommand.append("cd '");
664        untarCommand.append(FileUtil.makeShellPath(untarDir)); 
665        untarCommand.append("' ; ");
666        untarCommand.append("tar -xf ");
667    
668        if (gzipped) {
669          untarCommand.append(" -)");
670        } else {
671          untarCommand.append(FileUtil.makeShellPath(inFile));
672        }
673        String[] shellCmd = { "bash", "-c", untarCommand.toString() };
674        ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd);
675        shexec.execute();
676        int exitcode = shexec.getExitCode();
677        if (exitcode != 0) {
678          throw new IOException("Error untarring file " + inFile + 
679                      ". Tar process exited with exit code " + exitcode);
680        }
681      }
682      
683      private static void unTarUsingJava(File inFile, File untarDir,
684          boolean gzipped) throws IOException {
685        InputStream inputStream = null;
686        TarArchiveInputStream tis = null;
687        try {
688          if (gzipped) {
689            inputStream = new BufferedInputStream(new GZIPInputStream(
690                new FileInputStream(inFile)));
691          } else {
692            inputStream = new BufferedInputStream(new FileInputStream(inFile));
693          }
694    
695          tis = new TarArchiveInputStream(inputStream);
696    
697          for (TarArchiveEntry entry = tis.getNextTarEntry(); entry != null;) {
698            unpackEntries(tis, entry, untarDir);
699            entry = tis.getNextTarEntry();
700          }
701        } finally {
702          IOUtils.cleanup(LOG, tis, inputStream);
703        }
704      }
705      
706      private static void unpackEntries(TarArchiveInputStream tis,
707          TarArchiveEntry entry, File outputDir) throws IOException {
708        if (entry.isDirectory()) {
709          File subDir = new File(outputDir, entry.getName());
710          if (!subDir.mkdirs() && !subDir.isDirectory()) {
711            throw new IOException("Mkdirs failed to create tar internal dir "
712                + outputDir);
713          }
714    
715          for (TarArchiveEntry e : entry.getDirectoryEntries()) {
716            unpackEntries(tis, e, subDir);
717          }
718    
719          return;
720        }
721    
722        File outputFile = new File(outputDir, entry.getName());
723        if (!outputFile.getParentFile().exists()) {
724          if (!outputFile.getParentFile().mkdirs()) {
725            throw new IOException("Mkdirs failed to create tar internal dir "
726                + outputDir);
727          }
728        }
729    
730        int count;
731        byte data[] = new byte[2048];
732        BufferedOutputStream outputStream = new BufferedOutputStream(
733            new FileOutputStream(outputFile));
734    
735        while ((count = tis.read(data)) != -1) {
736          outputStream.write(data, 0, count);
737        }
738    
739        outputStream.flush();
740        outputStream.close();
741      }
742      
743      /**
744       * Class for creating hardlinks.
745       * Supports Unix, WindXP.
746       * @deprecated Use {@link org.apache.hadoop.fs.HardLink}
747       */
748      @Deprecated
749      public static class HardLink extends org.apache.hadoop.fs.HardLink { 
750        // This is a stub to assist with coordinated change between
751        // COMMON and HDFS projects.  It will be removed after the
752        // corresponding change is committed to HDFS.
753      }
754    
755      /**
756       * Create a soft link between a src and destination
757       * only on a local disk. HDFS does not support this.
758       * On Windows, when symlink creation fails due to security
759       * setting, we will log a warning. The return code in this
760       * case is 2.
761       *
762       * @param target the target for symlink 
763       * @param linkname the symlink
764       * @return 0 on success
765       */
766      public static int symLink(String target, String linkname) throws IOException{
767        // Run the input paths through Java's File so that they are converted to the
768        // native OS form
769        File targetFile = new File(
770            Path.getPathWithoutSchemeAndAuthority(new Path(target)).toString());
771        File linkFile = new File(
772            Path.getPathWithoutSchemeAndAuthority(new Path(linkname)).toString());
773    
774        // If not on Java7+, copy a file instead of creating a symlink since
775        // Java6 has close to no support for symlinks on Windows. Specifically
776        // File#length and File#renameTo do not work as expected.
777        // (see HADOOP-9061 for additional details)
778        // We still create symlinks for directories, since the scenario in this
779        // case is different. The directory content could change in which
780        // case the symlink loses its purpose (for example task attempt log folder
781        // is symlinked under userlogs and userlogs are generated afterwards).
782        if (Shell.WINDOWS && !Shell.isJava7OrAbove() && targetFile.isFile()) {
783          try {
784            LOG.warn("FileUtil#symlink: On Windows+Java6, copying file instead " +
785                "of creating a symlink. Copying " + target + " -> " + linkname);
786    
787            if (!linkFile.getParentFile().exists()) {
788              LOG.warn("Parent directory " + linkFile.getParent() +
789                  " does not exist.");
790              return 1;
791            } else {
792              org.apache.commons.io.FileUtils.copyFile(targetFile, linkFile);
793            }
794          } catch (IOException ex) {
795            LOG.warn("FileUtil#symlink failed to copy the file with error: "
796                + ex.getMessage());
797            // Exit with non-zero exit code
798            return 1;
799          }
800          return 0;
801        }
802    
803        String[] cmd = Shell.getSymlinkCommand(
804            targetFile.toString(),
805            linkFile.toString());
806    
807        ShellCommandExecutor shExec;
808        try {
809          if (Shell.WINDOWS &&
810              linkFile.getParentFile() != null &&
811              !new Path(target).isAbsolute()) {
812            // Relative links on Windows must be resolvable at the time of
813            // creation. To ensure this we run the shell command in the directory
814            // of the link.
815            //
816            shExec = new ShellCommandExecutor(cmd, linkFile.getParentFile());
817          } else {
818            shExec = new ShellCommandExecutor(cmd);
819          }
820          shExec.execute();
821        } catch (Shell.ExitCodeException ec) {
822          int returnVal = ec.getExitCode();
823          if (Shell.WINDOWS && returnVal == SYMLINK_NO_PRIVILEGE) {
824            LOG.warn("Fail to create symbolic links on Windows. "
825                + "The default security settings in Windows disallow non-elevated "
826                + "administrators and all non-administrators from creating symbolic links. "
827                + "This behavior can be changed in the Local Security Policy management console");
828          } else if (returnVal != 0) {
829            LOG.warn("Command '" + StringUtils.join(" ", cmd) + "' failed "
830                + returnVal + " with: " + ec.getMessage());
831          }
832          return returnVal;
833        } catch (IOException e) {
834          if (LOG.isDebugEnabled()) {
835            LOG.debug("Error while create symlink " + linkname + " to " + target
836                + "." + " Exception: " + StringUtils.stringifyException(e));
837          }
838          throw e;
839        }
840        return shExec.getExitCode();
841      }
842    
843      /**
844       * Change the permissions on a filename.
845       * @param filename the name of the file to change
846       * @param perm the permission string
847       * @return the exit code from the command
848       * @throws IOException
849       * @throws InterruptedException
850       */
851      public static int chmod(String filename, String perm
852                              ) throws IOException, InterruptedException {
853        return chmod(filename, perm, false);
854      }
855    
856      /**
857       * Change the permissions on a file / directory, recursively, if
858       * needed.
859       * @param filename name of the file whose permissions are to change
860       * @param perm permission string
861       * @param recursive true, if permissions should be changed recursively
862       * @return the exit code from the command.
863       * @throws IOException
864       */
865      public static int chmod(String filename, String perm, boolean recursive)
866                                throws IOException {
867        String [] cmd = Shell.getSetPermissionCommand(perm, recursive);
868        String[] args = new String[cmd.length + 1];
869        System.arraycopy(cmd, 0, args, 0, cmd.length);
870        args[cmd.length] = new File(filename).getPath();
871        ShellCommandExecutor shExec = new ShellCommandExecutor(args);
872        try {
873          shExec.execute();
874        }catch(IOException e) {
875          if(LOG.isDebugEnabled()) {
876            LOG.debug("Error while changing permission : " + filename 
877                      +" Exception: " + StringUtils.stringifyException(e));
878          }
879        }
880        return shExec.getExitCode();
881      }
882    
883      /**
884       * Set the ownership on a file / directory. User name and group name
885       * cannot both be null.
886       * @param file the file to change
887       * @param username the new user owner name
888       * @param groupname the new group owner name
889       * @throws IOException
890       */
891      public static void setOwner(File file, String username,
892          String groupname) throws IOException {
893        if (username == null && groupname == null) {
894          throw new IOException("username == null && groupname == null");
895        }
896        String arg = (username == null ? "" : username)
897            + (groupname == null ? "" : ":" + groupname);
898        String [] cmd = Shell.getSetOwnerCommand(arg);
899        execCommand(file, cmd);
900      }
901    
902      /**
903       * Platform independent implementation for {@link File#setReadable(boolean)}
904       * File#setReadable does not work as expected on Windows.
905       * @param f input file
906       * @param readable
907       * @return true on success, false otherwise
908       */
909      public static boolean setReadable(File f, boolean readable) {
910        if (Shell.WINDOWS) {
911          try {
912            String permission = readable ? "u+r" : "u-r";
913            FileUtil.chmod(f.getCanonicalPath(), permission, false);
914            return true;
915          } catch (IOException ex) {
916            return false;
917          }
918        } else {
919          return f.setReadable(readable);
920        }
921      }
922    
923      /**
924       * Platform independent implementation for {@link File#setWritable(boolean)}
925       * File#setWritable does not work as expected on Windows.
926       * @param f input file
927       * @param writable
928       * @return true on success, false otherwise
929       */
930      public static boolean setWritable(File f, boolean writable) {
931        if (Shell.WINDOWS) {
932          try {
933            String permission = writable ? "u+w" : "u-w";
934            FileUtil.chmod(f.getCanonicalPath(), permission, false);
935            return true;
936          } catch (IOException ex) {
937            return false;
938          }
939        } else {
940          return f.setWritable(writable);
941        }
942      }
943    
944      /**
945       * Platform independent implementation for {@link File#setExecutable(boolean)}
946       * File#setExecutable does not work as expected on Windows.
947       * Note: revoking execute permission on folders does not have the same
948       * behavior on Windows as on Unix platforms. Creating, deleting or renaming
949       * a file within that folder will still succeed on Windows.
950       * @param f input file
951       * @param executable
952       * @return true on success, false otherwise
953       */
954      public static boolean setExecutable(File f, boolean executable) {
955        if (Shell.WINDOWS) {
956          try {
957            String permission = executable ? "u+x" : "u-x";
958            FileUtil.chmod(f.getCanonicalPath(), permission, false);
959            return true;
960          } catch (IOException ex) {
961            return false;
962          }
963        } else {
964          return f.setExecutable(executable);
965        }
966      }
967    
968      /**
969       * Platform independent implementation for {@link File#canRead()}
970       * @param f input file
971       * @return On Unix, same as {@link File#canRead()}
972       *         On Windows, true if process has read access on the path
973       */
974      public static boolean canRead(File f) {
975        if (Shell.WINDOWS) {
976          try {
977            return NativeIO.Windows.access(f.getCanonicalPath(),
978                NativeIO.Windows.AccessRight.ACCESS_READ);
979          } catch (IOException e) {
980            return false;
981          }
982        } else {
983          return f.canRead();
984        }
985      }
986    
987      /**
988       * Platform independent implementation for {@link File#canWrite()}
989       * @param f input file
990       * @return On Unix, same as {@link File#canWrite()}
991       *         On Windows, true if process has write access on the path
992       */
993      public static boolean canWrite(File f) {
994        if (Shell.WINDOWS) {
995          try {
996            return NativeIO.Windows.access(f.getCanonicalPath(),
997                NativeIO.Windows.AccessRight.ACCESS_WRITE);
998          } catch (IOException e) {
999            return false;
1000          }
1001        } else {
1002          return f.canWrite();
1003        }
1004      }
1005    
1006      /**
1007       * Platform independent implementation for {@link File#canExecute()}
1008       * @param f input file
1009       * @return On Unix, same as {@link File#canExecute()}
1010       *         On Windows, true if process has execute access on the path
1011       */
1012      public static boolean canExecute(File f) {
1013        if (Shell.WINDOWS) {
1014          try {
1015            return NativeIO.Windows.access(f.getCanonicalPath(),
1016                NativeIO.Windows.AccessRight.ACCESS_EXECUTE);
1017          } catch (IOException e) {
1018            return false;
1019          }
1020        } else {
1021          return f.canExecute();
1022        }
1023      }
1024    
1025      /**
1026       * Set permissions to the required value. Uses the java primitives instead
1027       * of forking if group == other.
1028       * @param f the file to change
1029       * @param permission the new permissions
1030       * @throws IOException
1031       */
1032      public static void setPermission(File f, FsPermission permission
1033                                       ) throws IOException {
1034        FsAction user = permission.getUserAction();
1035        FsAction group = permission.getGroupAction();
1036        FsAction other = permission.getOtherAction();
1037    
1038        // use the native/fork if the group/other permissions are different
1039        // or if the native is available or on Windows
1040        if (group != other || NativeIO.isAvailable() || Shell.WINDOWS) {
1041          execSetPermission(f, permission);
1042          return;
1043        }
1044        
1045        boolean rv = true;
1046        
1047        // read perms
1048        rv = f.setReadable(group.implies(FsAction.READ), false);
1049        checkReturnValue(rv, f, permission);
1050        if (group.implies(FsAction.READ) != user.implies(FsAction.READ)) {
1051          rv = f.setReadable(user.implies(FsAction.READ), true);
1052          checkReturnValue(rv, f, permission);
1053        }
1054    
1055        // write perms
1056        rv = f.setWritable(group.implies(FsAction.WRITE), false);
1057        checkReturnValue(rv, f, permission);
1058        if (group.implies(FsAction.WRITE) != user.implies(FsAction.WRITE)) {
1059          rv = f.setWritable(user.implies(FsAction.WRITE), true);
1060          checkReturnValue(rv, f, permission);
1061        }
1062    
1063        // exec perms
1064        rv = f.setExecutable(group.implies(FsAction.EXECUTE), false);
1065        checkReturnValue(rv, f, permission);
1066        if (group.implies(FsAction.EXECUTE) != user.implies(FsAction.EXECUTE)) {
1067          rv = f.setExecutable(user.implies(FsAction.EXECUTE), true);
1068          checkReturnValue(rv, f, permission);
1069        }
1070      }
1071    
1072      private static void checkReturnValue(boolean rv, File p, 
1073                                           FsPermission permission
1074                                           ) throws IOException {
1075        if (!rv) {
1076          throw new IOException("Failed to set permissions of path: " + p + 
1077                                " to " + 
1078                                String.format("%04o", permission.toShort()));
1079        }
1080      }
1081      
1082      private static void execSetPermission(File f, 
1083                                            FsPermission permission
1084                                           )  throws IOException {
1085        if (NativeIO.isAvailable()) {
1086          NativeIO.POSIX.chmod(f.getCanonicalPath(), permission.toShort());
1087        } else {
1088          execCommand(f, Shell.getSetPermissionCommand(
1089                      String.format("%04o", permission.toShort()), false));
1090        }
1091      }
1092      
1093      static String execCommand(File f, String... cmd) throws IOException {
1094        String[] args = new String[cmd.length + 1];
1095        System.arraycopy(cmd, 0, args, 0, cmd.length);
1096        args[cmd.length] = f.getCanonicalPath();
1097        String output = Shell.execCommand(args);
1098        return output;
1099      }
1100    
1101      /**
1102       * Create a tmp file for a base file.
1103       * @param basefile the base file of the tmp
1104       * @param prefix file name prefix of tmp
1105       * @param isDeleteOnExit if true, the tmp will be deleted when the VM exits
1106       * @return a newly created tmp file
1107       * @exception IOException If a tmp file cannot created
1108       * @see java.io.File#createTempFile(String, String, File)
1109       * @see java.io.File#deleteOnExit()
1110       */
1111      public static final File createLocalTempFile(final File basefile,
1112                                                   final String prefix,
1113                                                   final boolean isDeleteOnExit)
1114        throws IOException {
1115        File tmp = File.createTempFile(prefix + basefile.getName(),
1116                                       "", basefile.getParentFile());
1117        if (isDeleteOnExit) {
1118          tmp.deleteOnExit();
1119        }
1120        return tmp;
1121      }
1122    
1123      /**
1124       * Move the src file to the name specified by target.
1125       * @param src the source file
1126       * @param target the target file
1127       * @exception IOException If this operation fails
1128       */
1129      public static void replaceFile(File src, File target) throws IOException {
1130        /* renameTo() has two limitations on Windows platform.
1131         * src.renameTo(target) fails if
1132         * 1) If target already exists OR
1133         * 2) If target is already open for reading/writing.
1134         */
1135        if (!src.renameTo(target)) {
1136          int retries = 5;
1137          while (target.exists() && !target.delete() && retries-- >= 0) {
1138            try {
1139              Thread.sleep(1000);
1140            } catch (InterruptedException e) {
1141              throw new IOException("replaceFile interrupted.");
1142            }
1143          }
1144          if (!src.renameTo(target)) {
1145            throw new IOException("Unable to rename " + src +
1146                                  " to " + target);
1147          }
1148        }
1149      }
1150      
1151      /**
1152       * A wrapper for {@link File#listFiles()}. This java.io API returns null 
1153       * when a dir is not a directory or for any I/O error. Instead of having
1154       * null check everywhere File#listFiles() is used, we will add utility API
1155       * to get around this problem. For the majority of cases where we prefer 
1156       * an IOException to be thrown.
1157       * @param dir directory for which listing should be performed
1158       * @return list of files or empty list
1159       * @exception IOException for invalid directory or for a bad disk.
1160       */
1161      public static File[] listFiles(File dir) throws IOException {
1162        File[] files = dir.listFiles();
1163        if(files == null) {
1164          throw new IOException("Invalid directory or I/O error occurred for dir: "
1165                    + dir.toString());
1166        }
1167        return files;
1168      }  
1169      
1170      /**
1171       * A wrapper for {@link File#list()}. This java.io API returns null 
1172       * when a dir is not a directory or for any I/O error. Instead of having
1173       * null check everywhere File#list() is used, we will add utility API
1174       * to get around this problem. For the majority of cases where we prefer 
1175       * an IOException to be thrown.
1176       * @param dir directory for which listing should be performed
1177       * @return list of file names or empty string list
1178       * @exception IOException for invalid directory or for a bad disk.
1179       */
1180      public static String[] list(File dir) throws IOException {
1181        String[] fileNames = dir.list();
1182        if(fileNames == null) {
1183          throw new IOException("Invalid directory or I/O error occurred for dir: "
1184                    + dir.toString());
1185        }
1186        return fileNames;
1187      }  
1188      
1189      public static String[] createJarWithClassPath(String inputClassPath, Path pwd,
1190          Map<String, String> callerEnv) throws IOException {
1191        return createJarWithClassPath(inputClassPath, pwd, pwd, callerEnv);
1192      }
1193      
1194      /**
1195       * Create a jar file at the given path, containing a manifest with a classpath
1196       * that references all specified entries.
1197       * 
1198       * Some platforms may have an upper limit on command line length.  For example,
1199       * the maximum command line length on Windows is 8191 characters, but the
1200       * length of the classpath may exceed this.  To work around this limitation,
1201       * use this method to create a small intermediate jar with a manifest that
1202       * contains the full classpath.  It returns the absolute path to the new jar,
1203       * which the caller may set as the classpath for a new process.
1204       * 
1205       * Environment variable evaluation is not supported within a jar manifest, so
1206       * this method expands environment variables before inserting classpath entries
1207       * to the manifest.  The method parses environment variables according to
1208       * platform-specific syntax (%VAR% on Windows, or $VAR otherwise).  On Windows,
1209       * environment variables are case-insensitive.  For example, %VAR% and %var%
1210       * evaluate to the same value.
1211       * 
1212       * Specifying the classpath in a jar manifest does not support wildcards, so
1213       * this method expands wildcards internally.  Any classpath entry that ends
1214       * with * is translated to all files at that path with extension .jar or .JAR.
1215       * 
1216       * @param inputClassPath String input classpath to bundle into the jar manifest
1217       * @param pwd Path to working directory to save jar
1218       * @param targetDir path to where the jar execution will have its working dir
1219       * @param callerEnv Map<String, String> caller's environment variables to use
1220       *   for expansion
1221       * @return String[] with absolute path to new jar in position 0 and
1222       *   unexpanded wild card entry path in position 1
1223       * @throws IOException if there is an I/O error while writing the jar file
1224       */
1225      public static String[] createJarWithClassPath(String inputClassPath, Path pwd,
1226          Path targetDir,
1227          Map<String, String> callerEnv) throws IOException {
1228        // Replace environment variables, case-insensitive on Windows
1229        @SuppressWarnings("unchecked")
1230        Map<String, String> env = Shell.WINDOWS ? new CaseInsensitiveMap(callerEnv) :
1231          callerEnv;
1232        String[] classPathEntries = inputClassPath.split(File.pathSeparator);
1233        for (int i = 0; i < classPathEntries.length; ++i) {
1234          classPathEntries[i] = StringUtils.replaceTokens(classPathEntries[i],
1235            StringUtils.ENV_VAR_PATTERN, env);
1236        }
1237        File workingDir = new File(pwd.toString());
1238        if (!workingDir.mkdirs()) {
1239          // If mkdirs returns false because the working directory already exists,
1240          // then this is acceptable.  If it returns false due to some other I/O
1241          // error, then this method will fail later with an IOException while saving
1242          // the jar.
1243          LOG.debug("mkdirs false for " + workingDir + ", execution will continue");
1244        }
1245    
1246        StringBuilder unexpandedWildcardClasspath = new StringBuilder();
1247        // Append all entries
1248        List<String> classPathEntryList = new ArrayList<String>(
1249          classPathEntries.length);
1250        for (String classPathEntry: classPathEntries) {
1251          if (classPathEntry.length() == 0) {
1252            continue;
1253          }
1254          if (classPathEntry.endsWith("*")) {
1255            boolean foundWildCardJar = false;
1256            // Append all jars that match the wildcard
1257            Path globPath = new Path(classPathEntry).suffix("{.jar,.JAR}");
1258            FileStatus[] wildcardJars = FileContext.getLocalFSFileContext().util()
1259              .globStatus(globPath);
1260            if (wildcardJars != null) {
1261              for (FileStatus wildcardJar: wildcardJars) {
1262                foundWildCardJar = true;
1263                classPathEntryList.add(wildcardJar.getPath().toUri().toURL()
1264                  .toExternalForm());
1265              }
1266            }
1267            if (!foundWildCardJar) {
1268              unexpandedWildcardClasspath.append(File.pathSeparator);
1269              unexpandedWildcardClasspath.append(classPathEntry);
1270            }
1271          } else {
1272            // Append just this entry
1273            File fileCpEntry = null;
1274            if(!new Path(classPathEntry).isAbsolute()) {
1275              fileCpEntry = new File(targetDir.toString(), classPathEntry);
1276            }
1277            else {
1278              fileCpEntry = new File(classPathEntry);
1279            }
1280            String classPathEntryUrl = fileCpEntry.toURI().toURL()
1281              .toExternalForm();
1282    
1283            // File.toURI only appends trailing '/' if it can determine that it is a
1284            // directory that already exists.  (See JavaDocs.)  If this entry had a
1285            // trailing '/' specified by the caller, then guarantee that the
1286            // classpath entry in the manifest has a trailing '/', and thus refers to
1287            // a directory instead of a file.  This can happen if the caller is
1288            // creating a classpath jar referencing a directory that hasn't been
1289            // created yet, but will definitely be created before running.
1290            if (classPathEntry.endsWith(Path.SEPARATOR) &&
1291                !classPathEntryUrl.endsWith(Path.SEPARATOR)) {
1292              classPathEntryUrl = classPathEntryUrl + Path.SEPARATOR;
1293            }
1294            classPathEntryList.add(classPathEntryUrl);
1295          }
1296        }
1297        String jarClassPath = StringUtils.join(" ", classPathEntryList);
1298    
1299        // Create the manifest
1300        Manifest jarManifest = new Manifest();
1301        jarManifest.getMainAttributes().putValue(
1302            Attributes.Name.MANIFEST_VERSION.toString(), "1.0");
1303        jarManifest.getMainAttributes().putValue(
1304            Attributes.Name.CLASS_PATH.toString(), jarClassPath);
1305    
1306        // Write the manifest to output JAR file
1307        File classPathJar = File.createTempFile("classpath-", ".jar", workingDir);
1308        FileOutputStream fos = null;
1309        BufferedOutputStream bos = null;
1310        JarOutputStream jos = null;
1311        try {
1312          fos = new FileOutputStream(classPathJar);
1313          bos = new BufferedOutputStream(fos);
1314          jos = new JarOutputStream(bos, jarManifest);
1315        } finally {
1316          IOUtils.cleanup(LOG, jos, bos, fos);
1317        }
1318        String[] jarCp = {classPathJar.getCanonicalPath(),
1319                            unexpandedWildcardClasspath.toString()};
1320        return jarCp;
1321      }
1322    }