001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.fs;
020
021 import java.io.*;
022 import java.util.ArrayList;
023 import java.util.Arrays;
024 import java.util.Enumeration;
025 import java.util.List;
026 import java.util.Map;
027 import java.util.jar.Attributes;
028 import java.util.jar.JarOutputStream;
029 import java.util.jar.Manifest;
030 import java.util.zip.GZIPInputStream;
031 import java.util.zip.ZipEntry;
032 import java.util.zip.ZipFile;
033
034 import org.apache.commons.collections.map.CaseInsensitiveMap;
035 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
036 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
037 import org.apache.hadoop.classification.InterfaceAudience;
038 import org.apache.hadoop.classification.InterfaceStability;
039 import org.apache.hadoop.conf.Configuration;
040 import org.apache.hadoop.fs.permission.FsAction;
041 import org.apache.hadoop.fs.permission.FsPermission;
042 import org.apache.hadoop.io.IOUtils;
043 import org.apache.hadoop.io.nativeio.NativeIO;
044 import org.apache.hadoop.util.StringUtils;
045 import org.apache.hadoop.util.Shell;
046 import org.apache.hadoop.util.Shell.ShellCommandExecutor;
047 import org.apache.commons.logging.Log;
048 import org.apache.commons.logging.LogFactory;
049
050 /**
051 * A collection of file-processing util methods
052 */
053 @InterfaceAudience.Public
054 @InterfaceStability.Evolving
055 public class FileUtil {
056
057 private static final Log LOG = LogFactory.getLog(FileUtil.class);
058
059 /* The error code is defined in winutils to indicate insufficient
060 * privilege to create symbolic links. This value need to keep in
061 * sync with the constant of the same name in:
062 * "src\winutils\common.h"
063 * */
064 public static final int SYMLINK_NO_PRIVILEGE = 2;
065
066 /**
067 * convert an array of FileStatus to an array of Path
068 *
069 * @param stats
070 * an array of FileStatus objects
071 * @return an array of paths corresponding to the input
072 */
073 public static Path[] stat2Paths(FileStatus[] stats) {
074 if (stats == null)
075 return null;
076 Path[] ret = new Path[stats.length];
077 for (int i = 0; i < stats.length; ++i) {
078 ret[i] = stats[i].getPath();
079 }
080 return ret;
081 }
082
083 /**
084 * convert an array of FileStatus to an array of Path.
085 * If stats if null, return path
086 * @param stats
087 * an array of FileStatus objects
088 * @param path
089 * default path to return in stats is null
090 * @return an array of paths corresponding to the input
091 */
092 public static Path[] stat2Paths(FileStatus[] stats, Path path) {
093 if (stats == null)
094 return new Path[]{path};
095 else
096 return stat2Paths(stats);
097 }
098
099 /**
100 * Delete a directory and all its contents. If
101 * we return false, the directory may be partially-deleted.
102 * (1) If dir is symlink to a file, the symlink is deleted. The file pointed
103 * to by the symlink is not deleted.
104 * (2) If dir is symlink to a directory, symlink is deleted. The directory
105 * pointed to by symlink is not deleted.
106 * (3) If dir is a normal file, it is deleted.
107 * (4) If dir is a normal directory, then dir and all its contents recursively
108 * are deleted.
109 */
110 public static boolean fullyDelete(final File dir) {
111 return fullyDelete(dir, false);
112 }
113
114 /**
115 * Delete a directory and all its contents. If
116 * we return false, the directory may be partially-deleted.
117 * (1) If dir is symlink to a file, the symlink is deleted. The file pointed
118 * to by the symlink is not deleted.
119 * (2) If dir is symlink to a directory, symlink is deleted. The directory
120 * pointed to by symlink is not deleted.
121 * (3) If dir is a normal file, it is deleted.
122 * (4) If dir is a normal directory, then dir and all its contents recursively
123 * are deleted.
124 * @param dir the file or directory to be deleted
125 * @param tryGrantPermissions true if permissions should be modified to delete a file.
126 * @return true on success false on failure.
127 */
128 public static boolean fullyDelete(final File dir, boolean tryGrantPermissions) {
129 if (tryGrantPermissions) {
130 // try to chmod +rwx the parent folder of the 'dir':
131 File parent = dir.getParentFile();
132 grantPermissions(parent);
133 }
134 if (deleteImpl(dir, false)) {
135 // dir is (a) normal file, (b) symlink to a file, (c) empty directory or
136 // (d) symlink to a directory
137 return true;
138 }
139 // handle nonempty directory deletion
140 if (!fullyDeleteContents(dir, tryGrantPermissions)) {
141 return false;
142 }
143 return deleteImpl(dir, true);
144 }
145
146 /**
147 * Returns the target of the given symlink. Returns the empty string if
148 * the given path does not refer to a symlink or there is an error
149 * accessing the symlink.
150 * @param f File representing the symbolic link.
151 * @return The target of the symbolic link, empty string on error or if not
152 * a symlink.
153 */
154 public static String readLink(File f) {
155 /* NB: Use readSymbolicLink in java.nio.file.Path once available. Could
156 * use getCanonicalPath in File to get the target of the symlink but that
157 * does not indicate if the given path refers to a symlink.
158 */
159 try {
160 return Shell.execCommand(
161 Shell.getReadlinkCommand(f.toString())).trim();
162 } catch (IOException x) {
163 return "";
164 }
165 }
166
167 /*
168 * Pure-Java implementation of "chmod +rwx f".
169 */
170 private static void grantPermissions(final File f) {
171 FileUtil.setExecutable(f, true);
172 FileUtil.setReadable(f, true);
173 FileUtil.setWritable(f, true);
174 }
175
176 private static boolean deleteImpl(final File f, final boolean doLog) {
177 if (f == null) {
178 LOG.warn("null file argument.");
179 return false;
180 }
181 final boolean wasDeleted = f.delete();
182 if (wasDeleted) {
183 return true;
184 }
185 final boolean ex = f.exists();
186 if (doLog && ex) {
187 LOG.warn("Failed to delete file or dir ["
188 + f.getAbsolutePath() + "]: it still exists.");
189 }
190 return !ex;
191 }
192
193 /**
194 * Delete the contents of a directory, not the directory itself. If
195 * we return false, the directory may be partially-deleted.
196 * If dir is a symlink to a directory, all the contents of the actual
197 * directory pointed to by dir will be deleted.
198 */
199 public static boolean fullyDeleteContents(final File dir) {
200 return fullyDeleteContents(dir, false);
201 }
202
203 /**
204 * Delete the contents of a directory, not the directory itself. If
205 * we return false, the directory may be partially-deleted.
206 * If dir is a symlink to a directory, all the contents of the actual
207 * directory pointed to by dir will be deleted.
208 * @param tryGrantPermissions if 'true', try grant +rwx permissions to this
209 * and all the underlying directories before trying to delete their contents.
210 */
211 public static boolean fullyDeleteContents(final File dir, final boolean tryGrantPermissions) {
212 if (tryGrantPermissions) {
213 // to be able to list the dir and delete files from it
214 // we must grant the dir rwx permissions:
215 grantPermissions(dir);
216 }
217 boolean deletionSucceeded = true;
218 final File[] contents = dir.listFiles();
219 if (contents != null) {
220 for (int i = 0; i < contents.length; i++) {
221 if (contents[i].isFile()) {
222 if (!deleteImpl(contents[i], true)) {// normal file or symlink to another file
223 deletionSucceeded = false;
224 continue; // continue deletion of other files/dirs under dir
225 }
226 } else {
227 // Either directory or symlink to another directory.
228 // Try deleting the directory as this might be a symlink
229 boolean b = false;
230 b = deleteImpl(contents[i], false);
231 if (b){
232 //this was indeed a symlink or an empty directory
233 continue;
234 }
235 // if not an empty directory or symlink let
236 // fullydelete handle it.
237 if (!fullyDelete(contents[i], tryGrantPermissions)) {
238 deletionSucceeded = false;
239 // continue deletion of other files/dirs under dir
240 }
241 }
242 }
243 }
244 return deletionSucceeded;
245 }
246
247 /**
248 * Recursively delete a directory.
249 *
250 * @param fs {@link FileSystem} on which the path is present
251 * @param dir directory to recursively delete
252 * @throws IOException
253 * @deprecated Use {@link FileSystem#delete(Path, boolean)}
254 */
255 @Deprecated
256 public static void fullyDelete(FileSystem fs, Path dir)
257 throws IOException {
258 fs.delete(dir, true);
259 }
260
261 //
262 // If the destination is a subdirectory of the source, then
263 // generate exception
264 //
265 private static void checkDependencies(FileSystem srcFS,
266 Path src,
267 FileSystem dstFS,
268 Path dst)
269 throws IOException {
270 if (srcFS == dstFS) {
271 String srcq = src.makeQualified(srcFS).toString() + Path.SEPARATOR;
272 String dstq = dst.makeQualified(dstFS).toString() + Path.SEPARATOR;
273 if (dstq.startsWith(srcq)) {
274 if (srcq.length() == dstq.length()) {
275 throw new IOException("Cannot copy " + src + " to itself.");
276 } else {
277 throw new IOException("Cannot copy " + src + " to its subdirectory " +
278 dst);
279 }
280 }
281 }
282 }
283
284 /** Copy files between FileSystems. */
285 public static boolean copy(FileSystem srcFS, Path src,
286 FileSystem dstFS, Path dst,
287 boolean deleteSource,
288 Configuration conf) throws IOException {
289 return copy(srcFS, src, dstFS, dst, deleteSource, true, conf);
290 }
291
292 public static boolean copy(FileSystem srcFS, Path[] srcs,
293 FileSystem dstFS, Path dst,
294 boolean deleteSource,
295 boolean overwrite, Configuration conf)
296 throws IOException {
297 boolean gotException = false;
298 boolean returnVal = true;
299 StringBuilder exceptions = new StringBuilder();
300
301 if (srcs.length == 1)
302 return copy(srcFS, srcs[0], dstFS, dst, deleteSource, overwrite, conf);
303
304 // Check if dest is directory
305 if (!dstFS.exists(dst)) {
306 throw new IOException("`" + dst +"': specified destination directory " +
307 "does not exist");
308 } else {
309 FileStatus sdst = dstFS.getFileStatus(dst);
310 if (!sdst.isDirectory())
311 throw new IOException("copying multiple files, but last argument `" +
312 dst + "' is not a directory");
313 }
314
315 for (Path src : srcs) {
316 try {
317 if (!copy(srcFS, src, dstFS, dst, deleteSource, overwrite, conf))
318 returnVal = false;
319 } catch (IOException e) {
320 gotException = true;
321 exceptions.append(e.getMessage());
322 exceptions.append("\n");
323 }
324 }
325 if (gotException) {
326 throw new IOException(exceptions.toString());
327 }
328 return returnVal;
329 }
330
331 /** Copy files between FileSystems. */
332 public static boolean copy(FileSystem srcFS, Path src,
333 FileSystem dstFS, Path dst,
334 boolean deleteSource,
335 boolean overwrite,
336 Configuration conf) throws IOException {
337 FileStatus fileStatus = srcFS.getFileStatus(src);
338 return copy(srcFS, fileStatus, dstFS, dst, deleteSource, overwrite, conf);
339 }
340
341 /** Copy files between FileSystems. */
342 public static boolean copy(FileSystem srcFS, FileStatus srcStatus,
343 FileSystem dstFS, Path dst,
344 boolean deleteSource,
345 boolean overwrite,
346 Configuration conf) throws IOException {
347 Path src = srcStatus.getPath();
348 dst = checkDest(src.getName(), dstFS, dst, overwrite);
349 if (srcStatus.isDirectory()) {
350 checkDependencies(srcFS, src, dstFS, dst);
351 if (!dstFS.mkdirs(dst)) {
352 return false;
353 }
354 FileStatus contents[] = srcFS.listStatus(src);
355 for (int i = 0; i < contents.length; i++) {
356 copy(srcFS, contents[i], dstFS,
357 new Path(dst, contents[i].getPath().getName()),
358 deleteSource, overwrite, conf);
359 }
360 } else {
361 InputStream in=null;
362 OutputStream out = null;
363 try {
364 in = srcFS.open(src);
365 out = dstFS.create(dst, overwrite);
366 IOUtils.copyBytes(in, out, conf, true);
367 } catch (IOException e) {
368 IOUtils.closeStream(out);
369 IOUtils.closeStream(in);
370 throw e;
371 }
372 }
373 if (deleteSource) {
374 return srcFS.delete(src, true);
375 } else {
376 return true;
377 }
378
379 }
380
381 /** Copy all files in a directory to one output file (merge). */
382 public static boolean copyMerge(FileSystem srcFS, Path srcDir,
383 FileSystem dstFS, Path dstFile,
384 boolean deleteSource,
385 Configuration conf, String addString) throws IOException {
386 dstFile = checkDest(srcDir.getName(), dstFS, dstFile, false);
387
388 if (!srcFS.getFileStatus(srcDir).isDirectory())
389 return false;
390
391 OutputStream out = dstFS.create(dstFile);
392
393 try {
394 FileStatus contents[] = srcFS.listStatus(srcDir);
395 Arrays.sort(contents);
396 for (int i = 0; i < contents.length; i++) {
397 if (contents[i].isFile()) {
398 InputStream in = srcFS.open(contents[i].getPath());
399 try {
400 IOUtils.copyBytes(in, out, conf, false);
401 if (addString!=null)
402 out.write(addString.getBytes("UTF-8"));
403
404 } finally {
405 in.close();
406 }
407 }
408 }
409 } finally {
410 out.close();
411 }
412
413
414 if (deleteSource) {
415 return srcFS.delete(srcDir, true);
416 } else {
417 return true;
418 }
419 }
420
421 /** Copy local files to a FileSystem. */
422 public static boolean copy(File src,
423 FileSystem dstFS, Path dst,
424 boolean deleteSource,
425 Configuration conf) throws IOException {
426 dst = checkDest(src.getName(), dstFS, dst, false);
427
428 if (src.isDirectory()) {
429 if (!dstFS.mkdirs(dst)) {
430 return false;
431 }
432 File contents[] = listFiles(src);
433 for (int i = 0; i < contents.length; i++) {
434 copy(contents[i], dstFS, new Path(dst, contents[i].getName()),
435 deleteSource, conf);
436 }
437 } else if (src.isFile()) {
438 InputStream in = null;
439 OutputStream out =null;
440 try {
441 in = new FileInputStream(src);
442 out = dstFS.create(dst);
443 IOUtils.copyBytes(in, out, conf);
444 } catch (IOException e) {
445 IOUtils.closeStream( out );
446 IOUtils.closeStream( in );
447 throw e;
448 }
449 } else {
450 throw new IOException(src.toString() +
451 ": No such file or directory");
452 }
453 if (deleteSource) {
454 return FileUtil.fullyDelete(src);
455 } else {
456 return true;
457 }
458 }
459
460 /** Copy FileSystem files to local files. */
461 public static boolean copy(FileSystem srcFS, Path src,
462 File dst, boolean deleteSource,
463 Configuration conf) throws IOException {
464 FileStatus filestatus = srcFS.getFileStatus(src);
465 return copy(srcFS, filestatus, dst, deleteSource, conf);
466 }
467
468 /** Copy FileSystem files to local files. */
469 private static boolean copy(FileSystem srcFS, FileStatus srcStatus,
470 File dst, boolean deleteSource,
471 Configuration conf) throws IOException {
472 Path src = srcStatus.getPath();
473 if (srcStatus.isDirectory()) {
474 if (!dst.mkdirs()) {
475 return false;
476 }
477 FileStatus contents[] = srcFS.listStatus(src);
478 for (int i = 0; i < contents.length; i++) {
479 copy(srcFS, contents[i],
480 new File(dst, contents[i].getPath().getName()),
481 deleteSource, conf);
482 }
483 } else {
484 InputStream in = srcFS.open(src);
485 IOUtils.copyBytes(in, new FileOutputStream(dst), conf);
486 }
487 if (deleteSource) {
488 return srcFS.delete(src, true);
489 } else {
490 return true;
491 }
492 }
493
494 private static Path checkDest(String srcName, FileSystem dstFS, Path dst,
495 boolean overwrite) throws IOException {
496 if (dstFS.exists(dst)) {
497 FileStatus sdst = dstFS.getFileStatus(dst);
498 if (sdst.isDirectory()) {
499 if (null == srcName) {
500 throw new IOException("Target " + dst + " is a directory");
501 }
502 return checkDest(null, dstFS, new Path(dst, srcName), overwrite);
503 } else if (!overwrite) {
504 throw new IOException("Target " + dst + " already exists");
505 }
506 }
507 return dst;
508 }
509
510 /**
511 * Convert a os-native filename to a path that works for the shell.
512 * @param filename The filename to convert
513 * @return The unix pathname
514 * @throws IOException on windows, there can be problems with the subprocess
515 */
516 public static String makeShellPath(String filename) throws IOException {
517 return filename;
518 }
519
520 /**
521 * Convert a os-native filename to a path that works for the shell.
522 * @param file The filename to convert
523 * @return The unix pathname
524 * @throws IOException on windows, there can be problems with the subprocess
525 */
526 public static String makeShellPath(File file) throws IOException {
527 return makeShellPath(file, false);
528 }
529
530 /**
531 * Convert a os-native filename to a path that works for the shell.
532 * @param file The filename to convert
533 * @param makeCanonicalPath
534 * Whether to make canonical path for the file passed
535 * @return The unix pathname
536 * @throws IOException on windows, there can be problems with the subprocess
537 */
538 public static String makeShellPath(File file, boolean makeCanonicalPath)
539 throws IOException {
540 if (makeCanonicalPath) {
541 return makeShellPath(file.getCanonicalPath());
542 } else {
543 return makeShellPath(file.toString());
544 }
545 }
546
547 /**
548 * Takes an input dir and returns the du on that local directory. Very basic
549 * implementation.
550 *
551 * @param dir
552 * The input dir to get the disk space of this local dir
553 * @return The total disk space of the input local directory
554 */
555 public static long getDU(File dir) {
556 long size = 0;
557 if (!dir.exists())
558 return 0;
559 if (!dir.isDirectory()) {
560 return dir.length();
561 } else {
562 File[] allFiles = dir.listFiles();
563 if(allFiles != null) {
564 for (int i = 0; i < allFiles.length; i++) {
565 boolean isSymLink;
566 try {
567 isSymLink = org.apache.commons.io.FileUtils.isSymlink(allFiles[i]);
568 } catch(IOException ioe) {
569 isSymLink = true;
570 }
571 if(!isSymLink) {
572 size += getDU(allFiles[i]);
573 }
574 }
575 }
576 return size;
577 }
578 }
579
580 /**
581 * Given a File input it will unzip the file in a the unzip directory
582 * passed as the second parameter
583 * @param inFile The zip file as input
584 * @param unzipDir The unzip directory where to unzip the zip file.
585 * @throws IOException
586 */
587 public static void unZip(File inFile, File unzipDir) throws IOException {
588 Enumeration<? extends ZipEntry> entries;
589 ZipFile zipFile = new ZipFile(inFile);
590
591 try {
592 entries = zipFile.entries();
593 while (entries.hasMoreElements()) {
594 ZipEntry entry = entries.nextElement();
595 if (!entry.isDirectory()) {
596 InputStream in = zipFile.getInputStream(entry);
597 try {
598 File file = new File(unzipDir, entry.getName());
599 if (!file.getParentFile().mkdirs()) {
600 if (!file.getParentFile().isDirectory()) {
601 throw new IOException("Mkdirs failed to create " +
602 file.getParentFile().toString());
603 }
604 }
605 OutputStream out = new FileOutputStream(file);
606 try {
607 byte[] buffer = new byte[8192];
608 int i;
609 while ((i = in.read(buffer)) != -1) {
610 out.write(buffer, 0, i);
611 }
612 } finally {
613 out.close();
614 }
615 } finally {
616 in.close();
617 }
618 }
619 }
620 } finally {
621 zipFile.close();
622 }
623 }
624
625 /**
626 * Given a Tar File as input it will untar the file in a the untar directory
627 * passed as the second parameter
628 *
629 * This utility will untar ".tar" files and ".tar.gz","tgz" files.
630 *
631 * @param inFile The tar file as input.
632 * @param untarDir The untar directory where to untar the tar file.
633 * @throws IOException
634 */
635 public static void unTar(File inFile, File untarDir) throws IOException {
636 if (!untarDir.mkdirs()) {
637 if (!untarDir.isDirectory()) {
638 throw new IOException("Mkdirs failed to create " + untarDir);
639 }
640 }
641
642 boolean gzipped = inFile.toString().endsWith("gz");
643 if(Shell.WINDOWS) {
644 // Tar is not native to Windows. Use simple Java based implementation for
645 // tests and simple tar archives
646 unTarUsingJava(inFile, untarDir, gzipped);
647 }
648 else {
649 // spawn tar utility to untar archive for full fledged unix behavior such
650 // as resolving symlinks in tar archives
651 unTarUsingTar(inFile, untarDir, gzipped);
652 }
653 }
654
655 private static void unTarUsingTar(File inFile, File untarDir,
656 boolean gzipped) throws IOException {
657 StringBuffer untarCommand = new StringBuffer();
658 if (gzipped) {
659 untarCommand.append(" gzip -dc '");
660 untarCommand.append(FileUtil.makeShellPath(inFile));
661 untarCommand.append("' | (");
662 }
663 untarCommand.append("cd '");
664 untarCommand.append(FileUtil.makeShellPath(untarDir));
665 untarCommand.append("' ; ");
666 untarCommand.append("tar -xf ");
667
668 if (gzipped) {
669 untarCommand.append(" -)");
670 } else {
671 untarCommand.append(FileUtil.makeShellPath(inFile));
672 }
673 String[] shellCmd = { "bash", "-c", untarCommand.toString() };
674 ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd);
675 shexec.execute();
676 int exitcode = shexec.getExitCode();
677 if (exitcode != 0) {
678 throw new IOException("Error untarring file " + inFile +
679 ". Tar process exited with exit code " + exitcode);
680 }
681 }
682
683 private static void unTarUsingJava(File inFile, File untarDir,
684 boolean gzipped) throws IOException {
685 InputStream inputStream = null;
686 TarArchiveInputStream tis = null;
687 try {
688 if (gzipped) {
689 inputStream = new BufferedInputStream(new GZIPInputStream(
690 new FileInputStream(inFile)));
691 } else {
692 inputStream = new BufferedInputStream(new FileInputStream(inFile));
693 }
694
695 tis = new TarArchiveInputStream(inputStream);
696
697 for (TarArchiveEntry entry = tis.getNextTarEntry(); entry != null;) {
698 unpackEntries(tis, entry, untarDir);
699 entry = tis.getNextTarEntry();
700 }
701 } finally {
702 IOUtils.cleanup(LOG, tis, inputStream);
703 }
704 }
705
706 private static void unpackEntries(TarArchiveInputStream tis,
707 TarArchiveEntry entry, File outputDir) throws IOException {
708 if (entry.isDirectory()) {
709 File subDir = new File(outputDir, entry.getName());
710 if (!subDir.mkdirs() && !subDir.isDirectory()) {
711 throw new IOException("Mkdirs failed to create tar internal dir "
712 + outputDir);
713 }
714
715 for (TarArchiveEntry e : entry.getDirectoryEntries()) {
716 unpackEntries(tis, e, subDir);
717 }
718
719 return;
720 }
721
722 File outputFile = new File(outputDir, entry.getName());
723 if (!outputFile.getParentFile().exists()) {
724 if (!outputFile.getParentFile().mkdirs()) {
725 throw new IOException("Mkdirs failed to create tar internal dir "
726 + outputDir);
727 }
728 }
729
730 int count;
731 byte data[] = new byte[2048];
732 BufferedOutputStream outputStream = new BufferedOutputStream(
733 new FileOutputStream(outputFile));
734
735 while ((count = tis.read(data)) != -1) {
736 outputStream.write(data, 0, count);
737 }
738
739 outputStream.flush();
740 outputStream.close();
741 }
742
743 /**
744 * Class for creating hardlinks.
745 * Supports Unix, WindXP.
746 * @deprecated Use {@link org.apache.hadoop.fs.HardLink}
747 */
748 @Deprecated
749 public static class HardLink extends org.apache.hadoop.fs.HardLink {
750 // This is a stub to assist with coordinated change between
751 // COMMON and HDFS projects. It will be removed after the
752 // corresponding change is committed to HDFS.
753 }
754
755 /**
756 * Create a soft link between a src and destination
757 * only on a local disk. HDFS does not support this.
758 * On Windows, when symlink creation fails due to security
759 * setting, we will log a warning. The return code in this
760 * case is 2.
761 *
762 * @param target the target for symlink
763 * @param linkname the symlink
764 * @return 0 on success
765 */
766 public static int symLink(String target, String linkname) throws IOException{
767 // Run the input paths through Java's File so that they are converted to the
768 // native OS form
769 File targetFile = new File(
770 Path.getPathWithoutSchemeAndAuthority(new Path(target)).toString());
771 File linkFile = new File(
772 Path.getPathWithoutSchemeAndAuthority(new Path(linkname)).toString());
773
774 // If not on Java7+, copy a file instead of creating a symlink since
775 // Java6 has close to no support for symlinks on Windows. Specifically
776 // File#length and File#renameTo do not work as expected.
777 // (see HADOOP-9061 for additional details)
778 // We still create symlinks for directories, since the scenario in this
779 // case is different. The directory content could change in which
780 // case the symlink loses its purpose (for example task attempt log folder
781 // is symlinked under userlogs and userlogs are generated afterwards).
782 if (Shell.WINDOWS && !Shell.isJava7OrAbove() && targetFile.isFile()) {
783 try {
784 LOG.warn("FileUtil#symlink: On Windows+Java6, copying file instead " +
785 "of creating a symlink. Copying " + target + " -> " + linkname);
786
787 if (!linkFile.getParentFile().exists()) {
788 LOG.warn("Parent directory " + linkFile.getParent() +
789 " does not exist.");
790 return 1;
791 } else {
792 org.apache.commons.io.FileUtils.copyFile(targetFile, linkFile);
793 }
794 } catch (IOException ex) {
795 LOG.warn("FileUtil#symlink failed to copy the file with error: "
796 + ex.getMessage());
797 // Exit with non-zero exit code
798 return 1;
799 }
800 return 0;
801 }
802
803 String[] cmd = Shell.getSymlinkCommand(
804 targetFile.toString(),
805 linkFile.toString());
806
807 ShellCommandExecutor shExec;
808 try {
809 if (Shell.WINDOWS &&
810 linkFile.getParentFile() != null &&
811 !new Path(target).isAbsolute()) {
812 // Relative links on Windows must be resolvable at the time of
813 // creation. To ensure this we run the shell command in the directory
814 // of the link.
815 //
816 shExec = new ShellCommandExecutor(cmd, linkFile.getParentFile());
817 } else {
818 shExec = new ShellCommandExecutor(cmd);
819 }
820 shExec.execute();
821 } catch (Shell.ExitCodeException ec) {
822 int returnVal = ec.getExitCode();
823 if (Shell.WINDOWS && returnVal == SYMLINK_NO_PRIVILEGE) {
824 LOG.warn("Fail to create symbolic links on Windows. "
825 + "The default security settings in Windows disallow non-elevated "
826 + "administrators and all non-administrators from creating symbolic links. "
827 + "This behavior can be changed in the Local Security Policy management console");
828 } else if (returnVal != 0) {
829 LOG.warn("Command '" + StringUtils.join(" ", cmd) + "' failed "
830 + returnVal + " with: " + ec.getMessage());
831 }
832 return returnVal;
833 } catch (IOException e) {
834 if (LOG.isDebugEnabled()) {
835 LOG.debug("Error while create symlink " + linkname + " to " + target
836 + "." + " Exception: " + StringUtils.stringifyException(e));
837 }
838 throw e;
839 }
840 return shExec.getExitCode();
841 }
842
843 /**
844 * Change the permissions on a filename.
845 * @param filename the name of the file to change
846 * @param perm the permission string
847 * @return the exit code from the command
848 * @throws IOException
849 * @throws InterruptedException
850 */
851 public static int chmod(String filename, String perm
852 ) throws IOException, InterruptedException {
853 return chmod(filename, perm, false);
854 }
855
856 /**
857 * Change the permissions on a file / directory, recursively, if
858 * needed.
859 * @param filename name of the file whose permissions are to change
860 * @param perm permission string
861 * @param recursive true, if permissions should be changed recursively
862 * @return the exit code from the command.
863 * @throws IOException
864 */
865 public static int chmod(String filename, String perm, boolean recursive)
866 throws IOException {
867 String [] cmd = Shell.getSetPermissionCommand(perm, recursive);
868 String[] args = new String[cmd.length + 1];
869 System.arraycopy(cmd, 0, args, 0, cmd.length);
870 args[cmd.length] = new File(filename).getPath();
871 ShellCommandExecutor shExec = new ShellCommandExecutor(args);
872 try {
873 shExec.execute();
874 }catch(IOException e) {
875 if(LOG.isDebugEnabled()) {
876 LOG.debug("Error while changing permission : " + filename
877 +" Exception: " + StringUtils.stringifyException(e));
878 }
879 }
880 return shExec.getExitCode();
881 }
882
883 /**
884 * Set the ownership on a file / directory. User name and group name
885 * cannot both be null.
886 * @param file the file to change
887 * @param username the new user owner name
888 * @param groupname the new group owner name
889 * @throws IOException
890 */
891 public static void setOwner(File file, String username,
892 String groupname) throws IOException {
893 if (username == null && groupname == null) {
894 throw new IOException("username == null && groupname == null");
895 }
896 String arg = (username == null ? "" : username)
897 + (groupname == null ? "" : ":" + groupname);
898 String [] cmd = Shell.getSetOwnerCommand(arg);
899 execCommand(file, cmd);
900 }
901
902 /**
903 * Platform independent implementation for {@link File#setReadable(boolean)}
904 * File#setReadable does not work as expected on Windows.
905 * @param f input file
906 * @param readable
907 * @return true on success, false otherwise
908 */
909 public static boolean setReadable(File f, boolean readable) {
910 if (Shell.WINDOWS) {
911 try {
912 String permission = readable ? "u+r" : "u-r";
913 FileUtil.chmod(f.getCanonicalPath(), permission, false);
914 return true;
915 } catch (IOException ex) {
916 return false;
917 }
918 } else {
919 return f.setReadable(readable);
920 }
921 }
922
923 /**
924 * Platform independent implementation for {@link File#setWritable(boolean)}
925 * File#setWritable does not work as expected on Windows.
926 * @param f input file
927 * @param writable
928 * @return true on success, false otherwise
929 */
930 public static boolean setWritable(File f, boolean writable) {
931 if (Shell.WINDOWS) {
932 try {
933 String permission = writable ? "u+w" : "u-w";
934 FileUtil.chmod(f.getCanonicalPath(), permission, false);
935 return true;
936 } catch (IOException ex) {
937 return false;
938 }
939 } else {
940 return f.setWritable(writable);
941 }
942 }
943
944 /**
945 * Platform independent implementation for {@link File#setExecutable(boolean)}
946 * File#setExecutable does not work as expected on Windows.
947 * Note: revoking execute permission on folders does not have the same
948 * behavior on Windows as on Unix platforms. Creating, deleting or renaming
949 * a file within that folder will still succeed on Windows.
950 * @param f input file
951 * @param executable
952 * @return true on success, false otherwise
953 */
954 public static boolean setExecutable(File f, boolean executable) {
955 if (Shell.WINDOWS) {
956 try {
957 String permission = executable ? "u+x" : "u-x";
958 FileUtil.chmod(f.getCanonicalPath(), permission, false);
959 return true;
960 } catch (IOException ex) {
961 return false;
962 }
963 } else {
964 return f.setExecutable(executable);
965 }
966 }
967
968 /**
969 * Platform independent implementation for {@link File#canRead()}
970 * @param f input file
971 * @return On Unix, same as {@link File#canRead()}
972 * On Windows, true if process has read access on the path
973 */
974 public static boolean canRead(File f) {
975 if (Shell.WINDOWS) {
976 try {
977 return NativeIO.Windows.access(f.getCanonicalPath(),
978 NativeIO.Windows.AccessRight.ACCESS_READ);
979 } catch (IOException e) {
980 return false;
981 }
982 } else {
983 return f.canRead();
984 }
985 }
986
987 /**
988 * Platform independent implementation for {@link File#canWrite()}
989 * @param f input file
990 * @return On Unix, same as {@link File#canWrite()}
991 * On Windows, true if process has write access on the path
992 */
993 public static boolean canWrite(File f) {
994 if (Shell.WINDOWS) {
995 try {
996 return NativeIO.Windows.access(f.getCanonicalPath(),
997 NativeIO.Windows.AccessRight.ACCESS_WRITE);
998 } catch (IOException e) {
999 return false;
1000 }
1001 } else {
1002 return f.canWrite();
1003 }
1004 }
1005
1006 /**
1007 * Platform independent implementation for {@link File#canExecute()}
1008 * @param f input file
1009 * @return On Unix, same as {@link File#canExecute()}
1010 * On Windows, true if process has execute access on the path
1011 */
1012 public static boolean canExecute(File f) {
1013 if (Shell.WINDOWS) {
1014 try {
1015 return NativeIO.Windows.access(f.getCanonicalPath(),
1016 NativeIO.Windows.AccessRight.ACCESS_EXECUTE);
1017 } catch (IOException e) {
1018 return false;
1019 }
1020 } else {
1021 return f.canExecute();
1022 }
1023 }
1024
1025 /**
1026 * Set permissions to the required value. Uses the java primitives instead
1027 * of forking if group == other.
1028 * @param f the file to change
1029 * @param permission the new permissions
1030 * @throws IOException
1031 */
1032 public static void setPermission(File f, FsPermission permission
1033 ) throws IOException {
1034 FsAction user = permission.getUserAction();
1035 FsAction group = permission.getGroupAction();
1036 FsAction other = permission.getOtherAction();
1037
1038 // use the native/fork if the group/other permissions are different
1039 // or if the native is available or on Windows
1040 if (group != other || NativeIO.isAvailable() || Shell.WINDOWS) {
1041 execSetPermission(f, permission);
1042 return;
1043 }
1044
1045 boolean rv = true;
1046
1047 // read perms
1048 rv = f.setReadable(group.implies(FsAction.READ), false);
1049 checkReturnValue(rv, f, permission);
1050 if (group.implies(FsAction.READ) != user.implies(FsAction.READ)) {
1051 rv = f.setReadable(user.implies(FsAction.READ), true);
1052 checkReturnValue(rv, f, permission);
1053 }
1054
1055 // write perms
1056 rv = f.setWritable(group.implies(FsAction.WRITE), false);
1057 checkReturnValue(rv, f, permission);
1058 if (group.implies(FsAction.WRITE) != user.implies(FsAction.WRITE)) {
1059 rv = f.setWritable(user.implies(FsAction.WRITE), true);
1060 checkReturnValue(rv, f, permission);
1061 }
1062
1063 // exec perms
1064 rv = f.setExecutable(group.implies(FsAction.EXECUTE), false);
1065 checkReturnValue(rv, f, permission);
1066 if (group.implies(FsAction.EXECUTE) != user.implies(FsAction.EXECUTE)) {
1067 rv = f.setExecutable(user.implies(FsAction.EXECUTE), true);
1068 checkReturnValue(rv, f, permission);
1069 }
1070 }
1071
1072 private static void checkReturnValue(boolean rv, File p,
1073 FsPermission permission
1074 ) throws IOException {
1075 if (!rv) {
1076 throw new IOException("Failed to set permissions of path: " + p +
1077 " to " +
1078 String.format("%04o", permission.toShort()));
1079 }
1080 }
1081
1082 private static void execSetPermission(File f,
1083 FsPermission permission
1084 ) throws IOException {
1085 if (NativeIO.isAvailable()) {
1086 NativeIO.POSIX.chmod(f.getCanonicalPath(), permission.toShort());
1087 } else {
1088 execCommand(f, Shell.getSetPermissionCommand(
1089 String.format("%04o", permission.toShort()), false));
1090 }
1091 }
1092
1093 static String execCommand(File f, String... cmd) throws IOException {
1094 String[] args = new String[cmd.length + 1];
1095 System.arraycopy(cmd, 0, args, 0, cmd.length);
1096 args[cmd.length] = f.getCanonicalPath();
1097 String output = Shell.execCommand(args);
1098 return output;
1099 }
1100
1101 /**
1102 * Create a tmp file for a base file.
1103 * @param basefile the base file of the tmp
1104 * @param prefix file name prefix of tmp
1105 * @param isDeleteOnExit if true, the tmp will be deleted when the VM exits
1106 * @return a newly created tmp file
1107 * @exception IOException If a tmp file cannot created
1108 * @see java.io.File#createTempFile(String, String, File)
1109 * @see java.io.File#deleteOnExit()
1110 */
1111 public static final File createLocalTempFile(final File basefile,
1112 final String prefix,
1113 final boolean isDeleteOnExit)
1114 throws IOException {
1115 File tmp = File.createTempFile(prefix + basefile.getName(),
1116 "", basefile.getParentFile());
1117 if (isDeleteOnExit) {
1118 tmp.deleteOnExit();
1119 }
1120 return tmp;
1121 }
1122
1123 /**
1124 * Move the src file to the name specified by target.
1125 * @param src the source file
1126 * @param target the target file
1127 * @exception IOException If this operation fails
1128 */
1129 public static void replaceFile(File src, File target) throws IOException {
1130 /* renameTo() has two limitations on Windows platform.
1131 * src.renameTo(target) fails if
1132 * 1) If target already exists OR
1133 * 2) If target is already open for reading/writing.
1134 */
1135 if (!src.renameTo(target)) {
1136 int retries = 5;
1137 while (target.exists() && !target.delete() && retries-- >= 0) {
1138 try {
1139 Thread.sleep(1000);
1140 } catch (InterruptedException e) {
1141 throw new IOException("replaceFile interrupted.");
1142 }
1143 }
1144 if (!src.renameTo(target)) {
1145 throw new IOException("Unable to rename " + src +
1146 " to " + target);
1147 }
1148 }
1149 }
1150
1151 /**
1152 * A wrapper for {@link File#listFiles()}. This java.io API returns null
1153 * when a dir is not a directory or for any I/O error. Instead of having
1154 * null check everywhere File#listFiles() is used, we will add utility API
1155 * to get around this problem. For the majority of cases where we prefer
1156 * an IOException to be thrown.
1157 * @param dir directory for which listing should be performed
1158 * @return list of files or empty list
1159 * @exception IOException for invalid directory or for a bad disk.
1160 */
1161 public static File[] listFiles(File dir) throws IOException {
1162 File[] files = dir.listFiles();
1163 if(files == null) {
1164 throw new IOException("Invalid directory or I/O error occurred for dir: "
1165 + dir.toString());
1166 }
1167 return files;
1168 }
1169
1170 /**
1171 * A wrapper for {@link File#list()}. This java.io API returns null
1172 * when a dir is not a directory or for any I/O error. Instead of having
1173 * null check everywhere File#list() is used, we will add utility API
1174 * to get around this problem. For the majority of cases where we prefer
1175 * an IOException to be thrown.
1176 * @param dir directory for which listing should be performed
1177 * @return list of file names or empty string list
1178 * @exception IOException for invalid directory or for a bad disk.
1179 */
1180 public static String[] list(File dir) throws IOException {
1181 String[] fileNames = dir.list();
1182 if(fileNames == null) {
1183 throw new IOException("Invalid directory or I/O error occurred for dir: "
1184 + dir.toString());
1185 }
1186 return fileNames;
1187 }
1188
1189 public static String[] createJarWithClassPath(String inputClassPath, Path pwd,
1190 Map<String, String> callerEnv) throws IOException {
1191 return createJarWithClassPath(inputClassPath, pwd, pwd, callerEnv);
1192 }
1193
1194 /**
1195 * Create a jar file at the given path, containing a manifest with a classpath
1196 * that references all specified entries.
1197 *
1198 * Some platforms may have an upper limit on command line length. For example,
1199 * the maximum command line length on Windows is 8191 characters, but the
1200 * length of the classpath may exceed this. To work around this limitation,
1201 * use this method to create a small intermediate jar with a manifest that
1202 * contains the full classpath. It returns the absolute path to the new jar,
1203 * which the caller may set as the classpath for a new process.
1204 *
1205 * Environment variable evaluation is not supported within a jar manifest, so
1206 * this method expands environment variables before inserting classpath entries
1207 * to the manifest. The method parses environment variables according to
1208 * platform-specific syntax (%VAR% on Windows, or $VAR otherwise). On Windows,
1209 * environment variables are case-insensitive. For example, %VAR% and %var%
1210 * evaluate to the same value.
1211 *
1212 * Specifying the classpath in a jar manifest does not support wildcards, so
1213 * this method expands wildcards internally. Any classpath entry that ends
1214 * with * is translated to all files at that path with extension .jar or .JAR.
1215 *
1216 * @param inputClassPath String input classpath to bundle into the jar manifest
1217 * @param pwd Path to working directory to save jar
1218 * @param targetDir path to where the jar execution will have its working dir
1219 * @param callerEnv Map<String, String> caller's environment variables to use
1220 * for expansion
1221 * @return String[] with absolute path to new jar in position 0 and
1222 * unexpanded wild card entry path in position 1
1223 * @throws IOException if there is an I/O error while writing the jar file
1224 */
1225 public static String[] createJarWithClassPath(String inputClassPath, Path pwd,
1226 Path targetDir,
1227 Map<String, String> callerEnv) throws IOException {
1228 // Replace environment variables, case-insensitive on Windows
1229 @SuppressWarnings("unchecked")
1230 Map<String, String> env = Shell.WINDOWS ? new CaseInsensitiveMap(callerEnv) :
1231 callerEnv;
1232 String[] classPathEntries = inputClassPath.split(File.pathSeparator);
1233 for (int i = 0; i < classPathEntries.length; ++i) {
1234 classPathEntries[i] = StringUtils.replaceTokens(classPathEntries[i],
1235 StringUtils.ENV_VAR_PATTERN, env);
1236 }
1237 File workingDir = new File(pwd.toString());
1238 if (!workingDir.mkdirs()) {
1239 // If mkdirs returns false because the working directory already exists,
1240 // then this is acceptable. If it returns false due to some other I/O
1241 // error, then this method will fail later with an IOException while saving
1242 // the jar.
1243 LOG.debug("mkdirs false for " + workingDir + ", execution will continue");
1244 }
1245
1246 StringBuilder unexpandedWildcardClasspath = new StringBuilder();
1247 // Append all entries
1248 List<String> classPathEntryList = new ArrayList<String>(
1249 classPathEntries.length);
1250 for (String classPathEntry: classPathEntries) {
1251 if (classPathEntry.length() == 0) {
1252 continue;
1253 }
1254 if (classPathEntry.endsWith("*")) {
1255 boolean foundWildCardJar = false;
1256 // Append all jars that match the wildcard
1257 Path globPath = new Path(classPathEntry).suffix("{.jar,.JAR}");
1258 FileStatus[] wildcardJars = FileContext.getLocalFSFileContext().util()
1259 .globStatus(globPath);
1260 if (wildcardJars != null) {
1261 for (FileStatus wildcardJar: wildcardJars) {
1262 foundWildCardJar = true;
1263 classPathEntryList.add(wildcardJar.getPath().toUri().toURL()
1264 .toExternalForm());
1265 }
1266 }
1267 if (!foundWildCardJar) {
1268 unexpandedWildcardClasspath.append(File.pathSeparator);
1269 unexpandedWildcardClasspath.append(classPathEntry);
1270 }
1271 } else {
1272 // Append just this entry
1273 File fileCpEntry = null;
1274 if(!new Path(classPathEntry).isAbsolute()) {
1275 fileCpEntry = new File(targetDir.toString(), classPathEntry);
1276 }
1277 else {
1278 fileCpEntry = new File(classPathEntry);
1279 }
1280 String classPathEntryUrl = fileCpEntry.toURI().toURL()
1281 .toExternalForm();
1282
1283 // File.toURI only appends trailing '/' if it can determine that it is a
1284 // directory that already exists. (See JavaDocs.) If this entry had a
1285 // trailing '/' specified by the caller, then guarantee that the
1286 // classpath entry in the manifest has a trailing '/', and thus refers to
1287 // a directory instead of a file. This can happen if the caller is
1288 // creating a classpath jar referencing a directory that hasn't been
1289 // created yet, but will definitely be created before running.
1290 if (classPathEntry.endsWith(Path.SEPARATOR) &&
1291 !classPathEntryUrl.endsWith(Path.SEPARATOR)) {
1292 classPathEntryUrl = classPathEntryUrl + Path.SEPARATOR;
1293 }
1294 classPathEntryList.add(classPathEntryUrl);
1295 }
1296 }
1297 String jarClassPath = StringUtils.join(" ", classPathEntryList);
1298
1299 // Create the manifest
1300 Manifest jarManifest = new Manifest();
1301 jarManifest.getMainAttributes().putValue(
1302 Attributes.Name.MANIFEST_VERSION.toString(), "1.0");
1303 jarManifest.getMainAttributes().putValue(
1304 Attributes.Name.CLASS_PATH.toString(), jarClassPath);
1305
1306 // Write the manifest to output JAR file
1307 File classPathJar = File.createTempFile("classpath-", ".jar", workingDir);
1308 FileOutputStream fos = null;
1309 BufferedOutputStream bos = null;
1310 JarOutputStream jos = null;
1311 try {
1312 fos = new FileOutputStream(classPathJar);
1313 bos = new BufferedOutputStream(fos);
1314 jos = new JarOutputStream(bos, jarManifest);
1315 } finally {
1316 IOUtils.cleanup(LOG, jos, bos, fos);
1317 }
1318 String[] jarCp = {classPathJar.getCanonicalPath(),
1319 unexpandedWildcardClasspath.toString()};
1320 return jarCp;
1321 }
1322 }