001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.fs;
019
020 import java.io.FileNotFoundException;
021 import java.io.IOException;
022 import java.lang.reflect.Constructor;
023 import java.net.URI;
024 import java.net.URISyntaxException;
025 import java.util.ArrayList;
026 import java.util.EnumSet;
027 import java.util.HashMap;
028 import java.util.List;
029 import java.util.Map;
030 import java.util.NoSuchElementException;
031 import java.util.StringTokenizer;
032 import java.util.concurrent.ConcurrentHashMap;
033
034 import org.apache.commons.logging.Log;
035 import org.apache.commons.logging.LogFactory;
036 import org.apache.hadoop.HadoopIllegalArgumentException;
037 import org.apache.hadoop.classification.InterfaceAudience;
038 import org.apache.hadoop.classification.InterfaceStability;
039 import org.apache.hadoop.conf.Configuration;
040 import org.apache.hadoop.fs.FileSystem.Statistics;
041 import org.apache.hadoop.fs.Options.ChecksumOpt;
042 import org.apache.hadoop.fs.Options.CreateOpts;
043 import org.apache.hadoop.fs.Options.Rename;
044 import org.apache.hadoop.fs.permission.AclEntry;
045 import org.apache.hadoop.fs.permission.AclStatus;
046 import org.apache.hadoop.fs.permission.FsPermission;
047 import org.apache.hadoop.fs.InvalidPathException;
048 import org.apache.hadoop.security.AccessControlException;
049 import org.apache.hadoop.security.SecurityUtil;
050 import org.apache.hadoop.security.token.Token;
051 import org.apache.hadoop.util.Progressable;
052
053 /**
054 * This class provides an interface for implementors of a Hadoop file system
055 * (analogous to the VFS of Unix). Applications do not access this class;
056 * instead they access files across all file systems using {@link FileContext}.
057 *
058 * Pathnames passed to AbstractFileSystem can be fully qualified URI that
059 * matches the "this" file system (ie same scheme and authority)
060 * or a Slash-relative name that is assumed to be relative
061 * to the root of the "this" file system .
062 */
063 @InterfaceAudience.Public
064 @InterfaceStability.Evolving /*Evolving for a release,to be changed to Stable */
065 public abstract class AbstractFileSystem {
066 static final Log LOG = LogFactory.getLog(AbstractFileSystem.class);
067
068 /** Recording statistics per a file system class. */
069 private static final Map<URI, Statistics>
070 STATISTICS_TABLE = new HashMap<URI, Statistics>();
071
072 /** Cache of constructors for each file system class. */
073 private static final Map<Class<?>, Constructor<?>> CONSTRUCTOR_CACHE =
074 new ConcurrentHashMap<Class<?>, Constructor<?>>();
075
076 private static final Class<?>[] URI_CONFIG_ARGS =
077 new Class[]{URI.class, Configuration.class};
078
079 /** The statistics for this file system. */
080 protected Statistics statistics;
081
082 private final URI myUri;
083
084 public Statistics getStatistics() {
085 return statistics;
086 }
087
088 /**
089 * Returns true if the specified string is considered valid in the path part
090 * of a URI by this file system. The default implementation enforces the rules
091 * of HDFS, but subclasses may override this method to implement specific
092 * validation rules for specific file systems.
093 *
094 * @param src String source filename to check, path part of the URI
095 * @return boolean true if the specified string is considered valid
096 */
097 public boolean isValidName(String src) {
098 // Prohibit ".." "." and anything containing ":"
099 StringTokenizer tokens = new StringTokenizer(src, Path.SEPARATOR);
100 while(tokens.hasMoreTokens()) {
101 String element = tokens.nextToken();
102 if (element.equals("..") ||
103 element.equals(".") ||
104 (element.indexOf(":") >= 0)) {
105 return false;
106 }
107 }
108 return true;
109 }
110
111 /**
112 * Create an object for the given class and initialize it from conf.
113 * @param theClass class of which an object is created
114 * @param conf Configuration
115 * @return a new object
116 */
117 @SuppressWarnings("unchecked")
118 static <T> T newInstance(Class<T> theClass,
119 URI uri, Configuration conf) {
120 T result;
121 try {
122 Constructor<T> meth = (Constructor<T>) CONSTRUCTOR_CACHE.get(theClass);
123 if (meth == null) {
124 meth = theClass.getDeclaredConstructor(URI_CONFIG_ARGS);
125 meth.setAccessible(true);
126 CONSTRUCTOR_CACHE.put(theClass, meth);
127 }
128 result = meth.newInstance(uri, conf);
129 } catch (Exception e) {
130 throw new RuntimeException(e);
131 }
132 return result;
133 }
134
135 /**
136 * Create a file system instance for the specified uri using the conf. The
137 * conf is used to find the class name that implements the file system. The
138 * conf is also passed to the file system for its configuration.
139 *
140 * @param uri URI of the file system
141 * @param conf Configuration for the file system
142 *
143 * @return Returns the file system for the given URI
144 *
145 * @throws UnsupportedFileSystemException file system for <code>uri</code> is
146 * not found
147 */
148 public static AbstractFileSystem createFileSystem(URI uri, Configuration conf)
149 throws UnsupportedFileSystemException {
150 Class<?> clazz = conf.getClass("fs.AbstractFileSystem." +
151 uri.getScheme() + ".impl", null);
152 if (clazz == null) {
153 throw new UnsupportedFileSystemException(
154 "No AbstractFileSystem for scheme: " + uri.getScheme());
155 }
156 return (AbstractFileSystem) newInstance(clazz, uri, conf);
157 }
158
159 /**
160 * Get the statistics for a particular file system.
161 *
162 * @param uri
163 * used as key to lookup STATISTICS_TABLE. Only scheme and authority
164 * part of the uri are used.
165 * @return a statistics object
166 */
167 protected static synchronized Statistics getStatistics(URI uri) {
168 String scheme = uri.getScheme();
169 if (scheme == null) {
170 throw new IllegalArgumentException("Scheme not defined in the uri: "
171 + uri);
172 }
173 URI baseUri = getBaseUri(uri);
174 Statistics result = STATISTICS_TABLE.get(baseUri);
175 if (result == null) {
176 result = new Statistics(scheme);
177 STATISTICS_TABLE.put(baseUri, result);
178 }
179 return result;
180 }
181
182 private static URI getBaseUri(URI uri) {
183 String scheme = uri.getScheme();
184 String authority = uri.getAuthority();
185 String baseUriString = scheme + "://";
186 if (authority != null) {
187 baseUriString = baseUriString + authority;
188 } else {
189 baseUriString = baseUriString + "/";
190 }
191 return URI.create(baseUriString);
192 }
193
194 public static synchronized void clearStatistics() {
195 for(Statistics stat: STATISTICS_TABLE.values()) {
196 stat.reset();
197 }
198 }
199
200 /**
201 * Prints statistics for all file systems.
202 */
203 public static synchronized void printStatistics() {
204 for (Map.Entry<URI, Statistics> pair : STATISTICS_TABLE.entrySet()) {
205 System.out.println(" FileSystem " + pair.getKey().getScheme() + "://"
206 + pair.getKey().getAuthority() + ": " + pair.getValue());
207 }
208 }
209
210 protected static synchronized Map<URI, Statistics> getAllStatistics() {
211 Map<URI, Statistics> statsMap = new HashMap<URI, Statistics>(
212 STATISTICS_TABLE.size());
213 for (Map.Entry<URI, Statistics> pair : STATISTICS_TABLE.entrySet()) {
214 URI key = pair.getKey();
215 Statistics value = pair.getValue();
216 Statistics newStatsObj = new Statistics(value);
217 statsMap.put(URI.create(key.toString()), newStatsObj);
218 }
219 return statsMap;
220 }
221
222 /**
223 * The main factory method for creating a file system. Get a file system for
224 * the URI's scheme and authority. The scheme of the <code>uri</code>
225 * determines a configuration property name,
226 * <tt>fs.AbstractFileSystem.<i>scheme</i>.impl</tt> whose value names the
227 * AbstractFileSystem class.
228 *
229 * The entire URI and conf is passed to the AbstractFileSystem factory method.
230 *
231 * @param uri for the file system to be created.
232 * @param conf which is passed to the file system impl.
233 *
234 * @return file system for the given URI.
235 *
236 * @throws UnsupportedFileSystemException if the file system for
237 * <code>uri</code> is not supported.
238 */
239 public static AbstractFileSystem get(final URI uri, final Configuration conf)
240 throws UnsupportedFileSystemException {
241 return createFileSystem(uri, conf);
242 }
243
244 /**
245 * Constructor to be called by subclasses.
246 *
247 * @param uri for this file system.
248 * @param supportedScheme the scheme supported by the implementor
249 * @param authorityNeeded if true then theURI must have authority, if false
250 * then the URI must have null authority.
251 *
252 * @throws URISyntaxException <code>uri</code> has syntax error
253 */
254 public AbstractFileSystem(final URI uri, final String supportedScheme,
255 final boolean authorityNeeded, final int defaultPort)
256 throws URISyntaxException {
257 myUri = getUri(uri, supportedScheme, authorityNeeded, defaultPort);
258 statistics = getStatistics(uri);
259 }
260
261 /**
262 * Check that the Uri's scheme matches
263 * @param uri
264 * @param supportedScheme
265 */
266 public void checkScheme(URI uri, String supportedScheme) {
267 String scheme = uri.getScheme();
268 if (scheme == null) {
269 throw new HadoopIllegalArgumentException("Uri without scheme: " + uri);
270 }
271 if (!scheme.equals(supportedScheme)) {
272 throw new HadoopIllegalArgumentException("Uri scheme " + uri
273 + " does not match the scheme " + supportedScheme);
274 }
275 }
276
277 /**
278 * Get the URI for the file system based on the given URI. The path, query
279 * part of the given URI is stripped out and default file system port is used
280 * to form the URI.
281 *
282 * @param uri FileSystem URI.
283 * @param authorityNeeded if true authority cannot be null in the URI. If
284 * false authority must be null.
285 * @param defaultPort default port to use if port is not specified in the URI.
286 *
287 * @return URI of the file system
288 *
289 * @throws URISyntaxException <code>uri</code> has syntax error
290 */
291 private URI getUri(URI uri, String supportedScheme,
292 boolean authorityNeeded, int defaultPort) throws URISyntaxException {
293 checkScheme(uri, supportedScheme);
294 // A file system implementation that requires authority must always
295 // specify default port
296 if (defaultPort < 0 && authorityNeeded) {
297 throw new HadoopIllegalArgumentException(
298 "FileSystem implementation error - default port " + defaultPort
299 + " is not valid");
300 }
301 String authority = uri.getAuthority();
302 if (authority == null) {
303 if (authorityNeeded) {
304 throw new HadoopIllegalArgumentException("Uri without authority: " + uri);
305 } else {
306 return new URI(supportedScheme + ":///");
307 }
308 }
309 // authority is non null - AuthorityNeeded may be true or false.
310 int port = uri.getPort();
311 port = (port == -1 ? defaultPort : port);
312 if (port == -1) { // no port supplied and default port is not specified
313 return new URI(supportedScheme, authority, "/", null);
314 }
315 return new URI(supportedScheme + "://" + uri.getHost() + ":" + port);
316 }
317
318 /**
319 * The default port of this file system.
320 *
321 * @return default port of this file system's Uri scheme
322 * A uri with a port of -1 => default port;
323 */
324 public abstract int getUriDefaultPort();
325
326 /**
327 * Returns a URI whose scheme and authority identify this FileSystem.
328 *
329 * @return the uri of this file system.
330 */
331 public URI getUri() {
332 return myUri;
333 }
334
335 /**
336 * Check that a Path belongs to this FileSystem.
337 *
338 * If the path is fully qualified URI, then its scheme and authority
339 * matches that of this file system. Otherwise the path must be
340 * slash-relative name.
341 *
342 * @throws InvalidPathException if the path is invalid
343 */
344 public void checkPath(Path path) {
345 URI uri = path.toUri();
346 String thatScheme = uri.getScheme();
347 String thatAuthority = uri.getAuthority();
348 if (thatScheme == null) {
349 if (thatAuthority == null) {
350 if (path.isUriPathAbsolute()) {
351 return;
352 }
353 throw new InvalidPathException("relative paths not allowed:" +
354 path);
355 } else {
356 throw new InvalidPathException(
357 "Path without scheme with non-null authority:" + path);
358 }
359 }
360 String thisScheme = this.getUri().getScheme();
361 String thisHost = this.getUri().getHost();
362 String thatHost = uri.getHost();
363
364 // Schemes and hosts must match.
365 // Allow for null Authority for file:///
366 if (!thisScheme.equalsIgnoreCase(thatScheme) ||
367 (thisHost != null &&
368 !thisHost.equalsIgnoreCase(thatHost)) ||
369 (thisHost == null && thatHost != null)) {
370 throw new InvalidPathException("Wrong FS: " + path + ", expected: "
371 + this.getUri());
372 }
373
374 // Ports must match, unless this FS instance is using the default port, in
375 // which case the port may be omitted from the given URI
376 int thisPort = this.getUri().getPort();
377 int thatPort = uri.getPort();
378 if (thatPort == -1) { // -1 => defaultPort of Uri scheme
379 thatPort = this.getUriDefaultPort();
380 }
381 if (thisPort != thatPort) {
382 throw new InvalidPathException("Wrong FS: " + path + ", expected: "
383 + this.getUri());
384 }
385 }
386
387 /**
388 * Get the path-part of a pathname. Checks that URI matches this file system
389 * and that the path-part is a valid name.
390 *
391 * @param p path
392 *
393 * @return path-part of the Path p
394 */
395 public String getUriPath(final Path p) {
396 checkPath(p);
397 String s = p.toUri().getPath();
398 if (!isValidName(s)) {
399 throw new InvalidPathException("Path part " + s + " from URI " + p
400 + " is not a valid filename.");
401 }
402 return s;
403 }
404
405 /**
406 * Make the path fully qualified to this file system
407 * @param path
408 * @return the qualified path
409 */
410 public Path makeQualified(Path path) {
411 checkPath(path);
412 return path.makeQualified(this.getUri(), null);
413 }
414
415 /**
416 * Some file systems like LocalFileSystem have an initial workingDir
417 * that is used as the starting workingDir. For other file systems
418 * like HDFS there is no built in notion of an initial workingDir.
419 *
420 * @return the initial workingDir if the file system has such a notion
421 * otherwise return a null.
422 */
423 public Path getInitialWorkingDirectory() {
424 return null;
425 }
426
427 /**
428 * Return the current user's home directory in this file system.
429 * The default implementation returns "/user/$USER/".
430 *
431 * @return current user's home directory.
432 */
433 public Path getHomeDirectory() {
434 return new Path("/user/"+System.getProperty("user.name")).makeQualified(
435 getUri(), null);
436 }
437
438 /**
439 * Return a set of server default configuration values.
440 *
441 * @return server default configuration values
442 *
443 * @throws IOException an I/O error occurred
444 */
445 public abstract FsServerDefaults getServerDefaults() throws IOException;
446
447 /**
448 * Return the fully-qualified path of path f resolving the path
449 * through any internal symlinks or mount point
450 * @param p path to be resolved
451 * @return fully qualified path
452 * @throws FileNotFoundException, AccessControlException, IOException
453 * UnresolvedLinkException if symbolic link on path cannot be resolved
454 * internally
455 */
456 public Path resolvePath(final Path p) throws FileNotFoundException,
457 UnresolvedLinkException, AccessControlException, IOException {
458 checkPath(p);
459 return getFileStatus(p).getPath(); // default impl is to return the path
460 }
461
462 /**
463 * The specification of this method matches that of
464 * {@link FileContext#create(Path, EnumSet, Options.CreateOpts...)} except
465 * that the Path f must be fully qualified and the permission is absolute
466 * (i.e. umask has been applied).
467 */
468 public final FSDataOutputStream create(final Path f,
469 final EnumSet<CreateFlag> createFlag, Options.CreateOpts... opts)
470 throws AccessControlException, FileAlreadyExistsException,
471 FileNotFoundException, ParentNotDirectoryException,
472 UnsupportedFileSystemException, UnresolvedLinkException, IOException {
473 checkPath(f);
474 int bufferSize = -1;
475 short replication = -1;
476 long blockSize = -1;
477 int bytesPerChecksum = -1;
478 ChecksumOpt checksumOpt = null;
479 FsPermission permission = null;
480 Progressable progress = null;
481 Boolean createParent = null;
482
483 for (CreateOpts iOpt : opts) {
484 if (CreateOpts.BlockSize.class.isInstance(iOpt)) {
485 if (blockSize != -1) {
486 throw new HadoopIllegalArgumentException(
487 "BlockSize option is set multiple times");
488 }
489 blockSize = ((CreateOpts.BlockSize) iOpt).getValue();
490 } else if (CreateOpts.BufferSize.class.isInstance(iOpt)) {
491 if (bufferSize != -1) {
492 throw new HadoopIllegalArgumentException(
493 "BufferSize option is set multiple times");
494 }
495 bufferSize = ((CreateOpts.BufferSize) iOpt).getValue();
496 } else if (CreateOpts.ReplicationFactor.class.isInstance(iOpt)) {
497 if (replication != -1) {
498 throw new HadoopIllegalArgumentException(
499 "ReplicationFactor option is set multiple times");
500 }
501 replication = ((CreateOpts.ReplicationFactor) iOpt).getValue();
502 } else if (CreateOpts.BytesPerChecksum.class.isInstance(iOpt)) {
503 if (bytesPerChecksum != -1) {
504 throw new HadoopIllegalArgumentException(
505 "BytesPerChecksum option is set multiple times");
506 }
507 bytesPerChecksum = ((CreateOpts.BytesPerChecksum) iOpt).getValue();
508 } else if (CreateOpts.ChecksumParam.class.isInstance(iOpt)) {
509 if (checksumOpt != null) {
510 throw new HadoopIllegalArgumentException(
511 "CreateChecksumType option is set multiple times");
512 }
513 checksumOpt = ((CreateOpts.ChecksumParam) iOpt).getValue();
514 } else if (CreateOpts.Perms.class.isInstance(iOpt)) {
515 if (permission != null) {
516 throw new HadoopIllegalArgumentException(
517 "Perms option is set multiple times");
518 }
519 permission = ((CreateOpts.Perms) iOpt).getValue();
520 } else if (CreateOpts.Progress.class.isInstance(iOpt)) {
521 if (progress != null) {
522 throw new HadoopIllegalArgumentException(
523 "Progress option is set multiple times");
524 }
525 progress = ((CreateOpts.Progress) iOpt).getValue();
526 } else if (CreateOpts.CreateParent.class.isInstance(iOpt)) {
527 if (createParent != null) {
528 throw new HadoopIllegalArgumentException(
529 "CreateParent option is set multiple times");
530 }
531 createParent = ((CreateOpts.CreateParent) iOpt).getValue();
532 } else {
533 throw new HadoopIllegalArgumentException("Unkown CreateOpts of type " +
534 iOpt.getClass().getName());
535 }
536 }
537 if (permission == null) {
538 throw new HadoopIllegalArgumentException("no permission supplied");
539 }
540
541
542 FsServerDefaults ssDef = getServerDefaults();
543 if (ssDef.getBlockSize() % ssDef.getBytesPerChecksum() != 0) {
544 throw new IOException("Internal error: default blockSize is" +
545 " not a multiple of default bytesPerChecksum ");
546 }
547
548 if (blockSize == -1) {
549 blockSize = ssDef.getBlockSize();
550 }
551
552 // Create a checksum option honoring user input as much as possible.
553 // If bytesPerChecksum is specified, it will override the one set in
554 // checksumOpt. Any missing value will be filled in using the default.
555 ChecksumOpt defaultOpt = new ChecksumOpt(
556 ssDef.getChecksumType(),
557 ssDef.getBytesPerChecksum());
558 checksumOpt = ChecksumOpt.processChecksumOpt(defaultOpt,
559 checksumOpt, bytesPerChecksum);
560
561 if (bufferSize == -1) {
562 bufferSize = ssDef.getFileBufferSize();
563 }
564 if (replication == -1) {
565 replication = ssDef.getReplication();
566 }
567 if (createParent == null) {
568 createParent = false;
569 }
570
571 if (blockSize % bytesPerChecksum != 0) {
572 throw new HadoopIllegalArgumentException(
573 "blockSize should be a multiple of checksumsize");
574 }
575
576 return this.createInternal(f, createFlag, permission, bufferSize,
577 replication, blockSize, progress, checksumOpt, createParent);
578 }
579
580 /**
581 * The specification of this method matches that of
582 * {@link #create(Path, EnumSet, Options.CreateOpts...)} except that the opts
583 * have been declared explicitly.
584 */
585 public abstract FSDataOutputStream createInternal(Path f,
586 EnumSet<CreateFlag> flag, FsPermission absolutePermission,
587 int bufferSize, short replication, long blockSize, Progressable progress,
588 ChecksumOpt checksumOpt, boolean createParent)
589 throws AccessControlException, FileAlreadyExistsException,
590 FileNotFoundException, ParentNotDirectoryException,
591 UnsupportedFileSystemException, UnresolvedLinkException, IOException;
592
593 /**
594 * The specification of this method matches that of
595 * {@link FileContext#mkdir(Path, FsPermission, boolean)} except that the Path
596 * f must be fully qualified and the permission is absolute (i.e.
597 * umask has been applied).
598 */
599 public abstract void mkdir(final Path dir, final FsPermission permission,
600 final boolean createParent) throws AccessControlException,
601 FileAlreadyExistsException, FileNotFoundException,
602 UnresolvedLinkException, IOException;
603
604 /**
605 * The specification of this method matches that of
606 * {@link FileContext#delete(Path, boolean)} except that Path f must be for
607 * this file system.
608 */
609 public abstract boolean delete(final Path f, final boolean recursive)
610 throws AccessControlException, FileNotFoundException,
611 UnresolvedLinkException, IOException;
612
613 /**
614 * The specification of this method matches that of
615 * {@link FileContext#open(Path)} except that Path f must be for this
616 * file system.
617 */
618 public FSDataInputStream open(final Path f) throws AccessControlException,
619 FileNotFoundException, UnresolvedLinkException, IOException {
620 return open(f, getServerDefaults().getFileBufferSize());
621 }
622
623 /**
624 * The specification of this method matches that of
625 * {@link FileContext#open(Path, int)} except that Path f must be for this
626 * file system.
627 */
628 public abstract FSDataInputStream open(final Path f, int bufferSize)
629 throws AccessControlException, FileNotFoundException,
630 UnresolvedLinkException, IOException;
631
632 /**
633 * The specification of this method matches that of
634 * {@link FileContext#setReplication(Path, short)} except that Path f must be
635 * for this file system.
636 */
637 public abstract boolean setReplication(final Path f,
638 final short replication) throws AccessControlException,
639 FileNotFoundException, UnresolvedLinkException, IOException;
640
641 /**
642 * The specification of this method matches that of
643 * {@link FileContext#rename(Path, Path, Options.Rename...)} except that Path
644 * f must be for this file system.
645 */
646 public final void rename(final Path src, final Path dst,
647 final Options.Rename... options) throws AccessControlException,
648 FileAlreadyExistsException, FileNotFoundException,
649 ParentNotDirectoryException, UnresolvedLinkException, IOException {
650 boolean overwrite = false;
651 if (null != options) {
652 for (Rename option : options) {
653 if (option == Rename.OVERWRITE) {
654 overwrite = true;
655 }
656 }
657 }
658 renameInternal(src, dst, overwrite);
659 }
660
661 /**
662 * The specification of this method matches that of
663 * {@link FileContext#rename(Path, Path, Options.Rename...)} except that Path
664 * f must be for this file system and NO OVERWRITE is performed.
665 *
666 * File systems that do not have a built in overwrite need implement only this
667 * method and can take advantage of the default impl of the other
668 * {@link #renameInternal(Path, Path, boolean)}
669 */
670 public abstract void renameInternal(final Path src, final Path dst)
671 throws AccessControlException, FileAlreadyExistsException,
672 FileNotFoundException, ParentNotDirectoryException,
673 UnresolvedLinkException, IOException;
674
675 /**
676 * The specification of this method matches that of
677 * {@link FileContext#rename(Path, Path, Options.Rename...)} except that Path
678 * f must be for this file system.
679 */
680 public void renameInternal(final Path src, final Path dst,
681 boolean overwrite) throws AccessControlException,
682 FileAlreadyExistsException, FileNotFoundException,
683 ParentNotDirectoryException, UnresolvedLinkException, IOException {
684 // Default implementation deals with overwrite in a non-atomic way
685 final FileStatus srcStatus = getFileLinkStatus(src);
686
687 FileStatus dstStatus;
688 try {
689 dstStatus = getFileLinkStatus(dst);
690 } catch (IOException e) {
691 dstStatus = null;
692 }
693 if (dstStatus != null) {
694 if (dst.equals(src)) {
695 throw new FileAlreadyExistsException(
696 "The source "+src+" and destination "+dst+" are the same");
697 }
698 if (srcStatus.isSymlink() && dst.equals(srcStatus.getSymlink())) {
699 throw new FileAlreadyExistsException(
700 "Cannot rename symlink "+src+" to its target "+dst);
701 }
702 // It's OK to rename a file to a symlink and vice versa
703 if (srcStatus.isDirectory() != dstStatus.isDirectory()) {
704 throw new IOException("Source " + src + " and destination " + dst
705 + " must both be directories");
706 }
707 if (!overwrite) {
708 throw new FileAlreadyExistsException("Rename destination " + dst
709 + " already exists.");
710 }
711 // Delete the destination that is a file or an empty directory
712 if (dstStatus.isDirectory()) {
713 RemoteIterator<FileStatus> list = listStatusIterator(dst);
714 if (list != null && list.hasNext()) {
715 throw new IOException(
716 "Rename cannot overwrite non empty destination directory " + dst);
717 }
718 }
719 delete(dst, false);
720 } else {
721 final Path parent = dst.getParent();
722 final FileStatus parentStatus = getFileStatus(parent);
723 if (parentStatus.isFile()) {
724 throw new ParentNotDirectoryException("Rename destination parent "
725 + parent + " is a file.");
726 }
727 }
728 renameInternal(src, dst);
729 }
730
731 /**
732 * Returns true if the file system supports symlinks, false otherwise.
733 * @return true if filesystem supports symlinks
734 */
735 public boolean supportsSymlinks() {
736 return false;
737 }
738
739 /**
740 * The specification of this method matches that of
741 * {@link FileContext#createSymlink(Path, Path, boolean)};
742 */
743 public void createSymlink(final Path target, final Path link,
744 final boolean createParent) throws IOException, UnresolvedLinkException {
745 throw new IOException("File system does not support symlinks");
746 }
747
748 /**
749 * Partially resolves the path. This is used during symlink resolution in
750 * {@link FSLinkResolver}, and differs from the similarly named method
751 * {@link FileContext#getLinkTarget(Path)}.
752 * @throws IOException subclass implementations may throw IOException
753 */
754 public Path getLinkTarget(final Path f) throws IOException {
755 throw new AssertionError("Implementation Error: " + getClass()
756 + " that threw an UnresolvedLinkException, causing this method to be"
757 + " called, needs to override this method.");
758 }
759
760 /**
761 * The specification of this method matches that of
762 * {@link FileContext#setPermission(Path, FsPermission)} except that Path f
763 * must be for this file system.
764 */
765 public abstract void setPermission(final Path f,
766 final FsPermission permission) throws AccessControlException,
767 FileNotFoundException, UnresolvedLinkException, IOException;
768
769 /**
770 * The specification of this method matches that of
771 * {@link FileContext#setOwner(Path, String, String)} except that Path f must
772 * be for this file system.
773 */
774 public abstract void setOwner(final Path f, final String username,
775 final String groupname) throws AccessControlException,
776 FileNotFoundException, UnresolvedLinkException, IOException;
777
778 /**
779 * The specification of this method matches that of
780 * {@link FileContext#setTimes(Path, long, long)} except that Path f must be
781 * for this file system.
782 */
783 public abstract void setTimes(final Path f, final long mtime,
784 final long atime) throws AccessControlException, FileNotFoundException,
785 UnresolvedLinkException, IOException;
786
787 /**
788 * The specification of this method matches that of
789 * {@link FileContext#getFileChecksum(Path)} except that Path f must be for
790 * this file system.
791 */
792 public abstract FileChecksum getFileChecksum(final Path f)
793 throws AccessControlException, FileNotFoundException,
794 UnresolvedLinkException, IOException;
795
796 /**
797 * The specification of this method matches that of
798 * {@link FileContext#getFileStatus(Path)}
799 * except that an UnresolvedLinkException may be thrown if a symlink is
800 * encountered in the path.
801 */
802 public abstract FileStatus getFileStatus(final Path f)
803 throws AccessControlException, FileNotFoundException,
804 UnresolvedLinkException, IOException;
805
806 /**
807 * The specification of this method matches that of
808 * {@link FileContext#getFileLinkStatus(Path)}
809 * except that an UnresolvedLinkException may be thrown if a symlink is
810 * encountered in the path leading up to the final path component.
811 * If the file system does not support symlinks then the behavior is
812 * equivalent to {@link AbstractFileSystem#getFileStatus(Path)}.
813 */
814 public FileStatus getFileLinkStatus(final Path f)
815 throws AccessControlException, FileNotFoundException,
816 UnsupportedFileSystemException, IOException {
817 return getFileStatus(f);
818 }
819
820 /**
821 * The specification of this method matches that of
822 * {@link FileContext#getFileBlockLocations(Path, long, long)} except that
823 * Path f must be for this file system.
824 */
825 public abstract BlockLocation[] getFileBlockLocations(final Path f,
826 final long start, final long len) throws AccessControlException,
827 FileNotFoundException, UnresolvedLinkException, IOException;
828
829 /**
830 * The specification of this method matches that of
831 * {@link FileContext#getFsStatus(Path)} except that Path f must be for this
832 * file system.
833 */
834 public FsStatus getFsStatus(final Path f) throws AccessControlException,
835 FileNotFoundException, UnresolvedLinkException, IOException {
836 // default impl gets FsStatus of root
837 return getFsStatus();
838 }
839
840 /**
841 * The specification of this method matches that of
842 * {@link FileContext#getFsStatus(Path)}.
843 */
844 public abstract FsStatus getFsStatus() throws AccessControlException,
845 FileNotFoundException, IOException;
846
847 /**
848 * The specification of this method matches that of
849 * {@link FileContext#listStatus(Path)} except that Path f must be for this
850 * file system.
851 */
852 public RemoteIterator<FileStatus> listStatusIterator(final Path f)
853 throws AccessControlException, FileNotFoundException,
854 UnresolvedLinkException, IOException {
855 return new RemoteIterator<FileStatus>() {
856 private int i = 0;
857 private FileStatus[] statusList = listStatus(f);
858
859 @Override
860 public boolean hasNext() {
861 return i < statusList.length;
862 }
863
864 @Override
865 public FileStatus next() {
866 if (!hasNext()) {
867 throw new NoSuchElementException();
868 }
869 return statusList[i++];
870 }
871 };
872 }
873
874 /**
875 * The specification of this method matches that of
876 * {@link FileContext#listLocatedStatus(Path)} except that Path f
877 * must be for this file system.
878 */
879 public RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f)
880 throws AccessControlException, FileNotFoundException,
881 UnresolvedLinkException, IOException {
882 return new RemoteIterator<LocatedFileStatus>() {
883 private RemoteIterator<FileStatus> itor = listStatusIterator(f);
884
885 @Override
886 public boolean hasNext() throws IOException {
887 return itor.hasNext();
888 }
889
890 @Override
891 public LocatedFileStatus next() throws IOException {
892 if (!hasNext()) {
893 throw new NoSuchElementException("No more entry in " + f);
894 }
895 FileStatus result = itor.next();
896 BlockLocation[] locs = null;
897 if (result.isFile()) {
898 locs = getFileBlockLocations(
899 result.getPath(), 0, result.getLen());
900 }
901 return new LocatedFileStatus(result, locs);
902 }
903 };
904 }
905
906 /**
907 * The specification of this method matches that of
908 * {@link FileContext.Util#listStatus(Path)} except that Path f must be
909 * for this file system.
910 */
911 public abstract FileStatus[] listStatus(final Path f)
912 throws AccessControlException, FileNotFoundException,
913 UnresolvedLinkException, IOException;
914
915 /**
916 * @return an iterator over the corrupt files under the given path
917 * (may contain duplicates if a file has more than one corrupt block)
918 * @throws IOException
919 */
920 public RemoteIterator<Path> listCorruptFileBlocks(Path path)
921 throws IOException {
922 throw new UnsupportedOperationException(getClass().getCanonicalName() +
923 " does not support" +
924 " listCorruptFileBlocks");
925 }
926
927 /**
928 * The specification of this method matches that of
929 * {@link FileContext#setVerifyChecksum(boolean, Path)} except that Path f
930 * must be for this file system.
931 */
932 public abstract void setVerifyChecksum(final boolean verifyChecksum)
933 throws AccessControlException, IOException;
934
935 /**
936 * Get a canonical name for this file system.
937 * @return a URI string that uniquely identifies this file system
938 */
939 public String getCanonicalServiceName() {
940 return SecurityUtil.buildDTServiceName(getUri(), getUriDefaultPort());
941 }
942
943 /**
944 * Get one or more delegation tokens associated with the filesystem. Normally
945 * a file system returns a single delegation token. A file system that manages
946 * multiple file systems underneath, could return set of delegation tokens for
947 * all the file systems it manages
948 *
949 * @param renewer the account name that is allowed to renew the token.
950 * @return List of delegation tokens.
951 * If delegation tokens not supported then return a list of size zero.
952 * @throws IOException
953 */
954 @InterfaceAudience.LimitedPrivate( { "HDFS", "MapReduce" })
955 public List<Token<?>> getDelegationTokens(String renewer) throws IOException {
956 return new ArrayList<Token<?>>(0);
957 }
958
959 /**
960 * Modifies ACL entries of files and directories. This method can add new ACL
961 * entries or modify the permissions on existing ACL entries. All existing
962 * ACL entries that are not specified in this call are retained without
963 * changes. (Modifications are merged into the current ACL.)
964 *
965 * @param path Path to modify
966 * @param aclSpec List<AclEntry> describing modifications
967 * @throws IOException if an ACL could not be modified
968 */
969 public void modifyAclEntries(Path path, List<AclEntry> aclSpec)
970 throws IOException {
971 throw new UnsupportedOperationException(getClass().getSimpleName()
972 + " doesn't support modifyAclEntries");
973 }
974
975 /**
976 * Removes ACL entries from files and directories. Other ACL entries are
977 * retained.
978 *
979 * @param path Path to modify
980 * @param aclSpec List<AclEntry> describing entries to remove
981 * @throws IOException if an ACL could not be modified
982 */
983 public void removeAclEntries(Path path, List<AclEntry> aclSpec)
984 throws IOException {
985 throw new UnsupportedOperationException(getClass().getSimpleName()
986 + " doesn't support removeAclEntries");
987 }
988
989 /**
990 * Removes all default ACL entries from files and directories.
991 *
992 * @param path Path to modify
993 * @throws IOException if an ACL could not be modified
994 */
995 public void removeDefaultAcl(Path path)
996 throws IOException {
997 throw new UnsupportedOperationException(getClass().getSimpleName()
998 + " doesn't support removeDefaultAcl");
999 }
1000
1001 /**
1002 * Removes all but the base ACL entries of files and directories. The entries
1003 * for user, group, and others are retained for compatibility with permission
1004 * bits.
1005 *
1006 * @param path Path to modify
1007 * @throws IOException if an ACL could not be removed
1008 */
1009 public void removeAcl(Path path)
1010 throws IOException {
1011 throw new UnsupportedOperationException(getClass().getSimpleName()
1012 + " doesn't support removeAcl");
1013 }
1014
1015 /**
1016 * Fully replaces ACL of files and directories, discarding all existing
1017 * entries.
1018 *
1019 * @param path Path to modify
1020 * @param aclSpec List<AclEntry> describing modifications, must include entries
1021 * for user, group, and others for compatibility with permission bits.
1022 * @throws IOException if an ACL could not be modified
1023 */
1024 public void setAcl(Path path, List<AclEntry> aclSpec) throws IOException {
1025 throw new UnsupportedOperationException(getClass().getSimpleName()
1026 + " doesn't support setAcl");
1027 }
1028
1029 /**
1030 * Gets the ACLs of files and directories.
1031 *
1032 * @param path Path to get
1033 * @return RemoteIterator<AclStatus> which returns each AclStatus
1034 * @throws IOException if an ACL could not be read
1035 */
1036 public AclStatus getAclStatus(Path path) throws IOException {
1037 throw new UnsupportedOperationException(getClass().getSimpleName()
1038 + " doesn't support getAclStatus");
1039 }
1040
1041 /**
1042 * Set an xattr of a file or directory.
1043 * The name must be prefixed with user/trusted/security/system and
1044 * followed by ".". For example, "user.attr".
1045 * <p/>
1046 * A regular user can only set an xattr for the "user" namespace.
1047 * The super user can set an xattr of either the "user" or "trusted" namespaces.
1048 * The xattrs of the "security" and "system" namespaces are only used/exposed
1049 * internally by/to the FS impl.
1050 * <p/>
1051 * The access permissions of an xattr in the "user" namespace are
1052 * defined by the file and directory permission bits.
1053 * An xattr can only be set when the logged-in user has the correct permissions.
1054 * If the xattr exists, it will be replaced.
1055 * <p/>
1056 * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
1057 * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
1058 *
1059 * @param path Path to modify
1060 * @param name xattr name.
1061 * @param value xattr value.
1062 * @throws IOException
1063 */
1064 public void setXAttr(Path path, String name, byte[] value)
1065 throws IOException {
1066 setXAttr(path, name, value, EnumSet.of(XAttrSetFlag.CREATE,
1067 XAttrSetFlag.REPLACE));
1068 }
1069
1070 /**
1071 * Set an xattr of a file or directory.
1072 * The name must be prefixed with user/trusted/security/system and
1073 * followed by ".". For example, "user.attr".
1074 * <p/>
1075 * A regular user can only set an xattr for the "user" namespace.
1076 * The super user can set an xattr of either the "user" or "trusted" namespaces.
1077 * The xattrs of the "security" and "system" namespaces are only used/exposed
1078 * internally by/to the FS impl.
1079 * <p/>
1080 * The access permissions of an xattr in the "user" namespace are
1081 * defined by the file and directory permission bits.
1082 * An xattr can only be set when the logged-in user has the correct permissions.
1083 * If the xattr exists, it will be replaced.
1084 * <p/>
1085 * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
1086 * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
1087 *
1088 * @param path Path to modify
1089 * @param name xattr name.
1090 * @param value xattr value.
1091 * @param flag xattr set flag
1092 * @throws IOException
1093 */
1094 public void setXAttr(Path path, String name, byte[] value,
1095 EnumSet<XAttrSetFlag> flag) throws IOException {
1096 throw new UnsupportedOperationException(getClass().getSimpleName()
1097 + " doesn't support setXAttr");
1098 }
1099
1100 /**
1101 * Get an xattr for a file or directory.
1102 * The name must be prefixed with user/trusted/security/system and
1103 * followed by ".". For example, "user.attr".
1104 * <p/>
1105 * A regular user can only get an xattr for the "user" namespace.
1106 * The super user can get an xattr of either the "user" or "trusted" namespaces.
1107 * The xattrs of the "security" and "system" namespaces are only used/exposed
1108 * internally by/to the FS impl.
1109 * <p/>
1110 * An xattr will only be returned when the logged-in user has the correct permissions.
1111 * <p/>
1112 * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
1113 * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
1114 *
1115 * @param path Path to get extended attribute
1116 * @param name xattr name.
1117 * @return byte[] xattr value.
1118 * @throws IOException
1119 */
1120 public byte[] getXAttr(Path path, String name) throws IOException {
1121 throw new UnsupportedOperationException(getClass().getSimpleName()
1122 + " doesn't support getXAttr");
1123 }
1124
1125 /**
1126 * Get all of the xattrs for a file or directory.
1127 * Only those xattrs for which the logged-in user has permissions to view
1128 * are returned.
1129 * <p/>
1130 * A regular user can only get xattrs for the "user" namespace.
1131 * The super user can only get xattrs for "user" and "trusted" namespaces.
1132 * The xattr of "security" and "system" namespaces are only used/exposed
1133 * internally by/to the FS impl.
1134 * <p/>
1135 * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
1136 * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
1137 *
1138 * @param path Path to get extended attributes
1139 * @return Map<String, byte[]> describing the XAttrs of the file or directory
1140 * @throws IOException
1141 */
1142 public Map<String, byte[]> getXAttrs(Path path) throws IOException {
1143 throw new UnsupportedOperationException(getClass().getSimpleName()
1144 + " doesn't support getXAttrs");
1145 }
1146
1147 /**
1148 * Get all of the xattrs for a file or directory.
1149 * Only those xattrs for which the logged-in user has permissions to view
1150 * are returned.
1151 * <p/>
1152 * A regular user can only get xattrs for the "user" namespace.
1153 * The super user can only get xattrs for "user" and "trusted" namespaces.
1154 * The xattr of "security" and "system" namespaces are only used/exposed
1155 * internally by/to the FS impl.
1156 * <p/>
1157 * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
1158 * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
1159 *
1160 * @param path Path to get extended attributes
1161 * @param names XAttr names.
1162 * @return Map<String, byte[]> describing the XAttrs of the file or directory
1163 * @throws IOException
1164 */
1165 public Map<String, byte[]> getXAttrs(Path path, List<String> names)
1166 throws IOException {
1167 throw new UnsupportedOperationException(getClass().getSimpleName()
1168 + " doesn't support getXAttrs");
1169 }
1170
1171 /**
1172 * Get all of the xattr names for a file or directory.
1173 * Only the xattr names for which the logged-in user has permissions to view
1174 * are returned.
1175 * <p/>
1176 * A regular user can only get xattr names for the "user" namespace.
1177 * The super user can only get xattr names for the "user" and "trusted"
1178 * namespaces.
1179 * The xattr names in the "security" and "system" namespaces are only
1180 * used/exposed internally by/to the FS impl.
1181 * <p/>
1182 * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
1183 * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
1184 *
1185 * @param path Path to get extended attributes
1186 * @return Map<String, byte[]> describing the XAttrs of the file or directory
1187 * @throws IOException
1188 */
1189 public List<String> listXAttrs(Path path)
1190 throws IOException {
1191 throw new UnsupportedOperationException(getClass().getSimpleName()
1192 + " doesn't support listXAttrs");
1193 }
1194
1195 /**
1196 * Remove an xattr of a file or directory.
1197 * The name must be prefixed with user/trusted/security/system and
1198 * followed by ".". For example, "user.attr".
1199 * <p/>
1200 * A regular user can only remove an xattr for the "user" namespace.
1201 * The super user can remove an xattr of either the "user" or "trusted" namespaces.
1202 * The xattrs of the "security" and "system" namespaces are only used/exposed
1203 * internally by/to the FS impl.
1204 * <p/>
1205 * The access permissions of an xattr in the "user" namespace are
1206 * defined by the file and directory permission bits.
1207 * An xattr can only be set when the logged-in user has the correct permissions.
1208 * If the xattr exists, it will be replaced.
1209 * <p/>
1210 * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
1211 * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
1212 *
1213 * @param path Path to remove extended attribute
1214 * @param name xattr name
1215 * @throws IOException
1216 */
1217 public void removeXAttr(Path path, String name) throws IOException {
1218 throw new UnsupportedOperationException(getClass().getSimpleName()
1219 + " doesn't support removeXAttr");
1220 }
1221
1222 @Override //Object
1223 public int hashCode() {
1224 return myUri.hashCode();
1225 }
1226
1227 @Override //Object
1228 public boolean equals(Object other) {
1229 if (other == null || !(other instanceof AbstractFileSystem)) {
1230 return false;
1231 }
1232 return myUri.equals(((AbstractFileSystem) other).myUri);
1233 }
1234 }