001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.fs;
019
020import java.io.FileNotFoundException;
021import java.io.IOException;
022import java.lang.reflect.Constructor;
023import java.net.URI;
024import java.net.URISyntaxException;
025import java.util.ArrayList;
026import java.util.EnumSet;
027import java.util.HashMap;
028import java.util.List;
029import java.util.Map;
030import java.util.NoSuchElementException;
031import java.util.StringTokenizer;
032import java.util.concurrent.ConcurrentHashMap;
033
034import org.apache.commons.logging.Log;
035import org.apache.commons.logging.LogFactory;
036import org.apache.hadoop.HadoopIllegalArgumentException;
037import org.apache.hadoop.classification.InterfaceAudience;
038import org.apache.hadoop.classification.InterfaceStability;
039import org.apache.hadoop.conf.Configuration;
040import org.apache.hadoop.fs.FileSystem.Statistics;
041import org.apache.hadoop.fs.Options.ChecksumOpt;
042import org.apache.hadoop.fs.Options.CreateOpts;
043import org.apache.hadoop.fs.Options.Rename;
044import org.apache.hadoop.fs.permission.AclEntry;
045import org.apache.hadoop.fs.permission.AclStatus;
046import org.apache.hadoop.fs.permission.FsAction;
047import org.apache.hadoop.fs.permission.FsPermission;
048import org.apache.hadoop.security.AccessControlException;
049import org.apache.hadoop.security.SecurityUtil;
050import org.apache.hadoop.security.token.Token;
051import org.apache.hadoop.util.Progressable;
052
053import com.google.common.annotations.VisibleForTesting;
054
055/**
056 * This class provides an interface for implementors of a Hadoop file system
057 * (analogous to the VFS of Unix). Applications do not access this class;
058 * instead they access files across all file systems using {@link FileContext}.
059 * 
060 * Pathnames passed to AbstractFileSystem can be fully qualified URI that
061 * matches the "this" file system (ie same scheme and authority) 
062 * or a Slash-relative name that is assumed to be relative
063 * to the root of the "this" file system .
064 */
065@InterfaceAudience.Public
066@InterfaceStability.Evolving /*Evolving for a release,to be changed to Stable */
067public abstract class AbstractFileSystem {
068  static final Log LOG = LogFactory.getLog(AbstractFileSystem.class);
069
070  /** Recording statistics per a file system class. */
071  private static final Map<URI, Statistics> 
072      STATISTICS_TABLE = new HashMap<URI, Statistics>();
073  
074  /** Cache of constructors for each file system class. */
075  private static final Map<Class<?>, Constructor<?>> CONSTRUCTOR_CACHE = 
076    new ConcurrentHashMap<Class<?>, Constructor<?>>();
077  
078  private static final Class<?>[] URI_CONFIG_ARGS = 
079    new Class[]{URI.class, Configuration.class};
080  
081  /** The statistics for this file system. */
082  protected Statistics statistics;
083
084  @VisibleForTesting
085  static final String NO_ABSTRACT_FS_ERROR = "No AbstractFileSystem configured for scheme";
086  
087  private final URI myUri;
088  
089  public Statistics getStatistics() {
090    return statistics;
091  }
092  
093  /**
094   * Returns true if the specified string is considered valid in the path part
095   * of a URI by this file system.  The default implementation enforces the rules
096   * of HDFS, but subclasses may override this method to implement specific
097   * validation rules for specific file systems.
098   * 
099   * @param src String source filename to check, path part of the URI
100   * @return boolean true if the specified string is considered valid
101   */
102  public boolean isValidName(String src) {
103    // Prohibit ".." "." and anything containing ":"
104    StringTokenizer tokens = new StringTokenizer(src, Path.SEPARATOR);
105    while(tokens.hasMoreTokens()) {
106      String element = tokens.nextToken();
107      if (element.equals("..") ||
108          element.equals(".")  ||
109          (element.indexOf(":") >= 0)) {
110        return false;
111      }
112    }
113    return true;
114  }
115  
116  /** 
117   * Create an object for the given class and initialize it from conf.
118   * @param theClass class of which an object is created
119   * @param conf Configuration
120   * @return a new object
121   */
122  @SuppressWarnings("unchecked")
123  static <T> T newInstance(Class<T> theClass,
124    URI uri, Configuration conf) {
125    T result;
126    try {
127      Constructor<T> meth = (Constructor<T>) CONSTRUCTOR_CACHE.get(theClass);
128      if (meth == null) {
129        meth = theClass.getDeclaredConstructor(URI_CONFIG_ARGS);
130        meth.setAccessible(true);
131        CONSTRUCTOR_CACHE.put(theClass, meth);
132      }
133      result = meth.newInstance(uri, conf);
134    } catch (Exception e) {
135      throw new RuntimeException(e);
136    }
137    return result;
138  }
139  
140  /**
141   * Create a file system instance for the specified uri using the conf. The
142   * conf is used to find the class name that implements the file system. The
143   * conf is also passed to the file system for its configuration.
144   *
145   * @param uri URI of the file system
146   * @param conf Configuration for the file system
147   * 
148   * @return Returns the file system for the given URI
149   *
150   * @throws UnsupportedFileSystemException file system for <code>uri</code> is
151   *           not found
152   */
153  public static AbstractFileSystem createFileSystem(URI uri, Configuration conf)
154      throws UnsupportedFileSystemException {
155    final String fsImplConf = String.format("fs.AbstractFileSystem.%s.impl",
156        uri.getScheme());
157
158    Class<?> clazz = conf.getClass(fsImplConf, null);
159    if (clazz == null) {
160      throw new UnsupportedFileSystemException(String.format(
161          "%s=null: %s: %s",
162          fsImplConf, NO_ABSTRACT_FS_ERROR, uri.getScheme()));
163    }
164    return (AbstractFileSystem) newInstance(clazz, uri, conf);
165  }
166
167  /**
168   * Get the statistics for a particular file system.
169   * 
170   * @param uri
171   *          used as key to lookup STATISTICS_TABLE. Only scheme and authority
172   *          part of the uri are used.
173   * @return a statistics object
174   */
175  protected static synchronized Statistics getStatistics(URI uri) {
176    String scheme = uri.getScheme();
177    if (scheme == null) {
178      throw new IllegalArgumentException("Scheme not defined in the uri: "
179          + uri);
180    }
181    URI baseUri = getBaseUri(uri);
182    Statistics result = STATISTICS_TABLE.get(baseUri);
183    if (result == null) {
184      result = new Statistics(scheme);
185      STATISTICS_TABLE.put(baseUri, result);
186    }
187    return result;
188  }
189  
190  private static URI getBaseUri(URI uri) {
191    String scheme = uri.getScheme();
192    String authority = uri.getAuthority();
193    String baseUriString = scheme + "://";
194    if (authority != null) {
195      baseUriString = baseUriString + authority;
196    } else {
197      baseUriString = baseUriString + "/";
198    }
199    return URI.create(baseUriString);
200  }
201  
202  public static synchronized void clearStatistics() {
203    for(Statistics stat: STATISTICS_TABLE.values()) {
204      stat.reset();
205    }
206  }
207
208  /**
209   * Prints statistics for all file systems.
210   */
211  public static synchronized void printStatistics() {
212    for (Map.Entry<URI, Statistics> pair : STATISTICS_TABLE.entrySet()) {
213      System.out.println("  FileSystem " + pair.getKey().getScheme() + "://"
214          + pair.getKey().getAuthority() + ": " + pair.getValue());
215    }
216  }
217  
218  protected static synchronized Map<URI, Statistics> getAllStatistics() {
219    Map<URI, Statistics> statsMap = new HashMap<URI, Statistics>(
220        STATISTICS_TABLE.size());
221    for (Map.Entry<URI, Statistics> pair : STATISTICS_TABLE.entrySet()) {
222      URI key = pair.getKey();
223      Statistics value = pair.getValue();
224      Statistics newStatsObj = new Statistics(value);
225      statsMap.put(URI.create(key.toString()), newStatsObj);
226    }
227    return statsMap;
228  }
229
230  /**
231   * The main factory method for creating a file system. Get a file system for
232   * the URI's scheme and authority. The scheme of the <code>uri</code>
233   * determines a configuration property name,
234   * <tt>fs.AbstractFileSystem.<i>scheme</i>.impl</tt> whose value names the
235   * AbstractFileSystem class.
236   * 
237   * The entire URI and conf is passed to the AbstractFileSystem factory method.
238   * 
239   * @param uri for the file system to be created.
240   * @param conf which is passed to the file system impl.
241   * 
242   * @return file system for the given URI.
243   * 
244   * @throws UnsupportedFileSystemException if the file system for
245   *           <code>uri</code> is not supported.
246   */
247  public static AbstractFileSystem get(final URI uri, final Configuration conf)
248      throws UnsupportedFileSystemException {
249    return createFileSystem(uri, conf);
250  }
251
252  /**
253   * Constructor to be called by subclasses.
254   * 
255   * @param uri for this file system.
256   * @param supportedScheme the scheme supported by the implementor
257   * @param authorityNeeded if true then theURI must have authority, if false
258   *          then the URI must have null authority.
259   *
260   * @throws URISyntaxException <code>uri</code> has syntax error
261   */
262  public AbstractFileSystem(final URI uri, final String supportedScheme,
263      final boolean authorityNeeded, final int defaultPort)
264      throws URISyntaxException {
265    myUri = getUri(uri, supportedScheme, authorityNeeded, defaultPort);
266    statistics = getStatistics(uri); 
267  }
268  
269  /**
270   * Check that the Uri's scheme matches
271   * @param uri
272   * @param supportedScheme
273   */
274  public void checkScheme(URI uri, String supportedScheme) {
275    String scheme = uri.getScheme();
276    if (scheme == null) {
277      throw new HadoopIllegalArgumentException("Uri without scheme: " + uri);
278    }
279    if (!scheme.equals(supportedScheme)) {
280      throw new HadoopIllegalArgumentException("Uri scheme " + uri
281          + " does not match the scheme " + supportedScheme);
282    }
283  }
284
285  /**
286   * Get the URI for the file system based on the given URI. The path, query
287   * part of the given URI is stripped out and default file system port is used
288   * to form the URI.
289   * 
290   * @param uri FileSystem URI.
291   * @param authorityNeeded if true authority cannot be null in the URI. If
292   *          false authority must be null.
293   * @param defaultPort default port to use if port is not specified in the URI.
294   * 
295   * @return URI of the file system
296   * 
297   * @throws URISyntaxException <code>uri</code> has syntax error
298   */
299  private URI getUri(URI uri, String supportedScheme,
300      boolean authorityNeeded, int defaultPort) throws URISyntaxException {
301    checkScheme(uri, supportedScheme);
302    // A file system implementation that requires authority must always
303    // specify default port
304    if (defaultPort < 0 && authorityNeeded) {
305      throw new HadoopIllegalArgumentException(
306          "FileSystem implementation error -  default port " + defaultPort
307              + " is not valid");
308    }
309    String authority = uri.getAuthority();
310    if (authority == null) {
311       if (authorityNeeded) {
312         throw new HadoopIllegalArgumentException("Uri without authority: " + uri);
313       } else {
314         return new URI(supportedScheme + ":///");
315       }   
316    }
317    // authority is non null  - AuthorityNeeded may be true or false.
318    int port = uri.getPort();
319    port = (port == -1 ? defaultPort : port);
320    if (port == -1) { // no port supplied and default port is not specified
321      return new URI(supportedScheme, authority, "/", null);
322    }
323    return new URI(supportedScheme + "://" + uri.getHost() + ":" + port);
324  }
325  
326  /**
327   * The default port of this file system.
328   * 
329   * @return default port of this file system's Uri scheme
330   *         A uri with a port of -1 => default port;
331   */
332  public abstract int getUriDefaultPort();
333
334  /**
335   * Returns a URI whose scheme and authority identify this FileSystem.
336   * 
337   * @return the uri of this file system.
338   */
339  public URI getUri() {
340    return myUri;
341  }
342  
343  /**
344   * Check that a Path belongs to this FileSystem.
345   * 
346   * If the path is fully qualified URI, then its scheme and authority
347   * matches that of this file system. Otherwise the path must be 
348   * slash-relative name.
349   * 
350   * @throws InvalidPathException if the path is invalid
351   */
352  public void checkPath(Path path) {
353    URI uri = path.toUri();
354    String thatScheme = uri.getScheme();
355    String thatAuthority = uri.getAuthority();
356    if (thatScheme == null) {
357      if (thatAuthority == null) {
358        if (path.isUriPathAbsolute()) {
359          return;
360        }
361        throw new InvalidPathException("relative paths not allowed:" + 
362            path);
363      } else {
364        throw new InvalidPathException(
365            "Path without scheme with non-null authority:" + path);
366      }
367    }
368    String thisScheme = this.getUri().getScheme();
369    String thisHost = this.getUri().getHost();
370    String thatHost = uri.getHost();
371    
372    // Schemes and hosts must match.
373    // Allow for null Authority for file:///
374    if (!thisScheme.equalsIgnoreCase(thatScheme) ||
375       (thisHost != null && 
376            !thisHost.equalsIgnoreCase(thatHost)) ||
377       (thisHost == null && thatHost != null)) {
378      throw new InvalidPathException("Wrong FS: " + path + ", expected: "
379          + this.getUri());
380    }
381    
382    // Ports must match, unless this FS instance is using the default port, in
383    // which case the port may be omitted from the given URI
384    int thisPort = this.getUri().getPort();
385    int thatPort = uri.getPort();
386    if (thatPort == -1) { // -1 => defaultPort of Uri scheme
387      thatPort = this.getUriDefaultPort();
388    }
389    if (thisPort != thatPort) {
390      throw new InvalidPathException("Wrong FS: " + path + ", expected: "
391          + this.getUri());
392    }
393  }
394  
395  /**
396   * Get the path-part of a pathname. Checks that URI matches this file system
397   * and that the path-part is a valid name.
398   * 
399   * @param p path
400   * 
401   * @return path-part of the Path p
402   */
403  public String getUriPath(final Path p) {
404    checkPath(p);
405    String s = p.toUri().getPath();
406    if (!isValidName(s)) {
407      throw new InvalidPathException("Path part " + s + " from URI " + p
408          + " is not a valid filename.");
409    }
410    return s;
411  }
412  
413  /**
414   * Make the path fully qualified to this file system
415   * @param path
416   * @return the qualified path
417   */
418  public Path makeQualified(Path path) {
419    checkPath(path);
420    return path.makeQualified(this.getUri(), null);
421  }
422  
423  /**
424   * Some file systems like LocalFileSystem have an initial workingDir
425   * that is used as the starting workingDir. For other file systems
426   * like HDFS there is no built in notion of an initial workingDir.
427   * 
428   * @return the initial workingDir if the file system has such a notion
429   *         otherwise return a null.
430   */
431  public Path getInitialWorkingDirectory() {
432    return null;
433  }
434  
435  /** 
436   * Return the current user's home directory in this file system.
437   * The default implementation returns "/user/$USER/".
438   * 
439   * @return current user's home directory.
440   */
441  public Path getHomeDirectory() {
442    return new Path("/user/"+System.getProperty("user.name")).makeQualified(
443                                                                getUri(), null);
444  }
445  
446  /**
447   * Return a set of server default configuration values.
448   * 
449   * @return server default configuration values
450   * 
451   * @throws IOException an I/O error occurred
452   */
453  public abstract FsServerDefaults getServerDefaults() throws IOException; 
454
455  /**
456   * Return the fully-qualified path of path f resolving the path
457   * through any internal symlinks or mount point
458   * @param p path to be resolved
459   * @return fully qualified path 
460   * @throws FileNotFoundException, AccessControlException, IOException
461   *         UnresolvedLinkException if symbolic link on path cannot be resolved
462   *          internally
463   */
464   public Path resolvePath(final Path p) throws FileNotFoundException,
465           UnresolvedLinkException, AccessControlException, IOException {
466     checkPath(p);
467     return getFileStatus(p).getPath(); // default impl is to return the path
468   }
469  
470  /**
471   * The specification of this method matches that of
472   * {@link FileContext#create(Path, EnumSet, Options.CreateOpts...)} except
473   * that the Path f must be fully qualified and the permission is absolute
474   * (i.e. umask has been applied).
475   */
476  public final FSDataOutputStream create(final Path f,
477      final EnumSet<CreateFlag> createFlag, Options.CreateOpts... opts)
478      throws AccessControlException, FileAlreadyExistsException,
479      FileNotFoundException, ParentNotDirectoryException,
480      UnsupportedFileSystemException, UnresolvedLinkException, IOException {
481    checkPath(f);
482    int bufferSize = -1;
483    short replication = -1;
484    long blockSize = -1;
485    int bytesPerChecksum = -1;
486    ChecksumOpt checksumOpt = null;
487    FsPermission permission = null;
488    Progressable progress = null;
489    Boolean createParent = null;
490 
491    for (CreateOpts iOpt : opts) {
492      if (CreateOpts.BlockSize.class.isInstance(iOpt)) {
493        if (blockSize != -1) {
494          throw new HadoopIllegalArgumentException(
495              "BlockSize option is set multiple times");
496        }
497        blockSize = ((CreateOpts.BlockSize) iOpt).getValue();
498      } else if (CreateOpts.BufferSize.class.isInstance(iOpt)) {
499        if (bufferSize != -1) {
500          throw new HadoopIllegalArgumentException(
501              "BufferSize option is set multiple times");
502        }
503        bufferSize = ((CreateOpts.BufferSize) iOpt).getValue();
504      } else if (CreateOpts.ReplicationFactor.class.isInstance(iOpt)) {
505        if (replication != -1) {
506          throw new HadoopIllegalArgumentException(
507              "ReplicationFactor option is set multiple times");
508        }
509        replication = ((CreateOpts.ReplicationFactor) iOpt).getValue();
510      } else if (CreateOpts.BytesPerChecksum.class.isInstance(iOpt)) {
511        if (bytesPerChecksum != -1) {
512          throw new HadoopIllegalArgumentException(
513              "BytesPerChecksum option is set multiple times");
514        }
515        bytesPerChecksum = ((CreateOpts.BytesPerChecksum) iOpt).getValue();
516      } else if (CreateOpts.ChecksumParam.class.isInstance(iOpt)) {
517        if (checksumOpt != null) {
518          throw new  HadoopIllegalArgumentException(
519              "CreateChecksumType option is set multiple times");
520        }
521        checksumOpt = ((CreateOpts.ChecksumParam) iOpt).getValue();
522      } else if (CreateOpts.Perms.class.isInstance(iOpt)) {
523        if (permission != null) {
524          throw new HadoopIllegalArgumentException(
525              "Perms option is set multiple times");
526        }
527        permission = ((CreateOpts.Perms) iOpt).getValue();
528      } else if (CreateOpts.Progress.class.isInstance(iOpt)) {
529        if (progress != null) {
530          throw new HadoopIllegalArgumentException(
531              "Progress option is set multiple times");
532        }
533        progress = ((CreateOpts.Progress) iOpt).getValue();
534      } else if (CreateOpts.CreateParent.class.isInstance(iOpt)) {
535        if (createParent != null) {
536          throw new HadoopIllegalArgumentException(
537              "CreateParent option is set multiple times");
538        }
539        createParent = ((CreateOpts.CreateParent) iOpt).getValue();
540      } else {
541        throw new HadoopIllegalArgumentException("Unkown CreateOpts of type " +
542            iOpt.getClass().getName());
543      }
544    }
545    if (permission == null) {
546      throw new HadoopIllegalArgumentException("no permission supplied");
547    }
548
549
550    FsServerDefaults ssDef = getServerDefaults();
551    if (ssDef.getBlockSize() % ssDef.getBytesPerChecksum() != 0) {
552      throw new IOException("Internal error: default blockSize is" + 
553          " not a multiple of default bytesPerChecksum ");
554    }
555    
556    if (blockSize == -1) {
557      blockSize = ssDef.getBlockSize();
558    }
559
560    // Create a checksum option honoring user input as much as possible.
561    // If bytesPerChecksum is specified, it will override the one set in
562    // checksumOpt. Any missing value will be filled in using the default.
563    ChecksumOpt defaultOpt = new ChecksumOpt(
564        ssDef.getChecksumType(),
565        ssDef.getBytesPerChecksum());
566    checksumOpt = ChecksumOpt.processChecksumOpt(defaultOpt,
567        checksumOpt, bytesPerChecksum);
568
569    if (bufferSize == -1) {
570      bufferSize = ssDef.getFileBufferSize();
571    }
572    if (replication == -1) {
573      replication = ssDef.getReplication();
574    }
575    if (createParent == null) {
576      createParent = false;
577    }
578
579    if (blockSize % bytesPerChecksum != 0) {
580      throw new HadoopIllegalArgumentException(
581             "blockSize should be a multiple of checksumsize");
582    }
583
584    return this.createInternal(f, createFlag, permission, bufferSize,
585      replication, blockSize, progress, checksumOpt, createParent);
586  }
587
588  /**
589   * The specification of this method matches that of
590   * {@link #create(Path, EnumSet, Options.CreateOpts...)} except that the opts
591   * have been declared explicitly.
592   */
593  public abstract FSDataOutputStream createInternal(Path f,
594      EnumSet<CreateFlag> flag, FsPermission absolutePermission,
595      int bufferSize, short replication, long blockSize, Progressable progress,
596      ChecksumOpt checksumOpt, boolean createParent)
597      throws AccessControlException, FileAlreadyExistsException,
598      FileNotFoundException, ParentNotDirectoryException,
599      UnsupportedFileSystemException, UnresolvedLinkException, IOException;
600
601  /**
602   * The specification of this method matches that of
603   * {@link FileContext#mkdir(Path, FsPermission, boolean)} except that the Path
604   * f must be fully qualified and the permission is absolute (i.e. 
605   * umask has been applied).
606   */
607  public abstract void mkdir(final Path dir, final FsPermission permission,
608      final boolean createParent) throws AccessControlException,
609      FileAlreadyExistsException, FileNotFoundException,
610      UnresolvedLinkException, IOException;
611
612  /**
613   * The specification of this method matches that of
614   * {@link FileContext#delete(Path, boolean)} except that Path f must be for
615   * this file system.
616   */
617  public abstract boolean delete(final Path f, final boolean recursive)
618      throws AccessControlException, FileNotFoundException,
619      UnresolvedLinkException, IOException;
620
621  /**
622   * The specification of this method matches that of
623   * {@link FileContext#open(Path)} except that Path f must be for this
624   * file system.
625   */
626  public FSDataInputStream open(final Path f) throws AccessControlException,
627      FileNotFoundException, UnresolvedLinkException, IOException {
628    return open(f, getServerDefaults().getFileBufferSize());
629  }
630
631  /**
632   * The specification of this method matches that of
633   * {@link FileContext#open(Path, int)} except that Path f must be for this
634   * file system.
635   */
636  public abstract FSDataInputStream open(final Path f, int bufferSize)
637      throws AccessControlException, FileNotFoundException,
638      UnresolvedLinkException, IOException;
639
640  /**
641   * The specification of this method matches that of
642   * {@link FileContext#truncate(Path, long)} except that Path f must be for
643   * this file system.
644   */
645  public boolean truncate(Path f, long newLength)
646      throws AccessControlException, FileNotFoundException,
647      UnresolvedLinkException, IOException {
648    throw new UnsupportedOperationException(getClass().getSimpleName()
649        + " doesn't support truncate");
650  }
651
652  /**
653   * The specification of this method matches that of
654   * {@link FileContext#setReplication(Path, short)} except that Path f must be
655   * for this file system.
656   */
657  public abstract boolean setReplication(final Path f,
658      final short replication) throws AccessControlException,
659      FileNotFoundException, UnresolvedLinkException, IOException;
660
661  /**
662   * The specification of this method matches that of
663   * {@link FileContext#rename(Path, Path, Options.Rename...)} except that Path
664   * f must be for this file system.
665   */
666  public final void rename(final Path src, final Path dst,
667      final Options.Rename... options) throws AccessControlException,
668      FileAlreadyExistsException, FileNotFoundException,
669      ParentNotDirectoryException, UnresolvedLinkException, IOException {
670    boolean overwrite = false;
671    if (null != options) {
672      for (Rename option : options) {
673        if (option == Rename.OVERWRITE) {
674          overwrite = true;
675        }
676      }
677    }
678    renameInternal(src, dst, overwrite);
679  }
680  
681  /**
682   * The specification of this method matches that of
683   * {@link FileContext#rename(Path, Path, Options.Rename...)} except that Path
684   * f must be for this file system and NO OVERWRITE is performed.
685   * 
686   * File systems that do not have a built in overwrite need implement only this
687   * method and can take advantage of the default impl of the other
688   * {@link #renameInternal(Path, Path, boolean)}
689   */
690  public abstract void renameInternal(final Path src, final Path dst)
691      throws AccessControlException, FileAlreadyExistsException,
692      FileNotFoundException, ParentNotDirectoryException,
693      UnresolvedLinkException, IOException;
694  
695  /**
696   * The specification of this method matches that of
697   * {@link FileContext#rename(Path, Path, Options.Rename...)} except that Path
698   * f must be for this file system.
699   */
700  public void renameInternal(final Path src, final Path dst,
701      boolean overwrite) throws AccessControlException,
702      FileAlreadyExistsException, FileNotFoundException,
703      ParentNotDirectoryException, UnresolvedLinkException, IOException {
704    // Default implementation deals with overwrite in a non-atomic way
705    final FileStatus srcStatus = getFileLinkStatus(src);
706
707    FileStatus dstStatus;
708    try {
709      dstStatus = getFileLinkStatus(dst);
710    } catch (IOException e) {
711      dstStatus = null;
712    }
713    if (dstStatus != null) {
714      if (dst.equals(src)) {
715        throw new FileAlreadyExistsException(
716            "The source "+src+" and destination "+dst+" are the same");
717      }
718      if (srcStatus.isSymlink() && dst.equals(srcStatus.getSymlink())) {
719        throw new FileAlreadyExistsException(
720            "Cannot rename symlink "+src+" to its target "+dst);
721      }
722      // It's OK to rename a file to a symlink and vice versa
723      if (srcStatus.isDirectory() != dstStatus.isDirectory()) {
724        throw new IOException("Source " + src + " and destination " + dst
725            + " must both be directories");
726      }
727      if (!overwrite) {
728        throw new FileAlreadyExistsException("Rename destination " + dst
729            + " already exists.");
730      }
731      // Delete the destination that is a file or an empty directory
732      if (dstStatus.isDirectory()) {
733        RemoteIterator<FileStatus> list = listStatusIterator(dst);
734        if (list != null && list.hasNext()) {
735          throw new IOException(
736              "Rename cannot overwrite non empty destination directory " + dst);
737        }
738      }
739      delete(dst, false);
740    } else {
741      final Path parent = dst.getParent();
742      final FileStatus parentStatus = getFileStatus(parent);
743      if (parentStatus.isFile()) {
744        throw new ParentNotDirectoryException("Rename destination parent "
745            + parent + " is a file.");
746      }
747    }
748    renameInternal(src, dst);
749  }
750  
751  /**
752   * Returns true if the file system supports symlinks, false otherwise.
753   * @return true if filesystem supports symlinks
754   */
755  public boolean supportsSymlinks() {
756    return false;
757  }
758  
759  /**
760   * The specification of this method matches that of  
761   * {@link FileContext#createSymlink(Path, Path, boolean)};
762   */
763  public void createSymlink(final Path target, final Path link,
764      final boolean createParent) throws IOException, UnresolvedLinkException {
765    throw new IOException("File system does not support symlinks");    
766  }
767
768  /**
769   * Partially resolves the path. This is used during symlink resolution in
770   * {@link FSLinkResolver}, and differs from the similarly named method
771   * {@link FileContext#getLinkTarget(Path)}.
772   * @throws IOException subclass implementations may throw IOException 
773   */
774  public Path getLinkTarget(final Path f) throws IOException {
775    throw new AssertionError("Implementation Error: " + getClass()
776        + " that threw an UnresolvedLinkException, causing this method to be"
777        + " called, needs to override this method.");
778  }
779    
780  /**
781   * The specification of this method matches that of
782   * {@link FileContext#setPermission(Path, FsPermission)} except that Path f
783   * must be for this file system.
784   */
785  public abstract void setPermission(final Path f,
786      final FsPermission permission) throws AccessControlException,
787      FileNotFoundException, UnresolvedLinkException, IOException;
788
789  /**
790   * The specification of this method matches that of
791   * {@link FileContext#setOwner(Path, String, String)} except that Path f must
792   * be for this file system.
793   */
794  public abstract void setOwner(final Path f, final String username,
795      final String groupname) throws AccessControlException,
796      FileNotFoundException, UnresolvedLinkException, IOException;
797
798  /**
799   * The specification of this method matches that of
800   * {@link FileContext#setTimes(Path, long, long)} except that Path f must be
801   * for this file system.
802   */
803  public abstract void setTimes(final Path f, final long mtime,
804    final long atime) throws AccessControlException, FileNotFoundException,
805      UnresolvedLinkException, IOException;
806
807  /**
808   * The specification of this method matches that of
809   * {@link FileContext#getFileChecksum(Path)} except that Path f must be for
810   * this file system.
811   */
812  public abstract FileChecksum getFileChecksum(final Path f)
813      throws AccessControlException, FileNotFoundException,
814      UnresolvedLinkException, IOException;
815  
816  /**
817   * The specification of this method matches that of
818   * {@link FileContext#getFileStatus(Path)} 
819   * except that an UnresolvedLinkException may be thrown if a symlink is 
820   * encountered in the path.
821   */
822  public abstract FileStatus getFileStatus(final Path f)
823      throws AccessControlException, FileNotFoundException,
824      UnresolvedLinkException, IOException;
825
826  /**
827   * The specification of this method matches that of
828   * {@link FileContext#access(Path, FsAction)}
829   * except that an UnresolvedLinkException may be thrown if a symlink is
830   * encountered in the path.
831   */
832  @InterfaceAudience.LimitedPrivate({"HDFS", "Hive"})
833  public void access(Path path, FsAction mode) throws AccessControlException,
834      FileNotFoundException, UnresolvedLinkException, IOException {
835    FileSystem.checkAccessPermissions(this.getFileStatus(path), mode);
836  }
837
838  /**
839   * The specification of this method matches that of
840   * {@link FileContext#getFileLinkStatus(Path)}
841   * except that an UnresolvedLinkException may be thrown if a symlink is  
842   * encountered in the path leading up to the final path component.
843   * If the file system does not support symlinks then the behavior is
844   * equivalent to {@link AbstractFileSystem#getFileStatus(Path)}.
845   */
846  public FileStatus getFileLinkStatus(final Path f)
847      throws AccessControlException, FileNotFoundException,
848      UnsupportedFileSystemException, IOException {
849    return getFileStatus(f);
850  }
851
852  /**
853   * The specification of this method matches that of
854   * {@link FileContext#getFileBlockLocations(Path, long, long)} except that
855   * Path f must be for this file system.
856   */
857  public abstract BlockLocation[] getFileBlockLocations(final Path f,
858      final long start, final long len) throws AccessControlException,
859      FileNotFoundException, UnresolvedLinkException, IOException;
860
861  /**
862   * The specification of this method matches that of
863   * {@link FileContext#getFsStatus(Path)} except that Path f must be for this
864   * file system.
865   */
866  public FsStatus getFsStatus(final Path f) throws AccessControlException,
867      FileNotFoundException, UnresolvedLinkException, IOException {
868    // default impl gets FsStatus of root
869    return getFsStatus();
870  }
871  
872  /**
873   * The specification of this method matches that of
874   * {@link FileContext#getFsStatus(Path)}.
875   */
876  public abstract FsStatus getFsStatus() throws AccessControlException,
877      FileNotFoundException, IOException;
878
879  /**
880   * The specification of this method matches that of
881   * {@link FileContext#listStatus(Path)} except that Path f must be for this
882   * file system.
883   */
884  public RemoteIterator<FileStatus> listStatusIterator(final Path f)
885      throws AccessControlException, FileNotFoundException,
886      UnresolvedLinkException, IOException {
887    return new RemoteIterator<FileStatus>() {
888      private int i = 0;
889      private FileStatus[] statusList = listStatus(f);
890      
891      @Override
892      public boolean hasNext() {
893        return i < statusList.length;
894      }
895      
896      @Override
897      public FileStatus next() {
898        if (!hasNext()) {
899          throw new NoSuchElementException();
900        }
901        return statusList[i++];
902      }
903    };
904  }
905
906  /**
907   * The specification of this method matches that of
908   * {@link FileContext#listLocatedStatus(Path)} except that Path f 
909   * must be for this file system.
910   */
911  public RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f)
912      throws AccessControlException, FileNotFoundException,
913      UnresolvedLinkException, IOException {
914    return new RemoteIterator<LocatedFileStatus>() {
915      private RemoteIterator<FileStatus> itor = listStatusIterator(f);
916      
917      @Override
918      public boolean hasNext() throws IOException {
919        return itor.hasNext();
920      }
921      
922      @Override
923      public LocatedFileStatus next() throws IOException {
924        if (!hasNext()) {
925          throw new NoSuchElementException("No more entry in " + f);
926        }
927        FileStatus result = itor.next();
928        BlockLocation[] locs = null;
929        if (result.isFile()) {
930          locs = getFileBlockLocations(
931              result.getPath(), 0, result.getLen());
932        }
933        return new LocatedFileStatus(result, locs);
934      }
935    };
936  }
937
938  /**
939   * The specification of this method matches that of
940   * {@link FileContext.Util#listStatus(Path)} except that Path f must be 
941   * for this file system.
942   */
943  public abstract FileStatus[] listStatus(final Path f)
944      throws AccessControlException, FileNotFoundException,
945      UnresolvedLinkException, IOException;
946
947  /**
948   * @return an iterator over the corrupt files under the given path
949   * (may contain duplicates if a file has more than one corrupt block)
950   * @throws IOException
951   */
952  public RemoteIterator<Path> listCorruptFileBlocks(Path path)
953    throws IOException {
954    throw new UnsupportedOperationException(getClass().getCanonicalName() +
955                                            " does not support" +
956                                            " listCorruptFileBlocks");
957  }
958
959  /**
960   * The specification of this method matches that of
961   * {@link FileContext#setVerifyChecksum(boolean, Path)} except that Path f
962   * must be for this file system.
963   */
964  public abstract void setVerifyChecksum(final boolean verifyChecksum)
965      throws AccessControlException, IOException;
966  
967  /**
968   * Get a canonical name for this file system.
969   * @return a URI string that uniquely identifies this file system
970   */
971  public String getCanonicalServiceName() {
972    return SecurityUtil.buildDTServiceName(getUri(), getUriDefaultPort());
973  }
974  
975  /**
976   * Get one or more delegation tokens associated with the filesystem. Normally
977   * a file system returns a single delegation token. A file system that manages
978   * multiple file systems underneath, could return set of delegation tokens for
979   * all the file systems it manages
980   * 
981   * @param renewer the account name that is allowed to renew the token.
982   * @return List of delegation tokens.
983   *   If delegation tokens not supported then return a list of size zero.
984   * @throws IOException
985   */
986  @InterfaceAudience.LimitedPrivate( { "HDFS", "MapReduce" })
987  public List<Token<?>> getDelegationTokens(String renewer) throws IOException {
988    return new ArrayList<Token<?>>(0);
989  }
990
991  /**
992   * Modifies ACL entries of files and directories.  This method can add new ACL
993   * entries or modify the permissions on existing ACL entries.  All existing
994   * ACL entries that are not specified in this call are retained without
995   * changes.  (Modifications are merged into the current ACL.)
996   *
997   * @param path Path to modify
998   * @param aclSpec List<AclEntry> describing modifications
999   * @throws IOException if an ACL could not be modified
1000   */
1001  public void modifyAclEntries(Path path, List<AclEntry> aclSpec)
1002      throws IOException {
1003    throw new UnsupportedOperationException(getClass().getSimpleName()
1004        + " doesn't support modifyAclEntries");
1005  }
1006
1007  /**
1008   * Removes ACL entries from files and directories.  Other ACL entries are
1009   * retained.
1010   *
1011   * @param path Path to modify
1012   * @param aclSpec List<AclEntry> describing entries to remove
1013   * @throws IOException if an ACL could not be modified
1014   */
1015  public void removeAclEntries(Path path, List<AclEntry> aclSpec)
1016      throws IOException {
1017    throw new UnsupportedOperationException(getClass().getSimpleName()
1018        + " doesn't support removeAclEntries");
1019  }
1020
1021  /**
1022   * Removes all default ACL entries from files and directories.
1023   *
1024   * @param path Path to modify
1025   * @throws IOException if an ACL could not be modified
1026   */
1027  public void removeDefaultAcl(Path path)
1028      throws IOException {
1029    throw new UnsupportedOperationException(getClass().getSimpleName()
1030        + " doesn't support removeDefaultAcl");
1031  }
1032
1033  /**
1034   * Removes all but the base ACL entries of files and directories.  The entries
1035   * for user, group, and others are retained for compatibility with permission
1036   * bits.
1037   *
1038   * @param path Path to modify
1039   * @throws IOException if an ACL could not be removed
1040   */
1041  public void removeAcl(Path path)
1042      throws IOException {
1043    throw new UnsupportedOperationException(getClass().getSimpleName()
1044        + " doesn't support removeAcl");
1045  }
1046
1047  /**
1048   * Fully replaces ACL of files and directories, discarding all existing
1049   * entries.
1050   *
1051   * @param path Path to modify
1052   * @param aclSpec List<AclEntry> describing modifications, must include entries
1053   *   for user, group, and others for compatibility with permission bits.
1054   * @throws IOException if an ACL could not be modified
1055   */
1056  public void setAcl(Path path, List<AclEntry> aclSpec) throws IOException {
1057    throw new UnsupportedOperationException(getClass().getSimpleName()
1058        + " doesn't support setAcl");
1059  }
1060
1061  /**
1062   * Gets the ACLs of files and directories.
1063   *
1064   * @param path Path to get
1065   * @return RemoteIterator<AclStatus> which returns each AclStatus
1066   * @throws IOException if an ACL could not be read
1067   */
1068  public AclStatus getAclStatus(Path path) throws IOException {
1069    throw new UnsupportedOperationException(getClass().getSimpleName()
1070        + " doesn't support getAclStatus");
1071  }
1072
1073  /**
1074   * Set an xattr of a file or directory.
1075   * The name must be prefixed with the namespace followed by ".". For example,
1076   * "user.attr".
1077   * <p/>
1078   * Refer to the HDFS extended attributes user documentation for details.
1079   *
1080   * @param path Path to modify
1081   * @param name xattr name.
1082   * @param value xattr value.
1083   * @throws IOException
1084   */
1085  public void setXAttr(Path path, String name, byte[] value)
1086      throws IOException {
1087    setXAttr(path, name, value, EnumSet.of(XAttrSetFlag.CREATE,
1088        XAttrSetFlag.REPLACE));
1089  }
1090
1091  /**
1092   * Set an xattr of a file or directory.
1093   * The name must be prefixed with the namespace followed by ".". For example,
1094   * "user.attr".
1095   * <p/>
1096   * Refer to the HDFS extended attributes user documentation for details.
1097   *
1098   * @param path Path to modify
1099   * @param name xattr name.
1100   * @param value xattr value.
1101   * @param flag xattr set flag
1102   * @throws IOException
1103   */
1104  public void setXAttr(Path path, String name, byte[] value,
1105      EnumSet<XAttrSetFlag> flag) throws IOException {
1106    throw new UnsupportedOperationException(getClass().getSimpleName()
1107        + " doesn't support setXAttr");
1108  }
1109
1110  /**
1111   * Get an xattr for a file or directory.
1112   * The name must be prefixed with the namespace followed by ".". For example,
1113   * "user.attr".
1114   * <p/>
1115   * Refer to the HDFS extended attributes user documentation for details.
1116   *
1117   * @param path Path to get extended attribute
1118   * @param name xattr name.
1119   * @return byte[] xattr value.
1120   * @throws IOException
1121   */
1122  public byte[] getXAttr(Path path, String name) throws IOException {
1123    throw new UnsupportedOperationException(getClass().getSimpleName()
1124        + " doesn't support getXAttr");
1125  }
1126
1127  /**
1128   * Get all of the xattrs for a file or directory.
1129   * Only those xattrs for which the logged-in user has permissions to view
1130   * are returned.
1131   * <p/>
1132   * Refer to the HDFS extended attributes user documentation for details.
1133   *
1134   * @param path Path to get extended attributes
1135   * @return Map<String, byte[]> describing the XAttrs of the file or directory
1136   * @throws IOException
1137   */
1138  public Map<String, byte[]> getXAttrs(Path path) throws IOException {
1139    throw new UnsupportedOperationException(getClass().getSimpleName()
1140        + " doesn't support getXAttrs");
1141  }
1142
1143  /**
1144   * Get all of the xattrs for a file or directory.
1145   * Only those xattrs for which the logged-in user has permissions to view
1146   * are returned.
1147   * <p/>
1148   * Refer to the HDFS extended attributes user documentation for details.
1149   *
1150   * @param path Path to get extended attributes
1151   * @param names XAttr names.
1152   * @return Map<String, byte[]> describing the XAttrs of the file or directory
1153   * @throws IOException
1154   */
1155  public Map<String, byte[]> getXAttrs(Path path, List<String> names)
1156      throws IOException {
1157    throw new UnsupportedOperationException(getClass().getSimpleName()
1158        + " doesn't support getXAttrs");
1159  }
1160
1161  /**
1162   * Get all of the xattr names for a file or directory.
1163   * Only the xattr names for which the logged-in user has permissions to view
1164   * are returned.
1165   * <p/>
1166   * Refer to the HDFS extended attributes user documentation for details.
1167   *
1168   * @param path Path to get extended attributes
1169   * @return Map<String, byte[]> describing the XAttrs of the file or directory
1170   * @throws IOException
1171   */
1172  public List<String> listXAttrs(Path path)
1173          throws IOException {
1174    throw new UnsupportedOperationException(getClass().getSimpleName()
1175            + " doesn't support listXAttrs");
1176  }
1177
1178  /**
1179   * Remove an xattr of a file or directory.
1180   * The name must be prefixed with the namespace followed by ".". For example,
1181   * "user.attr".
1182   * <p/>
1183   * Refer to the HDFS extended attributes user documentation for details.
1184   *
1185   * @param path Path to remove extended attribute
1186   * @param name xattr name
1187   * @throws IOException
1188   */
1189  public void removeXAttr(Path path, String name) throws IOException {
1190    throw new UnsupportedOperationException(getClass().getSimpleName()
1191        + " doesn't support removeXAttr");
1192  }
1193
1194  @Override //Object
1195  public int hashCode() {
1196    return myUri.hashCode();
1197  }
1198  
1199  @Override //Object
1200  public boolean equals(Object other) {
1201    if (other == null || !(other instanceof AbstractFileSystem)) {
1202      return false;
1203    }
1204    return myUri.equals(((AbstractFileSystem) other).myUri);
1205  }
1206}