org.apache.gobblin.util.filesystem

Class ThrottledFileSystem

java.lang.Object

org.apache.hadoop.conf.Configured

org.apache.hadoop.fs.FileSystem

org.apache.gobblin.util.filesystem.FileSystemInstrumentation

org.apache.gobblin.util.filesystem.ThrottledFileSystem

All Implemented Interfaces:

Closeable, AutoCloseable, Decorator, org.apache.hadoop.conf.Configurable

Direct Known Subclasses:

RateControlledFileSystem

public class ThrottledFileSystem
extends FileSystemInstrumentation

A FileSystemInstrumentation for throttling calls to the underlying FileSystem using the input Limiter.

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	ThrottledFileSystem.Factory<S extends ScopeType<S>> Factory for ThrottledFileSystem.

Nested classes/interfaces inherited from class org.apache.hadoop.fs.FileSystem

org.apache.hadoop.fs.FileSystem.Statistics

Field Summary

Fields

Modifier and Type	Field and Description
protected org.apache.hadoop.conf.Configuration	conf
static int	LISTING_FILES_PER_PERMIT Listing operations will use 1 permit per this many listed elements.
protected String	replacementScheme
protected org.apache.hadoop.fs.FileSystem	underlyingFs
protected String	underlyingScheme

Fields inherited from class org.apache.gobblin.util.filesystem.FileSystemInstrumentation

closed

Fields inherited from class org.apache.hadoop.fs.FileSystem

DEFAULT_FS, FS_DEFAULT_NAME_KEY, LOG, SHUTDOWN_HOOK_PRIORITY, statistics

Constructor Summary

Constructors

Constructor and Description
ThrottledFileSystem(org.apache.hadoop.fs.FileSystem fs, Limiter limiter, String serviceName)

Method Summary

Modifier and Type	Method and Description
org.apache.hadoop.security.token.Token<?>[]	addDelegationTokens(String renewer, org.apache.hadoop.security.Credentials credentials)
org.apache.hadoop.fs.FSDataOutputStream	append(org.apache.hadoop.fs.Path f)
org.apache.hadoop.fs.FSDataOutputStream	append(org.apache.hadoop.fs.Path f, int bufferSize)
org.apache.hadoop.fs.FSDataOutputStream	append(org.apache.hadoop.fs.Path path, int bufferSize, org.apache.hadoop.util.Progressable progress)
boolean	cancelDeleteOnExit(org.apache.hadoop.fs.Path f)
void	close()
void	completeLocalOutput(org.apache.hadoop.fs.Path fsOutputFile, org.apache.hadoop.fs.Path tmpLocalFile)
void	concat(org.apache.hadoop.fs.Path trg, org.apache.hadoop.fs.Path[] psrcs)
void	copyFromLocalFile(boolean delSrc, boolean overwrite, org.apache.hadoop.fs.Path[] srcs, org.apache.hadoop.fs.Path dst)
void	copyFromLocalFile(boolean delSrc, boolean overwrite, org.apache.hadoop.fs.Path src, org.apache.hadoop.fs.Path dst)
void	copyFromLocalFile(boolean delSrc, org.apache.hadoop.fs.Path src, org.apache.hadoop.fs.Path dst)
void	copyFromLocalFile(org.apache.hadoop.fs.Path src, org.apache.hadoop.fs.Path dst)
void	copyToLocalFile(boolean delSrc, org.apache.hadoop.fs.Path src, org.apache.hadoop.fs.Path dst)
void	copyToLocalFile(boolean delSrc, org.apache.hadoop.fs.Path src, org.apache.hadoop.fs.Path dst, boolean useRawLocalFileSystem)
void	copyToLocalFile(org.apache.hadoop.fs.Path src, org.apache.hadoop.fs.Path dst)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f, boolean overwrite)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f, boolean overwrite, int bufferSize)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f, boolean overwrite, int bufferSize, org.apache.hadoop.util.Progressable progress)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f, boolean overwrite, int bufferSize, short replication, long blockSize)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f, boolean overwrite, int bufferSize, short replication, long blockSize, org.apache.hadoop.util.Progressable progress)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path path, org.apache.hadoop.fs.permission.FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, org.apache.hadoop.util.Progressable progress)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f, org.apache.hadoop.fs.permission.FsPermission permission, EnumSet<org.apache.hadoop.fs.CreateFlag> flags, int bufferSize, short replication, long blockSize, org.apache.hadoop.util.Progressable progress)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f, org.apache.hadoop.fs.permission.FsPermission permission, EnumSet<org.apache.hadoop.fs.CreateFlag> flags, int bufferSize, short replication, long blockSize, org.apache.hadoop.util.Progressable progress, org.apache.hadoop.fs.Options.ChecksumOpt checksumOpt)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f, org.apache.hadoop.util.Progressable progress)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f, short replication)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f, short replication, org.apache.hadoop.util.Progressable progress)
boolean	createNewFile(org.apache.hadoop.fs.Path f)
org.apache.hadoop.fs.FSDataOutputStream	createNonRecursive(org.apache.hadoop.fs.Path f, boolean overwrite, int bufferSize, short replication, long blockSize, org.apache.hadoop.util.Progressable progress)
org.apache.hadoop.fs.FSDataOutputStream	createNonRecursive(org.apache.hadoop.fs.Path f, org.apache.hadoop.fs.permission.FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, org.apache.hadoop.util.Progressable progress)
org.apache.hadoop.fs.FSDataOutputStream	createNonRecursive(org.apache.hadoop.fs.Path f, org.apache.hadoop.fs.permission.FsPermission permission, EnumSet<org.apache.hadoop.fs.CreateFlag> flags, int bufferSize, short replication, long blockSize, org.apache.hadoop.util.Progressable progress)
org.apache.hadoop.fs.Path	createSnapshot(org.apache.hadoop.fs.Path path, String snapshotName)
void	createSymlink(org.apache.hadoop.fs.Path target, org.apache.hadoop.fs.Path link, boolean createParent)
boolean	delete(org.apache.hadoop.fs.Path path)
boolean	delete(org.apache.hadoop.fs.Path path, boolean recursive)
boolean	deleteOnExit(org.apache.hadoop.fs.Path f)
void	deleteSnapshot(org.apache.hadoop.fs.Path path, String snapshotName)
boolean	exists(org.apache.hadoop.fs.Path path)
long	getBlockSize(org.apache.hadoop.fs.Path f)
String	getCanonicalServiceName()
org.apache.hadoop.fs.FileSystem[]	getChildFileSystems()
org.apache.hadoop.conf.Configuration	getConf()
org.apache.hadoop.fs.ContentSummary	getContentSummary(org.apache.hadoop.fs.Path f)
Object	getDecoratedObject()
long	getDefaultBlockSize()
long	getDefaultBlockSize(org.apache.hadoop.fs.Path f)
short	getDefaultReplication()
short	getDefaultReplication(org.apache.hadoop.fs.Path path)
org.apache.hadoop.security.token.Token<?>	getDelegationToken(String renewer)
org.apache.hadoop.fs.BlockLocation[]	getFileBlockLocations(org.apache.hadoop.fs.FileStatus file, long start, long len)
org.apache.hadoop.fs.BlockLocation[]	getFileBlockLocations(org.apache.hadoop.fs.Path p, long start, long len)
org.apache.hadoop.fs.FileChecksum	getFileChecksum(org.apache.hadoop.fs.Path f)
org.apache.hadoop.fs.FileStatus	getFileLinkStatus(org.apache.hadoop.fs.Path f)
org.apache.hadoop.fs.FileStatus	getFileStatus(org.apache.hadoop.fs.Path path)
org.apache.hadoop.fs.Path	getHomeDirectory()
long	getLength(org.apache.hadoop.fs.Path f)
org.apache.hadoop.fs.Path	getLinkTarget(org.apache.hadoop.fs.Path f)
String	getName()
protected Limiter	getRateLimiter()
short	getReplication(org.apache.hadoop.fs.Path src)
String	getScheme()
org.apache.hadoop.fs.FsServerDefaults	getServerDefaults()
org.apache.hadoop.fs.FsServerDefaults	getServerDefaults(org.apache.hadoop.fs.Path p)
String	getServiceName()
org.apache.hadoop.fs.FsStatus	getStatus()
org.apache.hadoop.fs.FsStatus	getStatus(org.apache.hadoop.fs.Path p)
URI	getUri()
long	getUsed()
org.apache.hadoop.fs.Path	getWorkingDirectory()
org.apache.hadoop.fs.FileStatus[]	globStatus(org.apache.hadoop.fs.Path pathPattern)
org.apache.hadoop.fs.FileStatus[]	globStatus(org.apache.hadoop.fs.Path pathPattern, org.apache.hadoop.fs.PathFilter filter)
void	initialize(URI uri, org.apache.hadoop.conf.Configuration conf)
boolean	isDirectory(org.apache.hadoop.fs.Path f)
boolean	isFile(org.apache.hadoop.fs.Path f)
org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.Path>	listCorruptFileBlocks(org.apache.hadoop.fs.Path path)
org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.LocatedFileStatus>	listFiles(org.apache.hadoop.fs.Path f, boolean recursive)
org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.LocatedFileStatus>	listLocatedStatus(org.apache.hadoop.fs.Path f)
org.apache.hadoop.fs.FileStatus[]	listStatus(org.apache.hadoop.fs.Path path)
org.apache.hadoop.fs.FileStatus[]	listStatus(org.apache.hadoop.fs.Path[] files)
org.apache.hadoop.fs.FileStatus[]	listStatus(org.apache.hadoop.fs.Path[] files, org.apache.hadoop.fs.PathFilter filter)
org.apache.hadoop.fs.FileStatus[]	listStatus(org.apache.hadoop.fs.Path path, org.apache.hadoop.fs.PathFilter filter)
org.apache.hadoop.fs.Path	makeQualified(org.apache.hadoop.fs.Path path)
boolean	mkdirs(org.apache.hadoop.fs.Path f)
boolean	mkdirs(org.apache.hadoop.fs.Path path, org.apache.hadoop.fs.permission.FsPermission permission)
void	moveFromLocalFile(org.apache.hadoop.fs.Path[] srcs, org.apache.hadoop.fs.Path dst)
void	moveFromLocalFile(org.apache.hadoop.fs.Path src, org.apache.hadoop.fs.Path dst)
void	moveToLocalFile(org.apache.hadoop.fs.Path src, org.apache.hadoop.fs.Path dst)
org.apache.hadoop.fs.FSDataInputStream	open(org.apache.hadoop.fs.Path f)
org.apache.hadoop.fs.FSDataInputStream	open(org.apache.hadoop.fs.Path path, int bufferSize)
boolean	rename(org.apache.hadoop.fs.Path path0, org.apache.hadoop.fs.Path path1)
void	renameSnapshot(org.apache.hadoop.fs.Path path, String snapshotOldName, String snapshotNewName)
org.apache.hadoop.fs.Path	resolvePath(org.apache.hadoop.fs.Path p)
void	setConf(org.apache.hadoop.conf.Configuration conf)
void	setOwner(org.apache.hadoop.fs.Path p, String username, String groupname)
void	setPermission(org.apache.hadoop.fs.Path p, org.apache.hadoop.fs.permission.FsPermission permission)
boolean	setReplication(org.apache.hadoop.fs.Path src, short replication)
void	setTimes(org.apache.hadoop.fs.Path p, long mtime, long atime)
void	setVerifyChecksum(boolean verifyChecksum)
void	setWorkingDirectory(org.apache.hadoop.fs.Path new_dir)
void	setWriteChecksum(boolean writeChecksum)
org.apache.hadoop.fs.Path	startLocalOutput(org.apache.hadoop.fs.Path fsOutputFile, org.apache.hadoop.fs.Path tmpLocalFile)
boolean	supportsSymlinks()

Methods inherited from class org.apache.gobblin.util.filesystem.FileSystemInstrumentation

onClose

Methods inherited from class org.apache.hadoop.fs.FileSystem

areSymlinksEnabled, canonicalizeUri, checkPath, clearStatistics, closeAll, closeAllForUGI, create, createSnapshot, enableSymlinks, fixRelativePart, get, get, get, getAllStatistics, getCanonicalUri, getDefaultPort, getDefaultUri, getFileSystemClass, getFSofPath, getInitialWorkingDirectory, getLocal, getNamed, getStatistics, getStatistics, listLocatedStatus, mkdirs, newInstance, newInstance, newInstance, newInstanceLocal, primitiveCreate, primitiveMkdir, primitiveMkdir, printStatistics, processDeleteOnExit, rename, resolveLink, setDefaultUri, setDefaultUri

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

LISTING_FILES_PER_PERMIT

public static final int LISTING_FILES_PER_PERMIT

Listing operations will use 1 permit per this many listed elements.

See Also:

Constant Field Values

replacementScheme

protected String replacementScheme

underlyingScheme

protected String underlyingScheme

conf

protected org.apache.hadoop.conf.Configuration conf

underlyingFs

protected org.apache.hadoop.fs.FileSystem underlyingFs

Constructor Detail

ThrottledFileSystem

public ThrottledFileSystem(org.apache.hadoop.fs.FileSystem fs,
                           Limiter limiter,
                           String serviceName)

Method Detail

delete

public boolean delete(org.apache.hadoop.fs.Path path)
               throws IOException

Throws:

IOException

delete

public boolean delete(org.apache.hadoop.fs.Path path,
                      boolean recursive)
               throws IOException

Throws:

IOException

exists

public boolean exists(org.apache.hadoop.fs.Path path)
               throws IOException

Throws:

IOException

getFileStatus

public org.apache.hadoop.fs.FileStatus getFileStatus(org.apache.hadoop.fs.Path path)
                                              throws IOException

Throws:

IOException

globStatus

public org.apache.hadoop.fs.FileStatus[] globStatus(org.apache.hadoop.fs.Path pathPattern)
                                             throws IOException

Throws:

IOException

listStatus

public org.apache.hadoop.fs.FileStatus[] listStatus(org.apache.hadoop.fs.Path path)
                                             throws IOException

Throws:

IOException

listStatus

public org.apache.hadoop.fs.FileStatus[] listStatus(org.apache.hadoop.fs.Path path,
                                                    org.apache.hadoop.fs.PathFilter filter)
                                             throws IOException

Throws:

IOException

mkdirs

public boolean mkdirs(org.apache.hadoop.fs.Path path,
                      org.apache.hadoop.fs.permission.FsPermission permission)
               throws IOException

Throws:

IOException

rename

public boolean rename(org.apache.hadoop.fs.Path path0,
                      org.apache.hadoop.fs.Path path1)
               throws IOException

Throws:

IOException

append

public org.apache.hadoop.fs.FSDataOutputStream append(org.apache.hadoop.fs.Path path,
                                                      int bufferSize,
                                                      org.apache.hadoop.util.Progressable progress)
                                               throws IOException

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path path,
                                                      org.apache.hadoop.fs.permission.FsPermission permission,
                                                      boolean overwrite,
                                                      int bufferSize,
                                                      short replication,
                                                      long blockSize,
                                                      org.apache.hadoop.util.Progressable progress)
                                               throws IOException

Throws:

IOException

open

public org.apache.hadoop.fs.FSDataInputStream open(org.apache.hadoop.fs.Path path,
                                                   int bufferSize)
                                            throws IOException

Throws:

IOException

getRateLimiter

protected Limiter getRateLimiter()

getServiceName

public String getServiceName()

close

public void close()
           throws IOException

Specified by:

close in interface Closeable

Specified by:

close in interface AutoCloseable

Overrides:

close in class FileSystemInstrumentation

Throws:

IOException

getDecoratedObject

public Object getDecoratedObject()

Specified by:

getDecoratedObject in interface Decorator

getScheme

public String getScheme()

Overrides:

getScheme in class org.apache.hadoop.fs.FileSystem

initialize

public void initialize(URI uri,
                       org.apache.hadoop.conf.Configuration conf)
                throws IOException

Overrides:

initialize in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

setConf

public void setConf(org.apache.hadoop.conf.Configuration conf)

Specified by:

setConf in interface org.apache.hadoop.conf.Configurable

Overrides:

setConf in class org.apache.hadoop.conf.Configured

getUri

public URI getUri()

Specified by:

getUri in class org.apache.hadoop.fs.FileSystem

getFileLinkStatus

public org.apache.hadoop.fs.FileStatus getFileLinkStatus(org.apache.hadoop.fs.Path f)
                                                  throws IOException

Overrides:

getFileLinkStatus in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getStatus

public org.apache.hadoop.fs.FsStatus getStatus()
                                        throws IOException

Overrides:

getStatus in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

append

public org.apache.hadoop.fs.FSDataOutputStream append(org.apache.hadoop.fs.Path f)
                                               throws IOException

Overrides:

append in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getReplication

public short getReplication(org.apache.hadoop.fs.Path src)
                     throws IOException

Overrides:

getReplication in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

setWriteChecksum

public void setWriteChecksum(boolean writeChecksum)

Overrides:

setWriteChecksum in class org.apache.hadoop.fs.FileSystem

getFileChecksum

public org.apache.hadoop.fs.FileChecksum getFileChecksum(org.apache.hadoop.fs.Path f)
                                                  throws IOException

Overrides:

getFileChecksum in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

isDirectory

public boolean isDirectory(org.apache.hadoop.fs.Path f)
                    throws IOException

Overrides:

isDirectory in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

createSymlink

public void createSymlink(org.apache.hadoop.fs.Path target,
                          org.apache.hadoop.fs.Path link,
                          boolean createParent)
                   throws IOException

Overrides:

createSymlink in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

createSnapshot

public org.apache.hadoop.fs.Path createSnapshot(org.apache.hadoop.fs.Path path,
                                                String snapshotName)
                                         throws IOException

Overrides:

createSnapshot in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                                      org.apache.hadoop.fs.permission.FsPermission permission,
                                                      EnumSet<org.apache.hadoop.fs.CreateFlag> flags,
                                                      int bufferSize,
                                                      short replication,
                                                      long blockSize,
                                                      org.apache.hadoop.util.Progressable progress,
                                                      org.apache.hadoop.fs.Options.ChecksumOpt checksumOpt)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

resolvePath

public org.apache.hadoop.fs.Path resolvePath(org.apache.hadoop.fs.Path p)
                                      throws IOException

Overrides:

resolvePath in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getUsed

public long getUsed()
             throws IOException

Overrides:

getUsed in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getConf

public org.apache.hadoop.conf.Configuration getConf()

Specified by:

getConf in interface org.apache.hadoop.conf.Configurable

Overrides:

getConf in class org.apache.hadoop.conf.Configured

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                                      org.apache.hadoop.util.Progressable progress)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

isFile

public boolean isFile(org.apache.hadoop.fs.Path f)
               throws IOException

Overrides:

isFile in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getWorkingDirectory

public org.apache.hadoop.fs.Path getWorkingDirectory()

Specified by:

getWorkingDirectory in class org.apache.hadoop.fs.FileSystem

getServerDefaults

public org.apache.hadoop.fs.FsServerDefaults getServerDefaults()
                                                        throws IOException

Overrides:

getServerDefaults in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

copyToLocalFile

public void copyToLocalFile(boolean delSrc,
                            org.apache.hadoop.fs.Path src,
                            org.apache.hadoop.fs.Path dst,
                            boolean useRawLocalFileSystem)
                     throws IOException

Overrides:

copyToLocalFile in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

setWorkingDirectory

public void setWorkingDirectory(org.apache.hadoop.fs.Path new_dir)

Specified by:

setWorkingDirectory in class org.apache.hadoop.fs.FileSystem

getName

public String getName()

Overrides:

getName in class org.apache.hadoop.fs.FileSystem

createNewFile

public boolean createNewFile(org.apache.hadoop.fs.Path f)
                      throws IOException

Overrides:

createNewFile in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

listStatus

public org.apache.hadoop.fs.FileStatus[] listStatus(org.apache.hadoop.fs.Path[] files)
                                             throws IOException

Overrides:

listStatus in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getLinkTarget

public org.apache.hadoop.fs.Path getLinkTarget(org.apache.hadoop.fs.Path f)
                                        throws IOException

Overrides:

getLinkTarget in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

copyToLocalFile

public void copyToLocalFile(boolean delSrc,
                            org.apache.hadoop.fs.Path src,
                            org.apache.hadoop.fs.Path dst)
                     throws IOException

Overrides:

copyToLocalFile in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getDefaultReplication

public short getDefaultReplication()

Overrides:

getDefaultReplication in class org.apache.hadoop.fs.FileSystem

getDelegationToken

public org.apache.hadoop.security.token.Token<?> getDelegationToken(String renewer)
                                                             throws IOException

Overrides:

getDelegationToken in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getServerDefaults

public org.apache.hadoop.fs.FsServerDefaults getServerDefaults(org.apache.hadoop.fs.Path p)
                                                        throws IOException

Overrides:

getServerDefaults in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                                      short replication)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getFileBlockLocations

public org.apache.hadoop.fs.BlockLocation[] getFileBlockLocations(org.apache.hadoop.fs.Path p,
                                                                  long start,
                                                                  long len)
                                                           throws IOException

Overrides:

getFileBlockLocations in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

concat

public void concat(org.apache.hadoop.fs.Path trg,
                   org.apache.hadoop.fs.Path[] psrcs)
            throws IOException

Overrides:

concat in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getHomeDirectory

public org.apache.hadoop.fs.Path getHomeDirectory()

Overrides:

getHomeDirectory in class org.apache.hadoop.fs.FileSystem

supportsSymlinks

public boolean supportsSymlinks()

Overrides:

supportsSymlinks in class org.apache.hadoop.fs.FileSystem

listLocatedStatus

public org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.LocatedFileStatus> listLocatedStatus(org.apache.hadoop.fs.Path f)
                                                                                              throws FileNotFoundException,
                                                                                                     IOException

Overrides:

listLocatedStatus in class org.apache.hadoop.fs.FileSystem

Throws:

FileNotFoundException

IOException

listCorruptFileBlocks

public org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.Path> listCorruptFileBlocks(org.apache.hadoop.fs.Path path)
                                                                                     throws IOException

Overrides:

listCorruptFileBlocks in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                                      short replication,
                                                      org.apache.hadoop.util.Progressable progress)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

setReplication

public boolean setReplication(org.apache.hadoop.fs.Path src,
                              short replication)
                       throws IOException

Overrides:

setReplication in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

makeQualified

public org.apache.hadoop.fs.Path makeQualified(org.apache.hadoop.fs.Path path)

Overrides:

makeQualified in class org.apache.hadoop.fs.FileSystem

createNonRecursive

public org.apache.hadoop.fs.FSDataOutputStream createNonRecursive(org.apache.hadoop.fs.Path f,
                                                                  boolean overwrite,
                                                                  int bufferSize,
                                                                  short replication,
                                                                  long blockSize,
                                                                  org.apache.hadoop.util.Progressable progress)
                                                           throws IOException

Overrides:

createNonRecursive in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getBlockSize

public long getBlockSize(org.apache.hadoop.fs.Path f)
                  throws IOException

Overrides:

getBlockSize in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getDefaultReplication

public short getDefaultReplication(org.apache.hadoop.fs.Path path)

Overrides:

getDefaultReplication in class org.apache.hadoop.fs.FileSystem

addDelegationTokens

public org.apache.hadoop.security.token.Token<?>[] addDelegationTokens(String renewer,
                                                                       org.apache.hadoop.security.Credentials credentials)
                                                                throws IOException

Overrides:

addDelegationTokens in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                                      boolean overwrite,
                                                      int bufferSize,
                                                      short replication,
                                                      long blockSize,
                                                      org.apache.hadoop.util.Progressable progress)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

copyToLocalFile

public void copyToLocalFile(org.apache.hadoop.fs.Path src,
                            org.apache.hadoop.fs.Path dst)
                     throws IOException

Overrides:

copyToLocalFile in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

listStatus

public org.apache.hadoop.fs.FileStatus[] listStatus(org.apache.hadoop.fs.Path[] files,
                                                    org.apache.hadoop.fs.PathFilter filter)
                                             throws IOException

Overrides:

listStatus in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getLength

public long getLength(org.apache.hadoop.fs.Path f)
               throws IOException

Overrides:

getLength in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

moveToLocalFile

public void moveToLocalFile(org.apache.hadoop.fs.Path src,
                            org.apache.hadoop.fs.Path dst)
                     throws IOException

Overrides:

moveToLocalFile in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getStatus

public org.apache.hadoop.fs.FsStatus getStatus(org.apache.hadoop.fs.Path p)
                                        throws IOException

Overrides:

getStatus in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

createNonRecursive

public org.apache.hadoop.fs.FSDataOutputStream createNonRecursive(org.apache.hadoop.fs.Path f,
                                                                  org.apache.hadoop.fs.permission.FsPermission permission,
                                                                  boolean overwrite,
                                                                  int bufferSize,
                                                                  short replication,
                                                                  long blockSize,
                                                                  org.apache.hadoop.util.Progressable progress)
                                                           throws IOException

Overrides:

createNonRecursive in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getCanonicalServiceName

public String getCanonicalServiceName()

Overrides:

getCanonicalServiceName in class org.apache.hadoop.fs.FileSystem

cancelDeleteOnExit

public boolean cancelDeleteOnExit(org.apache.hadoop.fs.Path f)

Overrides:

cancelDeleteOnExit in class org.apache.hadoop.fs.FileSystem

copyFromLocalFile

public void copyFromLocalFile(boolean delSrc,
                              boolean overwrite,
                              org.apache.hadoop.fs.Path[] srcs,
                              org.apache.hadoop.fs.Path dst)
                       throws IOException

Overrides:

copyFromLocalFile in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

setPermission

public void setPermission(org.apache.hadoop.fs.Path p,
                          org.apache.hadoop.fs.permission.FsPermission permission)
                   throws IOException

Overrides:

setPermission in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

startLocalOutput

public org.apache.hadoop.fs.Path startLocalOutput(org.apache.hadoop.fs.Path fsOutputFile,
                                                  org.apache.hadoop.fs.Path tmpLocalFile)
                                           throws IOException

Overrides:

startLocalOutput in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

copyFromLocalFile

public void copyFromLocalFile(org.apache.hadoop.fs.Path src,
                              org.apache.hadoop.fs.Path dst)
                       throws IOException

Overrides:

copyFromLocalFile in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getChildFileSystems

public org.apache.hadoop.fs.FileSystem[] getChildFileSystems()

Overrides:

getChildFileSystems in class org.apache.hadoop.fs.FileSystem

copyFromLocalFile

public void copyFromLocalFile(boolean delSrc,
                              org.apache.hadoop.fs.Path src,
                              org.apache.hadoop.fs.Path dst)
                       throws IOException

Overrides:

copyFromLocalFile in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

setTimes

public void setTimes(org.apache.hadoop.fs.Path p,
                     long mtime,
                     long atime)
              throws IOException

Overrides:

setTimes in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getFileBlockLocations

public org.apache.hadoop.fs.BlockLocation[] getFileBlockLocations(org.apache.hadoop.fs.FileStatus file,
                                                                  long start,
                                                                  long len)
                                                           throws IOException

Overrides:

getFileBlockLocations in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getContentSummary

public org.apache.hadoop.fs.ContentSummary getContentSummary(org.apache.hadoop.fs.Path f)
                                                      throws IOException

Overrides:

getContentSummary in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

renameSnapshot

public void renameSnapshot(org.apache.hadoop.fs.Path path,
                           String snapshotOldName,
                           String snapshotNewName)
                    throws IOException

Overrides:

renameSnapshot in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

moveFromLocalFile

public void moveFromLocalFile(org.apache.hadoop.fs.Path[] srcs,
                              org.apache.hadoop.fs.Path dst)
                       throws IOException

Overrides:

moveFromLocalFile in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

createNonRecursive

public org.apache.hadoop.fs.FSDataOutputStream createNonRecursive(org.apache.hadoop.fs.Path f,
                                                                  org.apache.hadoop.fs.permission.FsPermission permission,
                                                                  EnumSet<org.apache.hadoop.fs.CreateFlag> flags,
                                                                  int bufferSize,
                                                                  short replication,
                                                                  long blockSize,
                                                                  org.apache.hadoop.util.Progressable progress)
                                                           throws IOException

Overrides:

createNonRecursive in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

mkdirs

public boolean mkdirs(org.apache.hadoop.fs.Path f)
               throws IOException

Overrides:

mkdirs in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                                      boolean overwrite,
                                                      int bufferSize,
                                                      short replication,
                                                      long blockSize)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

completeLocalOutput

public void completeLocalOutput(org.apache.hadoop.fs.Path fsOutputFile,
                                org.apache.hadoop.fs.Path tmpLocalFile)
                         throws IOException

Overrides:

completeLocalOutput in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

open

public org.apache.hadoop.fs.FSDataInputStream open(org.apache.hadoop.fs.Path f)
                                            throws IOException

Overrides:

open in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                                      boolean overwrite,
                                                      int bufferSize,
                                                      org.apache.hadoop.util.Progressable progress)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

setVerifyChecksum

public void setVerifyChecksum(boolean verifyChecksum)

Overrides:

setVerifyChecksum in class org.apache.hadoop.fs.FileSystem

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                                      org.apache.hadoop.fs.permission.FsPermission permission,
                                                      EnumSet<org.apache.hadoop.fs.CreateFlag> flags,
                                                      int bufferSize,
                                                      short replication,
                                                      long blockSize,
                                                      org.apache.hadoop.util.Progressable progress)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getDefaultBlockSize

public long getDefaultBlockSize()

Overrides:

getDefaultBlockSize in class org.apache.hadoop.fs.FileSystem

copyFromLocalFile

public void copyFromLocalFile(boolean delSrc,
                              boolean overwrite,
                              org.apache.hadoop.fs.Path src,
                              org.apache.hadoop.fs.Path dst)
                       throws IOException

Overrides:

copyFromLocalFile in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                                      boolean overwrite)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getDefaultBlockSize

public long getDefaultBlockSize(org.apache.hadoop.fs.Path f)

Overrides:

getDefaultBlockSize in class org.apache.hadoop.fs.FileSystem

deleteSnapshot

public void deleteSnapshot(org.apache.hadoop.fs.Path path,
                           String snapshotName)
                    throws IOException

Overrides:

deleteSnapshot in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                                      boolean overwrite,
                                                      int bufferSize)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

setOwner

public void setOwner(org.apache.hadoop.fs.Path p,
                     String username,
                     String groupname)
              throws IOException

Overrides:

setOwner in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

listFiles

public org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.LocatedFileStatus> listFiles(org.apache.hadoop.fs.Path f,
                                                                                             boolean recursive)
                                                                                      throws IOException

Overrides:

listFiles in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

append

public org.apache.hadoop.fs.FSDataOutputStream append(org.apache.hadoop.fs.Path f,
                                                      int bufferSize)
                                               throws IOException

Overrides:

append in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

deleteOnExit

public boolean deleteOnExit(org.apache.hadoop.fs.Path f)
                     throws IOException

Overrides:

deleteOnExit in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

moveFromLocalFile

public void moveFromLocalFile(org.apache.hadoop.fs.Path src,
                              org.apache.hadoop.fs.Path dst)
                       throws IOException

Overrides:

moveFromLocalFile in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

globStatus

public org.apache.hadoop.fs.FileStatus[] globStatus(org.apache.hadoop.fs.Path pathPattern,
                                                    org.apache.hadoop.fs.PathFilter filter)
                                             throws IOException

Overrides:

globStatus in class org.apache.hadoop.fs.FileSystem

Throws:

IOException