org.apache.gobblin.util

Class RateControlledFileSystem

java.lang.Object

org.apache.hadoop.conf.Configured

org.apache.hadoop.fs.FileSystem

org.apache.gobblin.util.filesystem.FileSystemInstrumentation

org.apache.gobblin.util.filesystem.ThrottledFileSystem

org.apache.gobblin.util.RateControlledFileSystem

All Implemented Interfaces:

Closeable, AutoCloseable, Decorator, org.apache.hadoop.conf.Configurable

Deprecated.

use ThrottledFileSystem

@Deprecated
public class RateControlledFileSystem
extends ThrottledFileSystem

Subclass of FileSystem that wraps with a Limiter to control HDFS call rate.

This classes uses Guava's Cache for storing FileSystem URI to Limiter mapping.

For methods that require HDFS calls, this class will first acquire a permit using Limiter, to make sure HDFS call rate is allowed by the uppper limit.

Nested Class Summary

Nested classes/interfaces inherited from class org.apache.gobblin.util.filesystem.ThrottledFileSystem

ThrottledFileSystem.Factory<S extends ScopeType<S>>

Nested classes/interfaces inherited from class org.apache.hadoop.fs.FileSystem

org.apache.hadoop.fs.FileSystem.Statistics

Field Summary

Fields

Modifier and Type	Field and Description
protected org.apache.hadoop.conf.Configuration	conf
protected String	replacementScheme
protected org.apache.hadoop.fs.FileSystem	underlyingFs
protected String	underlyingScheme

Fields inherited from class org.apache.gobblin.util.filesystem.ThrottledFileSystem

LISTING_FILES_PER_PERMIT

Fields inherited from class org.apache.gobblin.util.filesystem.FileSystemInstrumentation

closed

Fields inherited from class org.apache.hadoop.fs.FileSystem

DEFAULT_FS, FS_DEFAULT_NAME_KEY, LOG, SHUTDOWN_HOOK_PRIORITY, statistics

Constructor Summary

Constructors

Constructor and Description
RateControlledFileSystem(org.apache.hadoop.fs.FileSystem fs, long limitPerSecond) Deprecated.

Method Summary

Modifier and Type	Method and Description
org.apache.hadoop.security.token.Token<?>[]	addDelegationTokens(String renewer, org.apache.hadoop.security.Credentials credentials)
org.apache.hadoop.fs.FSDataOutputStream	append(org.apache.hadoop.fs.Path f)
org.apache.hadoop.fs.FSDataOutputStream	append(org.apache.hadoop.fs.Path f, int bufferSize)
boolean	cancelDeleteOnExit(org.apache.hadoop.fs.Path f)
void	completeLocalOutput(org.apache.hadoop.fs.Path fsOutputFile, org.apache.hadoop.fs.Path tmpLocalFile)
void	concat(org.apache.hadoop.fs.Path trg, org.apache.hadoop.fs.Path[] psrcs)
void	copyFromLocalFile(boolean delSrc, boolean overwrite, org.apache.hadoop.fs.Path[] srcs, org.apache.hadoop.fs.Path dst)
void	copyFromLocalFile(boolean delSrc, boolean overwrite, org.apache.hadoop.fs.Path src, org.apache.hadoop.fs.Path dst)
void	copyFromLocalFile(boolean delSrc, org.apache.hadoop.fs.Path src, org.apache.hadoop.fs.Path dst)
void	copyFromLocalFile(org.apache.hadoop.fs.Path src, org.apache.hadoop.fs.Path dst)
void	copyToLocalFile(boolean delSrc, org.apache.hadoop.fs.Path src, org.apache.hadoop.fs.Path dst)
void	copyToLocalFile(boolean delSrc, org.apache.hadoop.fs.Path src, org.apache.hadoop.fs.Path dst, boolean useRawLocalFileSystem)
void	copyToLocalFile(org.apache.hadoop.fs.Path src, org.apache.hadoop.fs.Path dst)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f, boolean overwrite)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f, boolean overwrite, int bufferSize)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f, boolean overwrite, int bufferSize, org.apache.hadoop.util.Progressable progress)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f, boolean overwrite, int bufferSize, short replication, long blockSize)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f, boolean overwrite, int bufferSize, short replication, long blockSize, org.apache.hadoop.util.Progressable progress)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f, org.apache.hadoop.fs.permission.FsPermission permission, EnumSet<org.apache.hadoop.fs.CreateFlag> flags, int bufferSize, short replication, long blockSize, org.apache.hadoop.util.Progressable progress)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f, org.apache.hadoop.fs.permission.FsPermission permission, EnumSet<org.apache.hadoop.fs.CreateFlag> flags, int bufferSize, short replication, long blockSize, org.apache.hadoop.util.Progressable progress, org.apache.hadoop.fs.Options.ChecksumOpt checksumOpt)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f, org.apache.hadoop.util.Progressable progress)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f, short replication)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f, short replication, org.apache.hadoop.util.Progressable progress)
boolean	createNewFile(org.apache.hadoop.fs.Path f)
org.apache.hadoop.fs.FSDataOutputStream	createNonRecursive(org.apache.hadoop.fs.Path f, boolean overwrite, int bufferSize, short replication, long blockSize, org.apache.hadoop.util.Progressable progress)
org.apache.hadoop.fs.FSDataOutputStream	createNonRecursive(org.apache.hadoop.fs.Path f, org.apache.hadoop.fs.permission.FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, org.apache.hadoop.util.Progressable progress)
org.apache.hadoop.fs.FSDataOutputStream	createNonRecursive(org.apache.hadoop.fs.Path f, org.apache.hadoop.fs.permission.FsPermission permission, EnumSet<org.apache.hadoop.fs.CreateFlag> flags, int bufferSize, short replication, long blockSize, org.apache.hadoop.util.Progressable progress)
org.apache.hadoop.fs.Path	createSnapshot(org.apache.hadoop.fs.Path path, String snapshotName)
void	createSymlink(org.apache.hadoop.fs.Path target, org.apache.hadoop.fs.Path link, boolean createParent)
boolean	deleteOnExit(org.apache.hadoop.fs.Path f)
void	deleteSnapshot(org.apache.hadoop.fs.Path path, String snapshotName)
long	getBlockSize(org.apache.hadoop.fs.Path f)
String	getCanonicalServiceName()
org.apache.hadoop.fs.FileSystem[]	getChildFileSystems()
org.apache.hadoop.conf.Configuration	getConf()
org.apache.hadoop.fs.ContentSummary	getContentSummary(org.apache.hadoop.fs.Path f)
Object	getDecoratedObject()
long	getDefaultBlockSize()
long	getDefaultBlockSize(org.apache.hadoop.fs.Path f)
short	getDefaultReplication()
short	getDefaultReplication(org.apache.hadoop.fs.Path path)
org.apache.hadoop.security.token.Token<?>	getDelegationToken(String renewer)
org.apache.hadoop.fs.BlockLocation[]	getFileBlockLocations(org.apache.hadoop.fs.FileStatus file, long start, long len)
org.apache.hadoop.fs.BlockLocation[]	getFileBlockLocations(org.apache.hadoop.fs.Path p, long start, long len)
org.apache.hadoop.fs.FileChecksum	getFileChecksum(org.apache.hadoop.fs.Path f)
org.apache.hadoop.fs.FileStatus	getFileLinkStatus(org.apache.hadoop.fs.Path f)
org.apache.hadoop.fs.Path	getHomeDirectory()
long	getLength(org.apache.hadoop.fs.Path f)
org.apache.hadoop.fs.Path	getLinkTarget(org.apache.hadoop.fs.Path f)
String	getName()
static com.google.common.base.Optional<Long>	getRateIfRateControlled(org.apache.hadoop.fs.FileSystem fs) Deprecated. Determines whether the file system is rate controlled, and if so, returns the allowed rate in operations per second.
protected Limiter	getRateLimiter() Deprecated.
short	getReplication(org.apache.hadoop.fs.Path src)
String	getScheme()
org.apache.hadoop.fs.FsServerDefaults	getServerDefaults()
org.apache.hadoop.fs.FsServerDefaults	getServerDefaults(org.apache.hadoop.fs.Path p)
org.apache.hadoop.fs.FsStatus	getStatus()
org.apache.hadoop.fs.FsStatus	getStatus(org.apache.hadoop.fs.Path p)
URI	getUri()
long	getUsed()
org.apache.hadoop.fs.Path	getWorkingDirectory()
org.apache.hadoop.fs.FileStatus[]	globStatus(org.apache.hadoop.fs.Path pathPattern, org.apache.hadoop.fs.PathFilter filter)
void	initialize(URI uri, org.apache.hadoop.conf.Configuration conf)
boolean	isDirectory(org.apache.hadoop.fs.Path f)
boolean	isFile(org.apache.hadoop.fs.Path f)
org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.Path>	listCorruptFileBlocks(org.apache.hadoop.fs.Path path)
org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.LocatedFileStatus>	listFiles(org.apache.hadoop.fs.Path f, boolean recursive)
org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.LocatedFileStatus>	listLocatedStatus(org.apache.hadoop.fs.Path f)
org.apache.hadoop.fs.FileStatus[]	listStatus(org.apache.hadoop.fs.Path[] files)
org.apache.hadoop.fs.FileStatus[]	listStatus(org.apache.hadoop.fs.Path[] files, org.apache.hadoop.fs.PathFilter filter)
org.apache.hadoop.fs.Path	makeQualified(org.apache.hadoop.fs.Path path)
boolean	mkdirs(org.apache.hadoop.fs.Path f)
void	moveFromLocalFile(org.apache.hadoop.fs.Path[] srcs, org.apache.hadoop.fs.Path dst)
void	moveFromLocalFile(org.apache.hadoop.fs.Path src, org.apache.hadoop.fs.Path dst)
void	moveToLocalFile(org.apache.hadoop.fs.Path src, org.apache.hadoop.fs.Path dst)
org.apache.hadoop.fs.FSDataInputStream	open(org.apache.hadoop.fs.Path f)
void	renameSnapshot(org.apache.hadoop.fs.Path path, String snapshotOldName, String snapshotNewName)
org.apache.hadoop.fs.Path	resolvePath(org.apache.hadoop.fs.Path p)
void	setConf(org.apache.hadoop.conf.Configuration conf)
void	setOwner(org.apache.hadoop.fs.Path p, String username, String groupname)
void	setPermission(org.apache.hadoop.fs.Path p, org.apache.hadoop.fs.permission.FsPermission permission)
boolean	setReplication(org.apache.hadoop.fs.Path src, short replication)
void	setTimes(org.apache.hadoop.fs.Path p, long mtime, long atime)
void	setVerifyChecksum(boolean verifyChecksum)
void	setWorkingDirectory(org.apache.hadoop.fs.Path new_dir)
void	setWriteChecksum(boolean writeChecksum)
org.apache.hadoop.fs.Path	startLocalOutput(org.apache.hadoop.fs.Path fsOutputFile, org.apache.hadoop.fs.Path tmpLocalFile)
void	startRateControl() Deprecated.
boolean	supportsSymlinks()

Methods inherited from class org.apache.gobblin.util.filesystem.ThrottledFileSystem

append, close, create, delete, delete, exists, getFileStatus, getServiceName, globStatus, listStatus, listStatus, mkdirs, open, rename

Methods inherited from class org.apache.gobblin.util.filesystem.FileSystemInstrumentation

onClose

Methods inherited from class org.apache.hadoop.fs.FileSystem

areSymlinksEnabled, canonicalizeUri, checkPath, clearStatistics, closeAll, closeAllForUGI, create, createSnapshot, enableSymlinks, fixRelativePart, get, get, get, getAllStatistics, getCanonicalUri, getDefaultPort, getDefaultUri, getFileSystemClass, getFSofPath, getInitialWorkingDirectory, getLocal, getNamed, getStatistics, getStatistics, listLocatedStatus, mkdirs, newInstance, newInstance, newInstance, newInstanceLocal, primitiveCreate, primitiveMkdir, primitiveMkdir, printStatistics, processDeleteOnExit, rename, resolveLink, setDefaultUri, setDefaultUri

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

replacementScheme

protected String replacementScheme

underlyingScheme

protected String underlyingScheme

conf

protected org.apache.hadoop.conf.Configuration conf

underlyingFs

protected org.apache.hadoop.fs.FileSystem underlyingFs

Constructor Detail

RateControlledFileSystem

public RateControlledFileSystem(org.apache.hadoop.fs.FileSystem fs,
                                long limitPerSecond)

Deprecated.

Method Detail

getRateIfRateControlled

public static com.google.common.base.Optional<Long> getRateIfRateControlled(org.apache.hadoop.fs.FileSystem fs)

Deprecated.

Determines whether the file system is rate controlled, and if so, returns the allowed rate in operations per second.

Parameters:

fs - FileSystem to check for rate control.

Returns:

Optional.absent() if file system is not rate controlled, otherwise, the rate in operations per second.

startRateControl

public void startRateControl()
                      throws ExecutionException

Deprecated.

Throws:

ExecutionException

getRateLimiter

protected Limiter getRateLimiter()

Deprecated.

Overrides:

getRateLimiter in class ThrottledFileSystem

getDecoratedObject

public Object getDecoratedObject()

Specified by:

getDecoratedObject in interface Decorator

getScheme

public String getScheme()

Overrides:

getScheme in class org.apache.hadoop.fs.FileSystem

initialize

public void initialize(URI uri,
                       org.apache.hadoop.conf.Configuration conf)
                throws IOException

Overrides:

initialize in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

setConf

public void setConf(org.apache.hadoop.conf.Configuration conf)

Specified by:

setConf in interface org.apache.hadoop.conf.Configurable

Overrides:

setConf in class org.apache.hadoop.conf.Configured

getUri

public URI getUri()

Specified by:

getUri in class org.apache.hadoop.fs.FileSystem

getFileLinkStatus

public org.apache.hadoop.fs.FileStatus getFileLinkStatus(org.apache.hadoop.fs.Path f)
                                                  throws IOException

Overrides:

getFileLinkStatus in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getStatus

public org.apache.hadoop.fs.FsStatus getStatus()
                                        throws IOException

Overrides:

getStatus in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

append

public org.apache.hadoop.fs.FSDataOutputStream append(org.apache.hadoop.fs.Path f)
                                               throws IOException

Overrides:

append in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getReplication

public short getReplication(org.apache.hadoop.fs.Path src)
                     throws IOException

Overrides:

getReplication in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

setWriteChecksum

public void setWriteChecksum(boolean writeChecksum)

Overrides:

setWriteChecksum in class org.apache.hadoop.fs.FileSystem

getFileChecksum

public org.apache.hadoop.fs.FileChecksum getFileChecksum(org.apache.hadoop.fs.Path f)
                                                  throws IOException

Overrides:

getFileChecksum in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

isDirectory

public boolean isDirectory(org.apache.hadoop.fs.Path f)
                    throws IOException

Overrides:

isDirectory in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

createSymlink

public void createSymlink(org.apache.hadoop.fs.Path target,
                          org.apache.hadoop.fs.Path link,
                          boolean createParent)
                   throws IOException

Overrides:

createSymlink in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

createSnapshot

public org.apache.hadoop.fs.Path createSnapshot(org.apache.hadoop.fs.Path path,
                                                String snapshotName)
                                         throws IOException

Overrides:

createSnapshot in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                                      org.apache.hadoop.fs.permission.FsPermission permission,
                                                      EnumSet<org.apache.hadoop.fs.CreateFlag> flags,
                                                      int bufferSize,
                                                      short replication,
                                                      long blockSize,
                                                      org.apache.hadoop.util.Progressable progress,
                                                      org.apache.hadoop.fs.Options.ChecksumOpt checksumOpt)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

resolvePath

public org.apache.hadoop.fs.Path resolvePath(org.apache.hadoop.fs.Path p)
                                      throws IOException

Overrides:

resolvePath in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getUsed

public long getUsed()
             throws IOException

Overrides:

getUsed in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getConf

public org.apache.hadoop.conf.Configuration getConf()

Specified by:

getConf in interface org.apache.hadoop.conf.Configurable

Overrides:

getConf in class org.apache.hadoop.conf.Configured

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                                      org.apache.hadoop.util.Progressable progress)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

isFile

public boolean isFile(org.apache.hadoop.fs.Path f)
               throws IOException

Overrides:

isFile in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getWorkingDirectory

public org.apache.hadoop.fs.Path getWorkingDirectory()

Specified by:

getWorkingDirectory in class org.apache.hadoop.fs.FileSystem

getServerDefaults

public org.apache.hadoop.fs.FsServerDefaults getServerDefaults()
                                                        throws IOException

Overrides:

getServerDefaults in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

copyToLocalFile

public void copyToLocalFile(boolean delSrc,
                            org.apache.hadoop.fs.Path src,
                            org.apache.hadoop.fs.Path dst,
                            boolean useRawLocalFileSystem)
                     throws IOException

Overrides:

copyToLocalFile in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

setWorkingDirectory

public void setWorkingDirectory(org.apache.hadoop.fs.Path new_dir)

Specified by:

setWorkingDirectory in class org.apache.hadoop.fs.FileSystem

getName

public String getName()

Overrides:

getName in class org.apache.hadoop.fs.FileSystem

createNewFile

public boolean createNewFile(org.apache.hadoop.fs.Path f)
                      throws IOException

Overrides:

createNewFile in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

listStatus

public org.apache.hadoop.fs.FileStatus[] listStatus(org.apache.hadoop.fs.Path[] files)
                                             throws IOException

Overrides:

listStatus in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getLinkTarget

public org.apache.hadoop.fs.Path getLinkTarget(org.apache.hadoop.fs.Path f)
                                        throws IOException

Overrides:

getLinkTarget in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

copyToLocalFile

public void copyToLocalFile(boolean delSrc,
                            org.apache.hadoop.fs.Path src,
                            org.apache.hadoop.fs.Path dst)
                     throws IOException

Overrides:

copyToLocalFile in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getDefaultReplication

public short getDefaultReplication()

Overrides:

getDefaultReplication in class org.apache.hadoop.fs.FileSystem

getDelegationToken

public org.apache.hadoop.security.token.Token<?> getDelegationToken(String renewer)
                                                             throws IOException

Overrides:

getDelegationToken in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getServerDefaults

public org.apache.hadoop.fs.FsServerDefaults getServerDefaults(org.apache.hadoop.fs.Path p)
                                                        throws IOException

Overrides:

getServerDefaults in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                                      short replication)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getFileBlockLocations

public org.apache.hadoop.fs.BlockLocation[] getFileBlockLocations(org.apache.hadoop.fs.Path p,
                                                                  long start,
                                                                  long len)
                                                           throws IOException

Overrides:

getFileBlockLocations in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

concat

public void concat(org.apache.hadoop.fs.Path trg,
                   org.apache.hadoop.fs.Path[] psrcs)
            throws IOException

Overrides:

concat in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getHomeDirectory

public org.apache.hadoop.fs.Path getHomeDirectory()

Overrides:

getHomeDirectory in class org.apache.hadoop.fs.FileSystem

supportsSymlinks

public boolean supportsSymlinks()

Overrides:

supportsSymlinks in class org.apache.hadoop.fs.FileSystem

listLocatedStatus

public org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.LocatedFileStatus> listLocatedStatus(org.apache.hadoop.fs.Path f)
                                                                                              throws FileNotFoundException,
                                                                                                     IOException

Overrides:

listLocatedStatus in class org.apache.hadoop.fs.FileSystem

Throws:

FileNotFoundException

IOException

listCorruptFileBlocks

public org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.Path> listCorruptFileBlocks(org.apache.hadoop.fs.Path path)
                                                                                     throws IOException

Overrides:

listCorruptFileBlocks in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                                      short replication,
                                                      org.apache.hadoop.util.Progressable progress)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

setReplication

public boolean setReplication(org.apache.hadoop.fs.Path src,
                              short replication)
                       throws IOException

Overrides:

setReplication in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

makeQualified

public org.apache.hadoop.fs.Path makeQualified(org.apache.hadoop.fs.Path path)

Overrides:

makeQualified in class org.apache.hadoop.fs.FileSystem

createNonRecursive

public org.apache.hadoop.fs.FSDataOutputStream createNonRecursive(org.apache.hadoop.fs.Path f,
                                                                  boolean overwrite,
                                                                  int bufferSize,
                                                                  short replication,
                                                                  long blockSize,
                                                                  org.apache.hadoop.util.Progressable progress)
                                                           throws IOException

Overrides:

createNonRecursive in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getBlockSize

public long getBlockSize(org.apache.hadoop.fs.Path f)
                  throws IOException

Overrides:

getBlockSize in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getDefaultReplication

public short getDefaultReplication(org.apache.hadoop.fs.Path path)

Overrides:

getDefaultReplication in class org.apache.hadoop.fs.FileSystem

addDelegationTokens

public org.apache.hadoop.security.token.Token<?>[] addDelegationTokens(String renewer,
                                                                       org.apache.hadoop.security.Credentials credentials)
                                                                throws IOException

Overrides:

addDelegationTokens in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                                      boolean overwrite,
                                                      int bufferSize,
                                                      short replication,
                                                      long blockSize,
                                                      org.apache.hadoop.util.Progressable progress)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

copyToLocalFile

public void copyToLocalFile(org.apache.hadoop.fs.Path src,
                            org.apache.hadoop.fs.Path dst)
                     throws IOException

Overrides:

copyToLocalFile in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

listStatus

public org.apache.hadoop.fs.FileStatus[] listStatus(org.apache.hadoop.fs.Path[] files,
                                                    org.apache.hadoop.fs.PathFilter filter)
                                             throws IOException

Overrides:

listStatus in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getLength

public long getLength(org.apache.hadoop.fs.Path f)
               throws IOException

Overrides:

getLength in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

moveToLocalFile

public void moveToLocalFile(org.apache.hadoop.fs.Path src,
                            org.apache.hadoop.fs.Path dst)
                     throws IOException

Overrides:

moveToLocalFile in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getStatus

public org.apache.hadoop.fs.FsStatus getStatus(org.apache.hadoop.fs.Path p)
                                        throws IOException

Overrides:

getStatus in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

createNonRecursive

public org.apache.hadoop.fs.FSDataOutputStream createNonRecursive(org.apache.hadoop.fs.Path f,
                                                                  org.apache.hadoop.fs.permission.FsPermission permission,
                                                                  boolean overwrite,
                                                                  int bufferSize,
                                                                  short replication,
                                                                  long blockSize,
                                                                  org.apache.hadoop.util.Progressable progress)
                                                           throws IOException

Overrides:

createNonRecursive in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getCanonicalServiceName

public String getCanonicalServiceName()

Overrides:

getCanonicalServiceName in class org.apache.hadoop.fs.FileSystem

cancelDeleteOnExit

public boolean cancelDeleteOnExit(org.apache.hadoop.fs.Path f)

Overrides:

cancelDeleteOnExit in class org.apache.hadoop.fs.FileSystem

copyFromLocalFile

public void copyFromLocalFile(boolean delSrc,
                              boolean overwrite,
                              org.apache.hadoop.fs.Path[] srcs,
                              org.apache.hadoop.fs.Path dst)
                       throws IOException

Overrides:

copyFromLocalFile in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

setPermission

public void setPermission(org.apache.hadoop.fs.Path p,
                          org.apache.hadoop.fs.permission.FsPermission permission)
                   throws IOException

Overrides:

setPermission in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

startLocalOutput

public org.apache.hadoop.fs.Path startLocalOutput(org.apache.hadoop.fs.Path fsOutputFile,
                                                  org.apache.hadoop.fs.Path tmpLocalFile)
                                           throws IOException

Overrides:

startLocalOutput in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

copyFromLocalFile

public void copyFromLocalFile(org.apache.hadoop.fs.Path src,
                              org.apache.hadoop.fs.Path dst)
                       throws IOException

Overrides:

copyFromLocalFile in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getChildFileSystems

public org.apache.hadoop.fs.FileSystem[] getChildFileSystems()

Overrides:

getChildFileSystems in class org.apache.hadoop.fs.FileSystem

copyFromLocalFile

public void copyFromLocalFile(boolean delSrc,
                              org.apache.hadoop.fs.Path src,
                              org.apache.hadoop.fs.Path dst)
                       throws IOException

Overrides:

copyFromLocalFile in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

setTimes

public void setTimes(org.apache.hadoop.fs.Path p,
                     long mtime,
                     long atime)
              throws IOException

Overrides:

setTimes in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getFileBlockLocations

public org.apache.hadoop.fs.BlockLocation[] getFileBlockLocations(org.apache.hadoop.fs.FileStatus file,
                                                                  long start,
                                                                  long len)
                                                           throws IOException

Overrides:

getFileBlockLocations in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getContentSummary

public org.apache.hadoop.fs.ContentSummary getContentSummary(org.apache.hadoop.fs.Path f)
                                                      throws IOException

Overrides:

getContentSummary in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

renameSnapshot

public void renameSnapshot(org.apache.hadoop.fs.Path path,
                           String snapshotOldName,
                           String snapshotNewName)
                    throws IOException

Overrides:

renameSnapshot in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

moveFromLocalFile

public void moveFromLocalFile(org.apache.hadoop.fs.Path[] srcs,
                              org.apache.hadoop.fs.Path dst)
                       throws IOException

Overrides:

moveFromLocalFile in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

createNonRecursive

public org.apache.hadoop.fs.FSDataOutputStream createNonRecursive(org.apache.hadoop.fs.Path f,
                                                                  org.apache.hadoop.fs.permission.FsPermission permission,
                                                                  EnumSet<org.apache.hadoop.fs.CreateFlag> flags,
                                                                  int bufferSize,
                                                                  short replication,
                                                                  long blockSize,
                                                                  org.apache.hadoop.util.Progressable progress)
                                                           throws IOException

Overrides:

createNonRecursive in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

mkdirs

public boolean mkdirs(org.apache.hadoop.fs.Path f)
               throws IOException

Overrides:

mkdirs in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                                      boolean overwrite,
                                                      int bufferSize,
                                                      short replication,
                                                      long blockSize)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

completeLocalOutput

public void completeLocalOutput(org.apache.hadoop.fs.Path fsOutputFile,
                                org.apache.hadoop.fs.Path tmpLocalFile)
                         throws IOException

Overrides:

completeLocalOutput in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

open

public org.apache.hadoop.fs.FSDataInputStream open(org.apache.hadoop.fs.Path f)
                                            throws IOException

Overrides:

open in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                                      boolean overwrite,
                                                      int bufferSize,
                                                      org.apache.hadoop.util.Progressable progress)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

setVerifyChecksum

public void setVerifyChecksum(boolean verifyChecksum)

Overrides:

setVerifyChecksum in class org.apache.hadoop.fs.FileSystem

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                                      org.apache.hadoop.fs.permission.FsPermission permission,
                                                      EnumSet<org.apache.hadoop.fs.CreateFlag> flags,
                                                      int bufferSize,
                                                      short replication,
                                                      long blockSize,
                                                      org.apache.hadoop.util.Progressable progress)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getDefaultBlockSize

public long getDefaultBlockSize()

Overrides:

getDefaultBlockSize in class org.apache.hadoop.fs.FileSystem

copyFromLocalFile

public void copyFromLocalFile(boolean delSrc,
                              boolean overwrite,
                              org.apache.hadoop.fs.Path src,
                              org.apache.hadoop.fs.Path dst)
                       throws IOException

Overrides:

copyFromLocalFile in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                                      boolean overwrite)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getDefaultBlockSize

public long getDefaultBlockSize(org.apache.hadoop.fs.Path f)

Overrides:

getDefaultBlockSize in class org.apache.hadoop.fs.FileSystem

deleteSnapshot

public void deleteSnapshot(org.apache.hadoop.fs.Path path,
                           String snapshotName)
                    throws IOException

Overrides:

deleteSnapshot in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                                      boolean overwrite,
                                                      int bufferSize)
                                               throws IOException

Overrides:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

setOwner

public void setOwner(org.apache.hadoop.fs.Path p,
                     String username,
                     String groupname)
              throws IOException

Overrides:

setOwner in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

listFiles

public org.apache.hadoop.fs.RemoteIterator<org.apache.hadoop.fs.LocatedFileStatus> listFiles(org.apache.hadoop.fs.Path f,
                                                                                             boolean recursive)
                                                                                      throws IOException

Overrides:

listFiles in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

append

public org.apache.hadoop.fs.FSDataOutputStream append(org.apache.hadoop.fs.Path f,
                                                      int bufferSize)
                                               throws IOException

Overrides:

append in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

deleteOnExit

public boolean deleteOnExit(org.apache.hadoop.fs.Path f)
                     throws IOException

Overrides:

deleteOnExit in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

moveFromLocalFile

public void moveFromLocalFile(org.apache.hadoop.fs.Path src,
                              org.apache.hadoop.fs.Path dst)
                       throws IOException

Overrides:

moveFromLocalFile in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

globStatus

public org.apache.hadoop.fs.FileStatus[] globStatus(org.apache.hadoop.fs.Path pathPattern,
                                                    org.apache.hadoop.fs.PathFilter filter)
                                             throws IOException

Overrides:

globStatus in class org.apache.hadoop.fs.FileSystem

Throws:

IOException