edu.umd.cloud9.io
Class FileMerger

java.lang.Object
  org.apache.hadoop.conf.Configured
      edu.umd.cloud9.io.FileMerger

All Implemented Interfaces:

Configurable, Tool

public class FileMerger
extends Configured
implements Tool

Field Summary

static int	DEFAULT_RANDOM_STRING_LENGTH
static boolean	DELETE_SOURCE
static String	DELETE_SOURCE_OPTION
static String	FILE_CONTENT_DELIMITER
static String	HELP_OPTION
static String	INPUT_OPTION
static String	INTEGER_INDICATOR
static boolean	LOCAL_MERGE
static String	LOCAL_MERGE_OPTION
static String	MAPPER_OPTION
static String	MERGE
static String	OUTPUT_OPTION
static String	PATH_INDICATOR
static Random	RANDOM_GENERATOR
static String	REDUCER_OPTION
static boolean	TEXT_FILE_INPUT
static String	TEXT_FILE_INPUT_FORMAT

Constructor Summary

FileMerger()

Method Summary

static String	generateRandomString() Generate a random string of default length.
static String	generateRandomString(int length) Generate a random string of given length.
static void	main(String[] args)
static Path	merge(Configuration configuration, String inputPath, String outputPath, int mapperTasks, boolean textFileFormat, boolean deleteSource)
static Path	mergeSequenceFiles(Configuration configuration, String inputFiles, String outputFile, int numberOfMappers, Class<? extends Writable> keyClass, Class<? extends Writable> valueClass, boolean deleteSource)
static Path	mergeSequenceFiles(Configuration configuration, String inputFiles, String outputFile, int numberOfMappers, Class<? extends Writable> keyClass, Class<? extends Writable> valueClass, boolean deleteSource, boolean deleteDestinationFileIfExist)
static Path	mergeTextFiles(Configuration configuration, String inputFiles, String outputFile, int numberOfMappers, boolean deleteSource) This method merges all files (in TextFormat) specified by the glob expression inputFiles
static Path	mergeTextFiles(Configuration configuration, String inputFiles, String outputFile, int numberOfMappers, boolean deleteSource, boolean deleteDestinationFileIfExist)
int	run(String[] args)

Methods inherited from class org.apache.hadoop.conf.Configured

getConf, setConf

Methods inherited from class java.lang.Object

equals, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface org.apache.hadoop.conf.Configurable

getConf, setConf

Field Detail

RANDOM_GENERATOR

public static final Random RANDOM_GENERATOR

DEFAULT_RANDOM_STRING_LENGTH

public static final int DEFAULT_RANDOM_STRING_LENGTH

See Also:

Constant Field Values

PATH_INDICATOR

public static final String PATH_INDICATOR

See Also:

Constant Field Values

INTEGER_INDICATOR

public static final String INTEGER_INDICATOR

See Also:

Constant Field Values

HELP_OPTION

public static final String HELP_OPTION

See Also:

Constant Field Values

INPUT_OPTION

public static final String INPUT_OPTION

See Also:

Constant Field Values

OUTPUT_OPTION

public static final String OUTPUT_OPTION

See Also:

Constant Field Values

MAPPER_OPTION

public static final String MAPPER_OPTION

See Also:

Constant Field Values

REDUCER_OPTION

public static final String REDUCER_OPTION

See Also:

Constant Field Values

MERGE

public static final String MERGE

See Also:

Constant Field Values

LOCAL_MERGE_OPTION

public static final String LOCAL_MERGE_OPTION

See Also:

Constant Field Values

LOCAL_MERGE

public static final boolean LOCAL_MERGE

See Also:

Constant Field Values

DELETE_SOURCE_OPTION

public static final String DELETE_SOURCE_OPTION

See Also:

Constant Field Values

DELETE_SOURCE

public static final boolean DELETE_SOURCE

See Also:

Constant Field Values

TEXT_FILE_INPUT_FORMAT

public static final String TEXT_FILE_INPUT_FORMAT

See Also:

Constant Field Values

TEXT_FILE_INPUT

public static final boolean TEXT_FILE_INPUT

See Also:

Constant Field Values

FILE_CONTENT_DELIMITER

public static final String FILE_CONTENT_DELIMITER

See Also:

Constant Field Values

Constructor Detail

FileMerger

public FileMerger()

Method Detail

generateRandomString

public static String generateRandomString(int length)

Generate a random string of given length.

Parameters:

length -

Returns:

generateRandomString

public static String generateRandomString()

Generate a random string of default length.

Returns:

mergeTextFiles

public static Path mergeTextFiles(Configuration configuration,
                                  String inputFiles,
                                  String outputFile,
                                  int numberOfMappers,
                                  boolean deleteSource)
                           throws IOException

This method merges all files (in TextFormat) specified by the glob expression inputFiles

Parameters:

inputFiles -

outputFile -

numberOfMappers -

deleteSource -

Returns:

Throws:

IOException

mergeTextFiles

public static Path mergeTextFiles(Configuration configuration,
                                  String inputFiles,
                                  String outputFile,
                                  int numberOfMappers,
                                  boolean deleteSource,
                                  boolean deleteDestinationFileIfExist)
                           throws IOException

Throws:

IOException

mergeSequenceFiles

public static Path mergeSequenceFiles(Configuration configuration,
                                      String inputFiles,
                                      String outputFile,
                                      int numberOfMappers,
                                      Class<? extends Writable> keyClass,
                                      Class<? extends Writable> valueClass,
                                      boolean deleteSource)
                               throws IOException,
                                      InstantiationException,
                                      IllegalAccessException

Throws:

IOException

InstantiationException

IllegalAccessException

mergeSequenceFiles

public static Path mergeSequenceFiles(Configuration configuration,
                                      String inputFiles,
                                      String outputFile,
                                      int numberOfMappers,
                                      Class<? extends Writable> keyClass,
                                      Class<? extends Writable> valueClass,
                                      boolean deleteSource,
                                      boolean deleteDestinationFileIfExist)
                               throws IOException,
                                      InstantiationException,
                                      IllegalAccessException

Throws:

IOException

InstantiationException

IllegalAccessException

run

public int run(String[] args)
        throws IOException

Specified by:

run in interface Tool

Throws:

IOException

merge

public static Path merge(Configuration configuration,
                         String inputPath,
                         String outputPath,
                         int mapperTasks,
                         boolean textFileFormat,
                         boolean deleteSource)
                  throws IOException,
                         InstantiationException,
                         IllegalAccessException

Throws:

IOException

InstantiationException

IllegalAccessException

main

public static void main(String[] args)
                 throws Exception

Throws:

Exception