org.apache.tez.runtime.library.common.sort.impl
Class TezMerger

java.lang.Object
  org.apache.tez.runtime.library.common.sort.impl.TezMerger

@InterfaceAudience.Private
@InterfaceStability.Unstable
public class TezMerger
extends Object

Merger is an utility class used by the Map and Reduce tasks for merging both their memory and disk segments

Nested Class Summary

static class

TezMerger.Segment<K,V>

Constructor Summary

TezMerger()

Method Summary

static TezRawKeyValueIterator	merge(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, Class keyClass, Class valueClass, org.apache.hadoop.io.compress.CompressionCodec codec, boolean ifileReadAhead, int ifileReadAheadLength, int ifileBufferSize, org.apache.hadoop.fs.Path[] inputs, boolean deleteInputs, int mergeFactor, org.apache.hadoop.fs.Path tmpDir, org.apache.hadoop.io.RawComparator comparator, org.apache.hadoop.util.Progressable reporter, org.apache.tez.common.counters.TezCounter readsCounter, org.apache.tez.common.counters.TezCounter writesCounter, org.apache.tez.common.counters.TezCounter bytesReadCounter, org.apache.hadoop.util.Progress mergePhase)
static TezRawKeyValueIterator	merge(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, Class keyClass, Class valueClass, org.apache.hadoop.io.compress.CompressionCodec codec, boolean ifileReadAhead, int ifileReadAheadLength, int ifileBufferSize, org.apache.hadoop.fs.Path[] inputs, boolean deleteInputs, int mergeFactor, org.apache.hadoop.fs.Path tmpDir, org.apache.hadoop.io.RawComparator comparator, org.apache.hadoop.util.Progressable reporter, org.apache.tez.common.counters.TezCounter readsCounter, org.apache.tez.common.counters.TezCounter writesCounter, org.apache.tez.common.counters.TezCounter mergedMapOutputsCounter, org.apache.tez.common.counters.TezCounter bytesReadCounter, org.apache.hadoop.util.Progress mergePhase)
static <K,V> TezRawKeyValueIterator	merge(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, Class keyClass, Class valueClass, org.apache.hadoop.io.compress.CompressionCodec codec, List<TezMerger.Segment> segments, int mergeFactor, int inMemSegments, org.apache.hadoop.fs.Path tmpDir, org.apache.hadoop.io.RawComparator comparator, org.apache.hadoop.util.Progressable reporter, boolean sortSegments, org.apache.tez.common.counters.TezCounter readsCounter, org.apache.tez.common.counters.TezCounter writesCounter, org.apache.tez.common.counters.TezCounter bytesReadCounter, org.apache.hadoop.util.Progress mergePhase)
static <K,V> TezRawKeyValueIterator	merge(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, Class keyClass, Class valueClass, org.apache.hadoop.io.compress.CompressionCodec codec, List<TezMerger.Segment> segments, int mergeFactor, org.apache.hadoop.fs.Path tmpDir, org.apache.hadoop.io.RawComparator comparator, org.apache.hadoop.util.Progressable reporter, boolean sortSegments, boolean considerFinalMergeForProgress, org.apache.tez.common.counters.TezCounter readsCounter, org.apache.tez.common.counters.TezCounter writesCounter, org.apache.tez.common.counters.TezCounter bytesReadCounter, org.apache.hadoop.util.Progress mergePhase)
static <K,V> TezRawKeyValueIterator	merge(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, Class keyClass, Class valueClass, List<TezMerger.Segment> segments, int mergeFactor, org.apache.hadoop.fs.Path tmpDir, org.apache.hadoop.io.RawComparator comparator, org.apache.hadoop.util.Progressable reporter, boolean sortSegments, org.apache.tez.common.counters.TezCounter readsCounter, org.apache.tez.common.counters.TezCounter writesCounter, org.apache.tez.common.counters.TezCounter bytesReadCounter, org.apache.hadoop.util.Progress mergePhase)
static TezRawKeyValueIterator	merge(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, Class keyClass, Class valueClass, List<TezMerger.Segment> segments, int mergeFactor, org.apache.hadoop.fs.Path tmpDir, org.apache.hadoop.io.RawComparator comparator, org.apache.hadoop.util.Progressable reporter, org.apache.tez.common.counters.TezCounter readsCounter, org.apache.tez.common.counters.TezCounter writesCounter, org.apache.tez.common.counters.TezCounter bytesReadCounter, org.apache.hadoop.util.Progress mergePhase)
static <K,V> void	writeFile(TezRawKeyValueIterator records, IFile.Writer writer, org.apache.hadoop.util.Progressable progressable, long recordsBeforeProgress)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

TezMerger

public TezMerger()

Method Detail

merge

public static TezRawKeyValueIterator merge(org.apache.hadoop.conf.Configuration conf,
                                           org.apache.hadoop.fs.FileSystem fs,
                                           Class keyClass,
                                           Class valueClass,
                                           org.apache.hadoop.io.compress.CompressionCodec codec,
                                           boolean ifileReadAhead,
                                           int ifileReadAheadLength,
                                           int ifileBufferSize,
                                           org.apache.hadoop.fs.Path[] inputs,
                                           boolean deleteInputs,
                                           int mergeFactor,
                                           org.apache.hadoop.fs.Path tmpDir,
                                           org.apache.hadoop.io.RawComparator comparator,
                                           org.apache.hadoop.util.Progressable reporter,
                                           org.apache.tez.common.counters.TezCounter readsCounter,
                                           org.apache.tez.common.counters.TezCounter writesCounter,
                                           org.apache.tez.common.counters.TezCounter bytesReadCounter,
                                           org.apache.hadoop.util.Progress mergePhase)
                                    throws IOException

Throws:

IOException

merge

public static TezRawKeyValueIterator merge(org.apache.hadoop.conf.Configuration conf,
                                           org.apache.hadoop.fs.FileSystem fs,
                                           Class keyClass,
                                           Class valueClass,
                                           org.apache.hadoop.io.compress.CompressionCodec codec,
                                           boolean ifileReadAhead,
                                           int ifileReadAheadLength,
                                           int ifileBufferSize,
                                           org.apache.hadoop.fs.Path[] inputs,
                                           boolean deleteInputs,
                                           int mergeFactor,
                                           org.apache.hadoop.fs.Path tmpDir,
                                           org.apache.hadoop.io.RawComparator comparator,
                                           org.apache.hadoop.util.Progressable reporter,
                                           org.apache.tez.common.counters.TezCounter readsCounter,
                                           org.apache.tez.common.counters.TezCounter writesCounter,
                                           org.apache.tez.common.counters.TezCounter mergedMapOutputsCounter,
                                           org.apache.tez.common.counters.TezCounter bytesReadCounter,
                                           org.apache.hadoop.util.Progress mergePhase)
                                    throws IOException

Throws:

IOException

merge

public static TezRawKeyValueIterator merge(org.apache.hadoop.conf.Configuration conf,
                                           org.apache.hadoop.fs.FileSystem fs,
                                           Class keyClass,
                                           Class valueClass,
                                           List<TezMerger.Segment> segments,
                                           int mergeFactor,
                                           org.apache.hadoop.fs.Path tmpDir,
                                           org.apache.hadoop.io.RawComparator comparator,
                                           org.apache.hadoop.util.Progressable reporter,
                                           org.apache.tez.common.counters.TezCounter readsCounter,
                                           org.apache.tez.common.counters.TezCounter writesCounter,
                                           org.apache.tez.common.counters.TezCounter bytesReadCounter,
                                           org.apache.hadoop.util.Progress mergePhase)
                                    throws IOException

Throws:

IOException

merge

public static <K,V> TezRawKeyValueIterator merge(org.apache.hadoop.conf.Configuration conf,
                                                 org.apache.hadoop.fs.FileSystem fs,
                                                 Class keyClass,
                                                 Class valueClass,
                                                 List<TezMerger.Segment> segments,
                                                 int mergeFactor,
                                                 org.apache.hadoop.fs.Path tmpDir,
                                                 org.apache.hadoop.io.RawComparator comparator,
                                                 org.apache.hadoop.util.Progressable reporter,
                                                 boolean sortSegments,
                                                 org.apache.tez.common.counters.TezCounter readsCounter,
                                                 org.apache.tez.common.counters.TezCounter writesCounter,
                                                 org.apache.tez.common.counters.TezCounter bytesReadCounter,
                                                 org.apache.hadoop.util.Progress mergePhase)
                                    throws IOException

Throws:

IOException

merge

public static <K,V> TezRawKeyValueIterator merge(org.apache.hadoop.conf.Configuration conf,
                                                 org.apache.hadoop.fs.FileSystem fs,
                                                 Class keyClass,
                                                 Class valueClass,
                                                 org.apache.hadoop.io.compress.CompressionCodec codec,
                                                 List<TezMerger.Segment> segments,
                                                 int mergeFactor,
                                                 org.apache.hadoop.fs.Path tmpDir,
                                                 org.apache.hadoop.io.RawComparator comparator,
                                                 org.apache.hadoop.util.Progressable reporter,
                                                 boolean sortSegments,
                                                 boolean considerFinalMergeForProgress,
                                                 org.apache.tez.common.counters.TezCounter readsCounter,
                                                 org.apache.tez.common.counters.TezCounter writesCounter,
                                                 org.apache.tez.common.counters.TezCounter bytesReadCounter,
                                                 org.apache.hadoop.util.Progress mergePhase)
                                    throws IOException

Throws:

IOException

merge

public static <K,V> TezRawKeyValueIterator merge(org.apache.hadoop.conf.Configuration conf,
                                                 org.apache.hadoop.fs.FileSystem fs,
                                                 Class keyClass,
                                                 Class valueClass,
                                                 org.apache.hadoop.io.compress.CompressionCodec codec,
                                                 List<TezMerger.Segment> segments,
                                                 int mergeFactor,
                                                 int inMemSegments,
                                                 org.apache.hadoop.fs.Path tmpDir,
                                                 org.apache.hadoop.io.RawComparator comparator,
                                                 org.apache.hadoop.util.Progressable reporter,
                                                 boolean sortSegments,
                                                 org.apache.tez.common.counters.TezCounter readsCounter,
                                                 org.apache.tez.common.counters.TezCounter writesCounter,
                                                 org.apache.tez.common.counters.TezCounter bytesReadCounter,
                                                 org.apache.hadoop.util.Progress mergePhase)
                                    throws IOException

Throws:

IOException

writeFile

public static <K,V> void writeFile(TezRawKeyValueIterator records,
                                   IFile.Writer writer,
                                   org.apache.hadoop.util.Progressable progressable,
                                   long recordsBeforeProgress)
                      throws IOException

Throws:

IOException