Package org.apache.druid.segment

Class IndexMergerBase

java.lang.Object

org.apache.druid.segment.IndexMergerBase

All Implemented Interfaces:

IndexMerger

Direct Known Subclasses:

IndexMergerV10, IndexMergerV9

public abstract class IndexMergerBase
extends Object
implements IndexMerger

Shared segment building functionality for segments built by merging IndexableAdapter

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
protected static class	IndexMergerBase.DimensionsSpecInspector
protected static class	IndexMergerBase.IndexMergeResult

Field Summary

Fields

Modifier and Type	Field	Description
protected final SegmentWriteOutMediumFactory	defaultSegmentWriteOutMediumFactory
protected final IndexIO	indexIO
protected final com.fasterxml.jackson.databind.ObjectMapper	mapper

Fields inherited from interface org.apache.druid.segment.IndexMerger

INVALID_ROW, log, SERIALIZER_UTILS, UNLIMITED_MAX_COLUMNS_TO_MERGE

Constructor Summary

Constructors

Modifier	Constructor	Description
protected	IndexMergerBase(com.fasterxml.jackson.databind.ObjectMapper mapper, IndexIO indexIO, SegmentWriteOutMediumFactory defaultSegmentWriteOutMediumFactory)

Method Summary

Modifier and Type	Method	Description
protected int	getIndexColumnCount(List<IndexableAdapter> indexableAdapters)
protected int	getIndexColumnCount(IndexableAdapter indexableAdapter)
protected List<List<IndexableAdapter>>	getMergePhases(List<IndexableAdapter> indexes, int maxColumnsToMerge)
protected abstract void	makeColumn(SegmentFileBuilder segmentFileBuilder, String columnName, ColumnDescriptor serdeficator)	Add a column specified by a ColumnDescriptor to a SegmentFileBuilder
protected Map<String,DimensionHandler>	makeDimensionHandlers(List<String> mergedDimensions, List<ColumnFormat> dimFormats)
protected abstract File	makeIndexFiles(List<IndexableAdapter> adapters, Metadata segmentMetadata, File outDir, ProgressIndicator progress, List<String> mergedDimensionsWithTime, IndexMergerBase.DimensionsSpecInspector dimensionsSpecInspector, List<String> mergedMetrics, Function<List<TransformableRowIterator>,TimeAndDimsIterator> rowMergerFn, IndexSpec indexSpec, SegmentWriteOutMediumFactory segmentWriteOutMediumFactory)	Merge IndexableAdapter and build the segment file
protected TimeAndDimsIterator	makeMergedTimeAndDimsIterator(List<IndexableAdapter> indexes, List<String> mergedDimensionsWithTime, List<String> mergedMetrics, Function<List<TransformableRowIterator>,TimeAndDimsIterator> rowMergerFn, Map<String,DimensionHandler> handlers, List<DimensionMergerV9> mergers)
protected void	makeMetricsColumns(SegmentFileBuilder segmentFileBuilder, ProgressIndicator progress, List<String> mergedMetrics, Map<String,ColumnFormat> metricsTypes, List<GenericColumnSerializer> metWriters, IndexSpec indexSpec, String namePrefix)
protected Metadata	makeProjections(SegmentFileBuilder segmentFileBuilder, List<AggregateProjectionMetadata> projections, List<IndexableAdapter> adapters, IndexSpec indexSpec, SegmentWriteOutMedium segmentWriteOutMedium, ProgressIndicator progress, File segmentBaseDir, Closer closer, Map<String,DimensionMergerV9> parentMergers, Metadata segmentMetadata)
protected void	makeTimeColumn(SegmentFileBuilder segmentFileBuilder, ProgressIndicator progress, GenericColumnSerializer timeWriter, IndexSpec indexSpec, String name)
File	merge(List<IndexableAdapter> indexes, boolean rollup, AggregatorFactory[] metricAggs, File outDir, DimensionsSpec dimensionsSpec, IndexSpec indexSpec, int maxColumnsToMerge)	The indexes here must have the same Metadata, otherwise an error would be thrown.
protected File	merge(List<IndexableAdapter> indexes, boolean rollup, AggregatorFactory[] metricAggs, DimensionsSpec dimensionsSpec, File outDir, IndexSpec indexSpec, ProgressIndicator progress, SegmentWriteOutMediumFactory segmentWriteOutMediumFactory)
protected void	mergeFormat(List<IndexableAdapter> adapters, List<String> mergedDimensions, Map<String,ColumnFormat> metricTypes, List<ColumnFormat> dimFormats)
protected IndexMergerBase.IndexMergeResult	mergeIndexesAndWriteColumns(List<IndexableAdapter> adapters, ProgressIndicator progress, TimeAndDimsIterator timeAndDimsIterator, GenericColumnSerializer timeWriter, ArrayList<GenericColumnSerializer> metricWriters, List<DimensionMergerV9> mergers)	Returns rowNumConversions, if fillRowNumConversions argument is true
File	mergeQueryableIndex(List<QueryableIndex> indexes, boolean rollup, AggregatorFactory[] metricAggs, DimensionsSpec dimensionsSpec, File outDir, IndexSpec indexSpec, IndexSpec indexSpecForIntermediatePersists, ProgressIndicator progress, SegmentWriteOutMediumFactory segmentWriteOutMediumFactory, int maxColumnsToMerge)	Merge a collection of QueryableIndex.
protected File	multiphaseMerge(List<IndexableAdapter> indexes, boolean rollup, AggregatorFactory[] metricAggs, DimensionsSpec dimensionsSpec, File outDir, IndexSpec indexSpec, IndexSpec indexSpecForIntermediatePersists, ProgressIndicator progress, SegmentWriteOutMediumFactory segmentWriteOutMediumFactory, int maxColumnsToMerge)
File	persist(IncrementalIndex index, org.joda.time.Interval dataInterval, File outDir, IndexSpec indexSpec, ProgressIndicator progress, SegmentWriteOutMediumFactory segmentWriteOutMediumFactory)	Persist an IncrementalIndex to disk in such a way that it can be loaded back up as a QueryableIndex.
protected ArrayList<GenericColumnSerializer>	setupMetricsWriters(SegmentWriteOutMedium segmentWriteOutMedium, List<String> mergedMetrics, Map<String,ColumnFormat> metricsTypes, IndexSpec indexSpec, String prefix)
protected GenericColumnSerializer	setupTimeWriter(SegmentWriteOutMedium segmentWriteOutMedium, IndexSpec indexSpec)
protected abstract boolean	shouldStoreEmptyColumns()	Should the segment track columns which were specified in the ingestion spec, but not present in the processed data?
protected void	writeDimValuesAndSetupDimConversion(List<IndexableAdapter> indexes, ProgressIndicator progress, List<String> mergedDimensions, List<DimensionMergerV9> mergers)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface org.apache.druid.segment.IndexMerger

mergeQueryableIndex, mergeQueryableIndex, persist, persist

Field Details

mapper

protected final com.fasterxml.jackson.databind.ObjectMapper mapper

indexIO

protected final IndexIO indexIO

defaultSegmentWriteOutMediumFactory

protected final SegmentWriteOutMediumFactory defaultSegmentWriteOutMediumFactory

Constructor Details

IndexMergerBase

protected IndexMergerBase(com.fasterxml.jackson.databind.ObjectMapper mapper,
 IndexIO indexIO,
 SegmentWriteOutMediumFactory defaultSegmentWriteOutMediumFactory)

Method Details

shouldStoreEmptyColumns

protected abstract boolean shouldStoreEmptyColumns()

Should the segment track columns which were specified in the ingestion spec, but not present in the processed data?

makeIndexFiles

protected abstract File makeIndexFiles(List<IndexableAdapter> adapters,
 @Nullable
 Metadata segmentMetadata,
 File outDir,
 ProgressIndicator progress,
 List<String> mergedDimensionsWithTime,
 IndexMergerBase.DimensionsSpecInspector dimensionsSpecInspector,
 List<String> mergedMetrics,
 Function<List<TransformableRowIterator>,TimeAndDimsIterator> rowMergerFn,
 IndexSpec indexSpec,
 @Nullable
 SegmentWriteOutMediumFactory segmentWriteOutMediumFactory)
                                throws IOException

Merge IndexableAdapter and build the segment file

Throws:

IOException

makeColumn

protected abstract void makeColumn(SegmentFileBuilder segmentFileBuilder,
 String columnName,
 ColumnDescriptor serdeficator)
                            throws IOException

Add a column specified by a ColumnDescriptor to a SegmentFileBuilder

Throws:

IOException

persist

public File persist(IncrementalIndex index,
 org.joda.time.Interval dataInterval,
 File outDir,
 IndexSpec indexSpec,
 ProgressIndicator progress,
 @Nullable
 SegmentWriteOutMediumFactory segmentWriteOutMediumFactory)
             throws IOException

Description copied from interface: IndexMerger

Persist an IncrementalIndex to disk in such a way that it can be loaded back up as a QueryableIndex. This is *not* thread-safe and havoc will ensue if this is called and writes are still occurring on the IncrementalIndex object.

Specified by:

persist in interface IndexMerger

Parameters:

index - the IncrementalIndex to persist

dataInterval - the Interval that the data represents. Typically, this is the same as the interval from the corresponding SegmentId.

outDir - the directory to persist the data to

indexSpec - storage and compression options

progress - an object that will receive progress updates

segmentWriteOutMediumFactory - controls allocation of temporary data structures

Returns:

the index output directory

Throws:

IOException - if an IO error occurs persisting the index

mergeQueryableIndex

public File mergeQueryableIndex(List<QueryableIndex> indexes,
 boolean rollup,
 AggregatorFactory[] metricAggs,
 @Nullable
 DimensionsSpec dimensionsSpec,
 File outDir,
 IndexSpec indexSpec,
 IndexSpec indexSpecForIntermediatePersists,
 ProgressIndicator progress,
 @Nullable
 SegmentWriteOutMediumFactory segmentWriteOutMediumFactory,
 int maxColumnsToMerge)
                         throws IOException

Description copied from interface: IndexMerger

Merge a collection of QueryableIndex.

Specified by:

mergeQueryableIndex in interface IndexMerger

Throws:

IOException

merge

public File merge(List<IndexableAdapter> indexes,
 boolean rollup,
 AggregatorFactory[] metricAggs,
 File outDir,
 DimensionsSpec dimensionsSpec,
 IndexSpec indexSpec,
 int maxColumnsToMerge)
           throws IOException

The indexes here must have the same Metadata, otherwise an error would be thrown.

Specified by:

merge in interface IndexMerger

Throws:

IOException

multiphaseMerge

protected File multiphaseMerge(List<IndexableAdapter> indexes,
 boolean rollup,
 AggregatorFactory[] metricAggs,
 @Nullable
 DimensionsSpec dimensionsSpec,
 File outDir,
 IndexSpec indexSpec,
 IndexSpec indexSpecForIntermediatePersists,
 ProgressIndicator progress,
 @Nullable
 SegmentWriteOutMediumFactory segmentWriteOutMediumFactory,
 int maxColumnsToMerge)
                        throws IOException

Throws:

IOException

getMergePhases

protected List<List<IndexableAdapter>> getMergePhases(List<IndexableAdapter> indexes,
 int maxColumnsToMerge)

getIndexColumnCount

protected int getIndexColumnCount(IndexableAdapter indexableAdapter)

getIndexColumnCount

protected int getIndexColumnCount(List<IndexableAdapter> indexableAdapters)

merge

protected File merge(List<IndexableAdapter> indexes,
 boolean rollup,
 AggregatorFactory[] metricAggs,
 @Nullable
 DimensionsSpec dimensionsSpec,
 File outDir,
 IndexSpec indexSpec,
 ProgressIndicator progress,
 @Nullable
 SegmentWriteOutMediumFactory segmentWriteOutMediumFactory)
              throws IOException

Throws:

IOException

makeProjections

protected Metadata makeProjections(SegmentFileBuilder segmentFileBuilder,
 List<AggregateProjectionMetadata> projections,
 List<IndexableAdapter> adapters,
 IndexSpec indexSpec,
 SegmentWriteOutMedium segmentWriteOutMedium,
 ProgressIndicator progress,
 File segmentBaseDir,
 Closer closer,
 Map<String,DimensionMergerV9> parentMergers,
 Metadata segmentMetadata)
                            throws IOException

Throws:

IOException

makeTimeColumn

protected void makeTimeColumn(SegmentFileBuilder segmentFileBuilder,
 ProgressIndicator progress,
 GenericColumnSerializer timeWriter,
 IndexSpec indexSpec,
 String name)
                       throws IOException

Throws:

IOException

makeMetricsColumns

protected void makeMetricsColumns(SegmentFileBuilder segmentFileBuilder,
 ProgressIndicator progress,
 List<String> mergedMetrics,
 Map<String,ColumnFormat> metricsTypes,
 List<GenericColumnSerializer> metWriters,
 IndexSpec indexSpec,
 String namePrefix)
                           throws IOException

Throws:

IOException

mergeIndexesAndWriteColumns

protected IndexMergerBase.IndexMergeResult mergeIndexesAndWriteColumns(List<IndexableAdapter> adapters,
 ProgressIndicator progress,
 TimeAndDimsIterator timeAndDimsIterator,
 GenericColumnSerializer timeWriter,
 ArrayList<GenericColumnSerializer> metricWriters,
 List<DimensionMergerV9> mergers)
                                                                throws IOException

Returns rowNumConversions, if fillRowNumConversions argument is true

Throws:

IOException

setupTimeWriter

protected GenericColumnSerializer setupTimeWriter(SegmentWriteOutMedium segmentWriteOutMedium,
 IndexSpec indexSpec)
                                           throws IOException

Throws:

IOException

setupMetricsWriters

protected ArrayList<GenericColumnSerializer> setupMetricsWriters(SegmentWriteOutMedium segmentWriteOutMedium,
 List<String> mergedMetrics,
 Map<String,ColumnFormat> metricsTypes,
 IndexSpec indexSpec,
 String prefix)
                                                          throws IOException

Throws:

IOException

writeDimValuesAndSetupDimConversion

protected void writeDimValuesAndSetupDimConversion(List<IndexableAdapter> indexes,
 ProgressIndicator progress,
 List<String> mergedDimensions,
 List<DimensionMergerV9> mergers)
                                            throws IOException

Throws:

IOException

mergeFormat

protected void mergeFormat(List<IndexableAdapter> adapters,
 List<String> mergedDimensions,
 Map<String,ColumnFormat> metricTypes,
 List<ColumnFormat> dimFormats)

makeDimensionHandlers

protected Map<String,DimensionHandler> makeDimensionHandlers(List<String> mergedDimensions,
 List<ColumnFormat> dimFormats)

makeMergedTimeAndDimsIterator

protected TimeAndDimsIterator makeMergedTimeAndDimsIterator(List<IndexableAdapter> indexes,
 List<String> mergedDimensionsWithTime,
 List<String> mergedMetrics,
 Function<List<TransformableRowIterator>,TimeAndDimsIterator> rowMergerFn,
 Map<String,DimensionHandler> handlers,
 List<DimensionMergerV9> mergers)