Serialized Form

Package org.apache.beam.runners.spark.coders

Class org.apache.beam.runners.spark.coders.CoderHelpers.FromByteFunction

class FromByteFunction extends Object implements Serializable

Serialized Fields

keyCoder

org.apache.beam.sdk.coders.Coder<K> keyCoder

valueCoder

org.apache.beam.sdk.coders.Coder<V> valueCoder

Package org.apache.beam.runners.spark.examples

Class org.apache.beam.runners.spark.examples.WordCount.CountWords

class CountWords extends org.apache.beam.sdk.transforms.PTransform<org.apache.beam.sdk.values.PCollection<String>,org.apache.beam.sdk.values.PCollection<org.apache.beam.sdk.values.KV<String,Long>>> implements Serializable

Class org.apache.beam.runners.spark.examples.WordCount.FormatAsTextFn

class FormatAsTextFn extends org.apache.beam.sdk.transforms.SimpleFunction<org.apache.beam.sdk.values.KV<String,Long>,String> implements Serializable

Package org.apache.beam.runners.spark.io

Class org.apache.beam.runners.spark.io.ConsoleIO.Write.Unbound

class Unbound extends org.apache.beam.sdk.transforms.PTransform<org.apache.beam.sdk.values.PCollection<T>,org.apache.beam.sdk.values.PDone> implements Serializable

Serialized Fields

num

int num

Class org.apache.beam.runners.spark.io.CreateStream

class CreateStream extends org.apache.beam.sdk.transforms.PTransform<org.apache.beam.sdk.values.PBegin,org.apache.beam.sdk.values.PCollection<T>> implements Serializable

Serialized Fields

batchDuration

org.joda.time.Duration batchDuration

batches

Queue<Iterable<org.apache.beam.sdk.values.TimestampedValue<T>>> batches

coder

org.apache.beam.sdk.coders.Coder<T> coder

forceWatermarkSync

boolean forceWatermarkSync

initialSystemTime

org.joda.time.Instant initialSystemTime

lowWatermark

org.joda.time.Instant lowWatermark

times

Deque<GlobalWatermarkHolder.SparkWatermarks> times

Class org.apache.beam.runners.spark.io.EmptyCheckpointMark

class EmptyCheckpointMark extends Object implements Serializable

Class org.apache.beam.runners.spark.io.MicrobatchSource

class MicrobatchSource extends org.apache.beam.sdk.io.Source<T> implements Serializable

Serialized Fields

maxNumRecords

long maxNumRecords

maxReadTime

org.joda.time.Duration maxReadTime

numInitialSplits

int numInitialSplits

readerCacheInterval

double readerCacheInterval

source

org.apache.beam.sdk.io.UnboundedSource<T,CheckpointMarkT extends org.apache.beam.sdk.io.UnboundedSource.CheckpointMark> source

sourceId

int sourceId

splitId

int splitId

Class org.apache.beam.runners.spark.io.SourceRDD.Bounded

class Bounded extends org.apache.spark.rdd.RDD<org.apache.beam.sdk.values.WindowedValue<T>> implements Serializable

Serialized Fields

bundleSize

long bundleSize

metricsAccum

MetricsContainerStepMapAccumulator metricsAccum

numPartitions

int numPartitions

options

org.apache.beam.runners.core.construction.SerializablePipelineOptions options

source

org.apache.beam.sdk.io.BoundedSource<T> source

stepName

String stepName

Class org.apache.beam.runners.spark.io.SourceRDD.Unbounded

class Unbounded extends org.apache.spark.rdd.RDD<scala.Tuple2<org.apache.beam.sdk.io.Source<T>,CheckpointMarkT extends org.apache.beam.sdk.io.UnboundedSource.CheckpointMark>> implements Serializable

Serialized Fields

microbatchSource

MicrobatchSource<T,CheckpointMarkT extends org.apache.beam.sdk.io.UnboundedSource.CheckpointMark> microbatchSource

options

org.apache.beam.runners.core.construction.SerializablePipelineOptions options

partitioner

org.apache.spark.Partitioner partitioner

Class org.apache.beam.runners.spark.io.SparkUnboundedSource.Metadata

class Metadata extends Object implements Serializable

Serialized Fields

highWatermark

org.joda.time.Instant highWatermark

lowWatermark

org.joda.time.Instant lowWatermark

metricsContainers

org.apache.beam.runners.core.metrics.MetricsContainerStepMap metricsContainers

numRecords

long numRecords

readDurationMillis

long readDurationMillis

Package org.apache.beam.runners.spark.metrics

Class org.apache.beam.runners.spark.metrics.MetricsContainerStepMapAccumulator

class MetricsContainerStepMapAccumulator extends org.apache.spark.util.AccumulatorV2<org.apache.beam.runners.core.metrics.MetricsContainerStepMap,org.apache.beam.runners.core.metrics.MetricsContainerStepMap> implements Serializable

Serialized Fields

value

org.apache.beam.runners.core.metrics.MetricsContainerStepMap value

Package org.apache.beam.runners.spark.stateful

Class org.apache.beam.runners.spark.stateful.SparkGroupAlsoByWindowViaWindowSet

class SparkGroupAlsoByWindowViaWindowSet extends Object implements Serializable

Class org.apache.beam.runners.spark.stateful.StateAndTimers

class StateAndTimers extends Object implements Serializable

Package org.apache.beam.runners.spark.structuredstreaming.examples

Class org.apache.beam.runners.spark.structuredstreaming.examples.WordCount.CountWords

class CountWords extends org.apache.beam.sdk.transforms.PTransform<org.apache.beam.sdk.values.PCollection<String>,org.apache.beam.sdk.values.PCollection<org.apache.beam.sdk.values.KV<String,Long>>> implements Serializable

Class org.apache.beam.runners.spark.structuredstreaming.examples.WordCount.FormatAsTextFn

class FormatAsTextFn extends org.apache.beam.sdk.transforms.SimpleFunction<org.apache.beam.sdk.values.KV<String,Long>,String> implements Serializable

Package org.apache.beam.runners.spark.structuredstreaming.metrics

Class org.apache.beam.runners.spark.structuredstreaming.metrics.MetricsAccumulator

class MetricsAccumulator extends org.apache.spark.util.AccumulatorV2<org.apache.beam.runners.core.metrics.MetricsContainerStepMap,org.apache.beam.runners.core.metrics.MetricsContainerStepMap> implements Serializable

Serialized Fields

value

org.apache.beam.runners.core.metrics.MetricsContainerStepMap value

Package org.apache.beam.runners.spark.structuredstreaming.translation.batch.functions

Class org.apache.beam.runners.spark.structuredstreaming.translation.batch.functions.GroupAlsoByWindowViaOutputBufferFn

class GroupAlsoByWindowViaOutputBufferFn extends Object implements Serializable

Serialized Fields

options

Supplier<org.apache.beam.sdk.options.PipelineOptions> options

reduceFn

org.apache.beam.runners.core.SystemReduceFn<K,InputT,Iterable<InputT>,Iterable<InputT>,W extends org.apache.beam.sdk.transforms.windowing.BoundedWindow> reduceFn

stateInternalsFactory

org.apache.beam.runners.core.StateInternalsFactory<K> stateInternalsFactory

windowingStrategy

org.apache.beam.sdk.values.WindowingStrategy<?,W extends org.apache.beam.sdk.transforms.windowing.BoundedWindow> windowingStrategy

Class org.apache.beam.runners.spark.structuredstreaming.translation.batch.functions.SideInputValues.BaseSideInputValues

class BaseSideInputValues extends Object implements Serializable

Serialized Fields

binaryValues

byte[][] binaryValues

coder

org.apache.beam.sdk.coders.Coder<BinaryT> coder

Class org.apache.beam.runners.spark.structuredstreaming.translation.batch.functions.SideInputValues.ByWindow

class ByWindow extends SideInputValues.BaseSideInputValues<org.apache.beam.sdk.values.WindowedValue<T>,Map<org.apache.beam.sdk.transforms.windowing.BoundedWindow,List<T>>,T> implements Serializable

Class org.apache.beam.runners.spark.structuredstreaming.translation.batch.functions.SideInputValues.Global

class Global extends SideInputValues.BaseSideInputValues<T,List<T>,T> implements Serializable

Class org.apache.beam.runners.spark.structuredstreaming.translation.batch.functions.SparkSideInputReader

class SparkSideInputReader extends Object implements Serializable

Serialized Fields

sideInputs

Map<String,org.apache.spark.broadcast.Broadcast<SideInputValues<?>>> sideInputs

Package org.apache.beam.runners.spark.structuredstreaming.translation.utils

Package org.apache.beam.runners.spark.translation

Class org.apache.beam.runners.spark.translation.BoundedDataset

class BoundedDataset extends Object implements Serializable

Serialized Fields

clientBytes

List<byte[]> clientBytes

coder

org.apache.beam.sdk.coders.Coder<T> coder

rdd

org.apache.spark.api.java.JavaRDD<org.apache.beam.sdk.values.WindowedValue<T>> rdd

windowedValues

Iterable<org.apache.beam.sdk.values.WindowedValue<T>> windowedValues

Class org.apache.beam.runners.spark.translation.MultiDoFnFunction

class MultiDoFnFunction extends Object implements Serializable

Serialized Fields

additionalOutputTags

List<org.apache.beam.sdk.values.TupleTag<?>> additionalOutputTags

doFn

org.apache.beam.sdk.transforms.DoFn<InputT,OutputT> doFn

doFnSchemaInformation

org.apache.beam.sdk.transforms.DoFnSchemaInformation doFnSchemaInformation

inputCoder

org.apache.beam.sdk.coders.Coder<InputT> inputCoder

mainOutputTag

org.apache.beam.sdk.values.TupleTag<OutputT> mainOutputTag

metricsAccum

MetricsContainerStepMapAccumulator metricsAccum

options

org.apache.beam.runners.core.construction.SerializablePipelineOptions options

outputCoders

Map<org.apache.beam.sdk.values.TupleTag<?>,org.apache.beam.sdk.coders.Coder<?>> outputCoders

sideInputMapping

Map<String,org.apache.beam.sdk.values.PCollectionView<?>> sideInputMapping

sideInputs

Map<org.apache.beam.sdk.values.TupleTag<?>,org.apache.beam.sdk.values.KV<org.apache.beam.sdk.values.WindowingStrategy<?,?>,SideInputBroadcast<?>>> sideInputs

stateful

boolean stateful

stepName

String stepName

useBoundedConcurrentOutput

boolean useBoundedConcurrentOutput

useStreamingSideInput

boolean useStreamingSideInput

windowingStrategy

org.apache.beam.sdk.values.WindowingStrategy<?,?> windowingStrategy

Class org.apache.beam.runners.spark.translation.ReifyTimestampsAndWindowsFunction

class ReifyTimestampsAndWindowsFunction extends Object implements Serializable

Class org.apache.beam.runners.spark.translation.SideInputMetadata

class SideInputMetadata extends Object implements Serializable

Serialized Fields

coder

org.apache.beam.sdk.coders.Coder<Iterable<org.apache.beam.sdk.values.WindowedValue<?>>> coder

data

byte[] data

type

SparkPCollectionView.Type type

Class org.apache.beam.runners.spark.translation.SingleEmitInputDStream

class SingleEmitInputDStream extends org.apache.spark.streaming.dstream.ConstantInputDStream<T> implements Serializable

Serialized Fields

emitted

boolean emitted

Class org.apache.beam.runners.spark.translation.SparkAssignWindowFn

class SparkAssignWindowFn extends Object implements Serializable

Serialized Fields

fn

org.apache.beam.sdk.transforms.windowing.WindowFn<? super T,W extends org.apache.beam.sdk.transforms.windowing.BoundedWindow> fn

Class org.apache.beam.runners.spark.translation.SparkCombineFn

class SparkCombineFn extends Object implements Serializable

Serialized Fields

combineFn

org.apache.beam.sdk.transforms.CombineWithContext.CombineFnWithContext<ValueT,AccumT,OutputT> combineFn

defaultNonMergingCombineStrategy

SparkCombineFn.WindowedAccumulator.Type defaultNonMergingCombineStrategy

globalCombine

boolean globalCombine

options

org.apache.beam.runners.core.construction.SerializablePipelineOptions options

sideInputs

Map<org.apache.beam.sdk.values.TupleTag<?>,org.apache.beam.sdk.values.KV<org.apache.beam.sdk.values.WindowingStrategy<?,?>,SideInputBroadcast<?>>> sideInputs

toValue

org.apache.spark.api.java.function.Function<InputT,ValueT> toValue

windowComparator

Comparator<org.apache.beam.sdk.transforms.windowing.BoundedWindow> windowComparator

windowingStrategy

org.apache.beam.sdk.values.WindowingStrategy<?,org.apache.beam.sdk.transforms.windowing.BoundedWindow> windowingStrategy

Class org.apache.beam.runners.spark.translation.SparkExecutableStageContextFactory

class SparkExecutableStageContextFactory extends Object implements Serializable

Class org.apache.beam.runners.spark.translation.SparkPCollectionView

class SparkPCollectionView extends Object implements Serializable

Serialized Fields

pviews

Map<org.apache.beam.sdk.values.PCollectionView<?>,SideInputMetadata> pviews

Class org.apache.beam.runners.spark.translation.TranslationUtils.CombineGroupedValues

class CombineGroupedValues extends Object implements Serializable

Serialized Fields

fn

SparkCombineFn<org.apache.beam.sdk.values.KV<K,InputT>,InputT,?,OutputT> fn

Class org.apache.beam.runners.spark.translation.TranslationUtils.TupleTagFilter

class TupleTagFilter extends Object implements Serializable

Serialized Fields

tag

org.apache.beam.sdk.values.TupleTag<V> tag

Class org.apache.beam.runners.spark.translation.ValueAndCoderLazySerializable

class ValueAndCoderLazySerializable extends Object implements Serializable

Serialization Methods

readObject

private void readObject(ObjectInputStream in)
                 throws IOException

Throws:

IOException

writeObject

private void writeObject(ObjectOutputStream out)
                  throws IOException

Throws:

IOException

Serialized Fields

coderOrBytes

Object coderOrBytes

value

T value

Package org.apache.beam.runners.spark.translation.streaming

Class org.apache.beam.runners.spark.translation.streaming.CreateStreamingSparkView

class CreateStreamingSparkView extends org.apache.beam.sdk.transforms.PTransform<org.apache.beam.sdk.values.PCollection<ElemT>,org.apache.beam.sdk.values.PCollection<ElemT>> implements Serializable

Serialized Fields

view

org.apache.beam.sdk.values.PCollectionView<ViewT> view

Class org.apache.beam.runners.spark.translation.streaming.CreateStreamingSparkView.CreateSparkPCollectionView

class CreateSparkPCollectionView extends org.apache.beam.sdk.transforms.PTransform<org.apache.beam.sdk.values.PCollection<List<ElemT>>,org.apache.beam.sdk.values.PCollection<List<ElemT>>> implements Serializable

Serialized Fields

view

org.apache.beam.sdk.values.PCollectionView<ViewT> view

Class org.apache.beam.runners.spark.translation.streaming.ParDoStateUpdateFn

class ParDoStateUpdateFn extends scala.runtime.AbstractFunction3<ByteArray,scala.Option<byte[]>,org.apache.spark.streaming.State<StateAndTimers>,List<scala.Tuple2<org.apache.beam.sdk.values.TupleTag<?>,byte[]>>> implements Serializable

Serialized Fields

additionalOutputTags

List<org.apache.beam.sdk.values.TupleTag<?>> additionalOutputTags

doFn

org.apache.beam.sdk.transforms.DoFn<InputT extends org.apache.beam.sdk.values.KV<KeyT,ValueT>,OutputT> doFn

doFnSchemaInformation

org.apache.beam.sdk.transforms.DoFnSchemaInformation doFnSchemaInformation

inputCoder

org.apache.beam.sdk.coders.Coder<InputT extends org.apache.beam.sdk.values.KV<KeyT,ValueT>> inputCoder

keyCoder

org.apache.beam.sdk.coders.Coder<KeyT> keyCoder

mainOutputTag

org.apache.beam.sdk.values.TupleTag<?> mainOutputTag

metricsAccum

MetricsContainerStepMapAccumulator metricsAccum

options

org.apache.beam.runners.core.construction.SerializablePipelineOptions options

outputCoders

Map<org.apache.beam.sdk.values.TupleTag<?>,org.apache.beam.sdk.coders.Coder<?>> outputCoders

sideInputMapping

Map<String,org.apache.beam.sdk.values.PCollectionView<?>> sideInputMapping

sideInputs

Map<org.apache.beam.sdk.values.TupleTag<?>,org.apache.beam.sdk.values.KV<org.apache.beam.sdk.values.WindowingStrategy<?,?>,SideInputBroadcast<?>>> sideInputs

sourceIds

List<Integer> sourceIds

stepName

String stepName

timerDataCoder

org.apache.beam.runners.core.TimerInternals.TimerDataCoderV2 timerDataCoder

useStreamingSideInput

boolean useStreamingSideInput

watermarks

Map<Integer,GlobalWatermarkHolder.SparkWatermarks> watermarks

windowingStrategy

org.apache.beam.sdk.values.WindowingStrategy<?,?> windowingStrategy

wvCoder

org.apache.beam.sdk.values.WindowedValues.FullWindowedValueCoder<ValueT> wvCoder

Class org.apache.beam.runners.spark.translation.streaming.SparkRunnerStreamingContextFactory

class SparkRunnerStreamingContextFactory extends Object implements Serializable

Class org.apache.beam.runners.spark.translation.streaming.StatefulStreamingParDoEvaluator

class StatefulStreamingParDoEvaluator extends Object implements Serializable

Class org.apache.beam.runners.spark.translation.streaming.TestDStream

class TestDStream extends org.apache.spark.streaming.dstream.InputDStream<org.apache.beam.sdk.values.WindowedValue<T>> implements Serializable

Serialized Fields

coder

org.apache.beam.sdk.coders.Coder<org.apache.beam.sdk.values.WindowedValue<T>> coder

currentEventIndex

int currentEventIndex

insertEmptyBatch

boolean insertEmptyBatch

lastValidTimeMs

long lastValidTimeMs

lastWatermark

org.joda.time.Instant lastWatermark

Class org.apache.beam.runners.spark.translation.streaming.UnboundedDataset

class UnboundedDataset extends Object implements Serializable

Serialized Fields

dStream

org.apache.spark.streaming.api.java.JavaDStream<org.apache.beam.sdk.values.WindowedValue<T>> dStream

streamSources

List<Integer> streamSources

Package org.apache.beam.runners.spark.util

Class org.apache.beam.runners.spark.util.ByteArray

class ByteArray extends Object implements Serializable

Serialized Fields

value

byte[] value

Class org.apache.beam.runners.spark.util.GlobalWatermarkHolder.SparkWatermarks

class SparkWatermarks extends Object implements Serializable

Serialized Fields

highWatermark

org.joda.time.Instant highWatermark

lowWatermark

org.joda.time.Instant lowWatermark

synchronizedProcessingTime

org.joda.time.Instant synchronizedProcessingTime

Class org.apache.beam.runners.spark.util.SideInputBroadcast

class SideInputBroadcast extends Object implements Serializable

Serialized Fields

bcast

org.apache.spark.broadcast.Broadcast<byte[]> bcast

coder

org.apache.beam.sdk.coders.Coder<T> coder

sparkPCollectionViewType

SparkPCollectionView.Type sparkPCollectionViewType

Class org.apache.beam.runners.spark.util.TimerUtils.TimerMarker

class TimerMarker extends Object implements Serializable

serialVersionUID:

1L