Package org.apache.beam.runners.spark.structuredstreaming.translation

Class TransformTranslator.Context

java.lang.Object
org.apache.beam.runners.spark.structuredstreaming.translation.TransformTranslator.Context
All Implemented Interfaces:
EncoderProvider, PipelineTranslator.TranslationState
Enclosing class:
TransformTranslator<InT extends org.apache.beam.sdk.values.PInput,OutT extends org.apache.beam.sdk.values.POutput,TransformT extends org.apache.beam.sdk.transforms.PTransform<InT,OutT>>
protected class TransformTranslator.Context extends Object implements PipelineTranslator.TranslationState
Available mutable context to translate a PTransform. The context is backed by the shared PipelineTranslator.TranslationState of the PipelineTranslator.

  • Nested Class Summary

    Nested classes/interfaces inherited from interface org.apache.beam.runners.spark.structuredstreaming.translation.helpers.EncoderProvider

    EncoderProvider.Factory<T>

  • Method Summary

    Modifier and Type
    Method
    Description
    <T> org.apache.spark.broadcast.Broadcast<T>
    broadcast(T value)
     
    <T> org.apache.spark.sql.Dataset<org.apache.beam.sdk.values.WindowedValue<T>>
    createDataset(List<org.apache.beam.sdk.values.WindowedValue<T>> data, org.apache.spark.sql.Encoder<org.apache.beam.sdk.values.WindowedValue<T>> enc)
     
    <T> org.apache.spark.sql.Encoder<T>
    encoderOf(org.apache.beam.sdk.coders.Coder<T> coder, EncoderProvider.Factory<T> factory)
     
    org.apache.beam.sdk.runners.AppliedPTransform<InT,OutT,TransformT>
    getCurrentTransform()
     
    <T> org.apache.spark.sql.Dataset<org.apache.beam.sdk.values.WindowedValue<T>>
    getDataset(org.apache.beam.sdk.values.PCollection<T> pCollection)
     
    InT
    getInput()
     
    Map<org.apache.beam.sdk.values.TupleTag<?>,org.apache.beam.sdk.values.PCollection<?>>
    getInputs()
     
    org.apache.beam.sdk.options.PipelineOptions
    getOptions()
     
    Supplier<org.apache.beam.sdk.options.PipelineOptions>
    getOptionsSupplier()
     
    OutT
    getOutput()
     
    <T> org.apache.beam.sdk.values.PCollection<T>
    getOutput(org.apache.beam.sdk.values.TupleTag<T> tag)
     
    Map<org.apache.beam.sdk.values.TupleTag<?>,org.apache.beam.sdk.values.PCollection<?>>
    getOutputs()
     
    <T> org.apache.spark.broadcast.Broadcast<SideInputValues<T>>
    getSideInputBroadcast(org.apache.beam.sdk.values.PCollection<T> pCollection, SideInputValues.Loader<T> loader)
     
    org.apache.spark.sql.SparkSession
    getSparkSession()
     
    boolean
    isLeaf(org.apache.beam.sdk.values.PCollection<?> pCollection)
     
    <T> void
    putDataset(org.apache.beam.sdk.values.PCollection<T> pCollection, org.apache.spark.sql.Dataset<org.apache.beam.sdk.values.WindowedValue<T>> dataset, boolean cache)
     
    <InputT, T>
    void
    putUnresolved(org.apache.beam.sdk.values.PCollection<T> out, PipelineTranslator.UnresolvedTranslation<InputT,T> unresolved)
     
    <T1, T2> org.apache.spark.sql.Encoder<scala.Tuple2<T1,T2>>
    tupleEncoder(org.apache.spark.sql.Encoder<T1> e1, org.apache.spark.sql.Encoder<T2> e2)
     
    <T> org.apache.spark.sql.Encoder<org.apache.beam.sdk.values.WindowedValue<T>>
    windowedEncoder(org.apache.beam.sdk.coders.Coder<T> coder)
     
    <T, W extends org.apache.beam.sdk.transforms.windowing.BoundedWindow>
    org.apache.spark.sql.Encoder<org.apache.beam.sdk.values.WindowedValue<T>>
    windowedEncoder(org.apache.beam.sdk.coders.Coder<T> coder, org.apache.beam.sdk.coders.Coder<W> windowCoder)
     
    <T> org.apache.spark.sql.Encoder<org.apache.beam.sdk.values.WindowedValue<T>>
    windowedEncoder(org.apache.spark.sql.Encoder<T> enc)
     
    org.apache.spark.sql.Encoder<org.apache.beam.sdk.transforms.windowing.BoundedWindow>
    windowEncoder()
     

    Methods inherited from class java.lang.Object

    clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

    Methods inherited from interface org.apache.beam.runners.spark.structuredstreaming.translation.helpers.EncoderProvider

    encoderFactory, encoderOf, keyEncoderOf, kvEncoderOf, valueEncoderOf

    Methods inherited from interface org.apache.beam.runners.spark.structuredstreaming.translation.PipelineTranslator.TranslationState

    putDataset

  • Method Details

    • getInput

      public InT getInput()

    • getInputs

      public Map<org.apache.beam.sdk.values.TupleTag<?>,org.apache.beam.sdk.values.PCollection<?>> getInputs()

    • getOutputs

      public Map<org.apache.beam.sdk.values.TupleTag<?>,org.apache.beam.sdk.values.PCollection<?>> getOutputs()

    • getOutput

      public OutT getOutput()

    • getOutput

      public <T> org.apache.beam.sdk.values.PCollection<T> getOutput(org.apache.beam.sdk.values.TupleTag<T> tag)

    • getCurrentTransform

      public org.apache.beam.sdk.runners.AppliedPTransform<InT,OutT,TransformT> getCurrentTransform()

    • getDataset

      public <T> org.apache.spark.sql.Dataset<org.apache.beam.sdk.values.WindowedValue<T>> getDataset(org.apache.beam.sdk.values.PCollection<T> pCollection)
      Specified by:
      getDataset in interface PipelineTranslator.TranslationState

    • getSideInputBroadcast

      public <T> org.apache.spark.broadcast.Broadcast<SideInputValues<T>> getSideInputBroadcast(org.apache.beam.sdk.values.PCollection<T> pCollection, SideInputValues.Loader<T> loader)
      Specified by:
      getSideInputBroadcast in interface PipelineTranslator.TranslationState

    • putDataset

      public <T> void putDataset(org.apache.beam.sdk.values.PCollection<T> pCollection, org.apache.spark.sql.Dataset<org.apache.beam.sdk.values.WindowedValue<T>> dataset, boolean cache)
      Specified by:
      putDataset in interface PipelineTranslator.TranslationState

    • putUnresolved

      public <InputT, T> void putUnresolved(org.apache.beam.sdk.values.PCollection<T> out, PipelineTranslator.UnresolvedTranslation<InputT,T> unresolved)
      Specified by:
      putUnresolved in interface PipelineTranslator.TranslationState

    • isLeaf

      public boolean isLeaf(org.apache.beam.sdk.values.PCollection<?> pCollection)
      Specified by:
      isLeaf in interface PipelineTranslator.TranslationState

    • getOptionsSupplier

      public Supplier<org.apache.beam.sdk.options.PipelineOptions> getOptionsSupplier()
      Specified by:
      getOptionsSupplier in interface PipelineTranslator.TranslationState

    • getOptions

      public org.apache.beam.sdk.options.PipelineOptions getOptions()
      Specified by:
      getOptions in interface PipelineTranslator.TranslationState

    • createDataset

      public <T> org.apache.spark.sql.Dataset<org.apache.beam.sdk.values.WindowedValue<T>> createDataset(List<org.apache.beam.sdk.values.WindowedValue<T>> data, org.apache.spark.sql.Encoder<org.apache.beam.sdk.values.WindowedValue<T>> enc)

    • broadcast

      public <T> org.apache.spark.broadcast.Broadcast<T> broadcast(T value)

    • getSparkSession

      public org.apache.spark.sql.SparkSession getSparkSession()
      Specified by:
      getSparkSession in interface PipelineTranslator.TranslationState

    • encoderOf

      public <T> org.apache.spark.sql.Encoder<T> encoderOf(org.apache.beam.sdk.coders.Coder<T> coder, EncoderProvider.Factory<T> factory)
      Specified by:
      encoderOf in interface EncoderProvider

    • tupleEncoder

      public <T1, T2> org.apache.spark.sql.Encoder<scala.Tuple2<T1,T2>> tupleEncoder(org.apache.spark.sql.Encoder<T1> e1, org.apache.spark.sql.Encoder<T2> e2)

    • windowedEncoder

      public <T> org.apache.spark.sql.Encoder<org.apache.beam.sdk.values.WindowedValue<T>> windowedEncoder(org.apache.beam.sdk.coders.Coder<T> coder)

    • windowedEncoder

      public <T> org.apache.spark.sql.Encoder<org.apache.beam.sdk.values.WindowedValue<T>> windowedEncoder(org.apache.spark.sql.Encoder<T> enc)

    • windowedEncoder

      public <T, W extends org.apache.beam.sdk.transforms.windowing.BoundedWindow> org.apache.spark.sql.Encoder<org.apache.beam.sdk.values.WindowedValue<T>> windowedEncoder(org.apache.beam.sdk.coders.Coder<T> coder, org.apache.beam.sdk.coders.Coder<W> windowCoder)

    • windowEncoder

      public org.apache.spark.sql.Encoder<org.apache.beam.sdk.transforms.windowing.BoundedWindow> windowEncoder()