Packages

case class MemoryStream[A](id: Int, sparkSession: SparkSession, numPartitions: Option[Int] = None)(implicit evidence$6: Encoder[A]) extends MemoryStreamBaseClass[A] with Product with Serializable

A Source that produces value stored in memory as they are added by the user. This Source is intended for use in unit tests as it can only replay data when the object is still available.

If numPartitions is provided, the rows will be redistributed to the given number of partitions.

Linear Supertypes
Serializable, Product, Equals, MemoryStreamBaseClass[A], Logging, SupportsTriggerAvailableNow, SupportsAdmissionControl, MicroBatchStream, MemoryStreamBase[A], SparkDataStream, AnyRef, Any
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. MemoryStream
  2. Serializable
  3. Product
  4. Equals
  5. MemoryStreamBaseClass
  6. Logging
  7. SupportsTriggerAvailableNow
  8. SupportsAdmissionControl
  9. MicroBatchStream
  10. MemoryStreamBase
  11. SparkDataStream
  12. AnyRef
  13. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. Protected

Instance Constructors

  1. new MemoryStream(id: Int, sparkSession: SparkSession, numPartitions: Option[Int] = None)(implicit arg0: Encoder[A])

Type Members

  1. implicit class LogStringContext extends AnyRef
    Definition Classes
    Logging

Value Members

  1. final def !=(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  2. final def ##: Int
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  4. def MDC(key: LogKey, value: Any): MDC
    Attributes
    protected
    Definition Classes
    Logging
  5. def addData(data: IterableOnce[A]): Offset
    Definition Classes
    MemoryStreamBaseClass → MemoryStreamBase
  6. def addData(data: A*): connector.read.streaming.Offset
    Definition Classes
    MemoryStreamBase
  7. final def asInstanceOf[T0]: T0
    Definition Classes
    Any
  8. val attributes: Seq[AttributeReference]
    Attributes
    protected
    Definition Classes
    MemoryStreamBase
  9. val batches: ListBuffer[Array[UnsafeRow]]

    All batches from lastCommittedOffset + 1 to currentOffset, inclusive.

    All batches from lastCommittedOffset + 1 to currentOffset, inclusive. Stored in a ListBuffer to facilitate removing committed batches.

    Attributes
    protected
    Definition Classes
    MemoryStreamBaseClass
  10. def clone(): AnyRef
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.CloneNotSupportedException]) @IntrinsicCandidate() @native()
  11. def commit(end: connector.read.streaming.Offset): Unit
    Definition Classes
    MemoryStreamBaseClass → MemoryStreamBase → SparkDataStream
  12. def createReaderFactory(): PartitionReaderFactory
    Definition Classes
    MemoryStreamBaseClass → MicroBatchStream
  13. var currentOffset: LongOffset
    Attributes
    protected
    Definition Classes
    MemoryStreamBaseClass
  14. def deserializeOffset(json: String): connector.read.streaming.Offset
    Definition Classes
    MemoryStreamBaseClass → MemoryStreamBase → SparkDataStream
  15. val encoder: ExpressionEncoder[A]
    Definition Classes
    MemoryStreamBase
  16. final def eq(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  17. def fullSchema(): StructType
    Definition Classes
    MemoryStreamBase
  18. final def getClass(): Class[_ <: AnyRef]
    Definition Classes
    AnyRef → Any
    Annotations
    @IntrinsicCandidate() @native()
  19. def getDefaultReadLimit(): ReadLimit
    Definition Classes
    SupportsAdmissionControl
  20. val id: Int
  21. def initialOffset(): connector.read.streaming.Offset
    Definition Classes
    MemoryStreamBaseClass → MemoryStreamBase → SparkDataStream
  22. def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean
    Attributes
    protected
    Definition Classes
    Logging
  23. def initializeLogIfNecessary(isInterpreter: Boolean): Unit
    Attributes
    protected
    Definition Classes
    Logging
  24. final def isInstanceOf[T0]: Boolean
    Definition Classes
    Any
  25. def isTraceEnabled(): Boolean
    Attributes
    protected
    Definition Classes
    Logging
  26. var lastOffsetCommitted: LongOffset

    Last offset that was discarded, or -1 if no commits have occurred.

    Last offset that was discarded, or -1 if no commits have occurred. Note that the value -1 is used in calculations below and isn't just an arbitrary constant.

    Attributes
    protected
    Definition Classes
    MemoryStreamBaseClass
  27. def latestOffset(startOffset: connector.read.streaming.Offset, limit: ReadLimit): connector.read.streaming.Offset
    Definition Classes
    MemoryStreamBaseClass → SupportsAdmissionControl
  28. def latestOffset(): connector.read.streaming.Offset
    Definition Classes
    MemoryStreamBaseClass → MicroBatchStream
  29. def log: Logger
    Attributes
    protected
    Definition Classes
    Logging
  30. def logBasedOnLevel(level: Level)(f: => MessageWithContext): Unit
    Attributes
    protected
    Definition Classes
    Logging
  31. def logDebug(msg: => String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  32. def logDebug(entry: LogEntry, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  33. def logDebug(entry: LogEntry): Unit
    Attributes
    protected
    Definition Classes
    Logging
  34. def logDebug(msg: => String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  35. def logError(msg: => String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  36. def logError(entry: LogEntry, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  37. def logError(entry: LogEntry): Unit
    Attributes
    protected
    Definition Classes
    Logging
  38. def logError(msg: => String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  39. def logInfo(msg: => String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  40. def logInfo(entry: LogEntry, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  41. def logInfo(entry: LogEntry): Unit
    Attributes
    protected
    Definition Classes
    Logging
  42. def logInfo(msg: => String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  43. def logName: String
    Attributes
    protected
    Definition Classes
    Logging
  44. def logTrace(msg: => String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  45. def logTrace(entry: LogEntry, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  46. def logTrace(entry: LogEntry): Unit
    Attributes
    protected
    Definition Classes
    Logging
  47. def logTrace(msg: => String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  48. def logWarning(msg: => String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  49. def logWarning(entry: LogEntry, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  50. def logWarning(entry: LogEntry): Unit
    Attributes
    protected
    Definition Classes
    Logging
  51. def logWarning(msg: => String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  52. val logicalPlan: LogicalPlan
    Attributes
    protected
    Definition Classes
    MemoryStreamBase
  53. final def ne(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  54. final def notify(): Unit
    Definition Classes
    AnyRef
    Annotations
    @IntrinsicCandidate() @native()
  55. final def notifyAll(): Unit
    Definition Classes
    AnyRef
    Annotations
    @IntrinsicCandidate() @native()
  56. val numPartitions: Option[Int]
  57. val output: Seq[Attribute]
    Attributes
    protected
    Definition Classes
    MemoryStreamBaseClass
  58. def planInputPartitions(start: connector.read.streaming.Offset, end: connector.read.streaming.Offset): Array[InputPartition]
    Definition Classes
    MemoryStreamBaseClass → MicroBatchStream
  59. def prepareForTriggerAvailableNow(): Unit
    Definition Classes
    MemoryStreamBaseClass → SupportsTriggerAvailableNow
  60. def productElementNames: Iterator[String]
    Definition Classes
    Product
  61. def reportLatestOffset(): connector.read.streaming.Offset
    Definition Classes
    SupportsAdmissionControl
  62. def reset(): Unit
    Definition Classes
    MemoryStreamBaseClass
  63. val sparkSession: SparkSession
  64. var startOffset: LongOffset
    Attributes
    protected
    Definition Classes
    MemoryStreamBaseClass
  65. def stop(): Unit
    Definition Classes
    MemoryStreamBaseClass → SparkDataStream
  66. final def synchronized[T0](arg0: => T0): T0
    Definition Classes
    AnyRef
  67. def toDF(): classic.DataFrame
    Definition Classes
    MemoryStreamBase
  68. def toDS(): classic.Dataset[A]
    Definition Classes
    MemoryStreamBase
  69. lazy val toRow: Serializer[A]
    Attributes
    protected
    Definition Classes
    MemoryStreamBase
  70. def toString(): String
    Definition Classes
    MemoryStreamBaseClass → AnyRef → Any
  71. final def wait(arg0: Long, arg1: Int): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException])
  72. final def wait(arg0: Long): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException]) @native()
  73. final def wait(): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException])
  74. def withLogContext(context: Map[String, String])(body: => Unit): Unit
    Attributes
    protected
    Definition Classes
    Logging

Deprecated Value Members

  1. def finalize(): Unit
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.Throwable]) @Deprecated
    Deprecated

    (Since version 9)

Inherited from Serializable

Inherited from Product

Inherited from Equals

Inherited from MemoryStreamBaseClass[A]

Inherited from Logging

Inherited from SupportsTriggerAvailableNow

Inherited from SupportsAdmissionControl

Inherited from MicroBatchStream

Inherited from MemoryStreamBase[A]

Inherited from SparkDataStream

Inherited from AnyRef

Inherited from Any

Ungrouped