case class MemoryStream[A](id: Int, sqlContext: SQLContext, numPartitions: Option[Int] = None)(implicit evidence$4: Encoder[A]) extends MemoryStreamBase[A] with MicroBatchStream with SupportsTriggerAvailableNow with Logging with Product with Serializable
A Source that produces value stored in memory as they are added by the user. This Source is intended for use in unit tests as it can only replay data when the object is still available.
If numPartitions is provided, the rows will be redistributed to the given number of partitions.
Linear Supertypes
Ordering
- Alphabetic
- By Inheritance
Inherited
- MemoryStream
- Serializable
- Product
- Equals
- Logging
- SupportsTriggerAvailableNow
- SupportsAdmissionControl
- MicroBatchStream
- MemoryStreamBase
- SparkDataStream
- AnyRef
- Any
- Hide All
- Show All
Visibility
- Public
- Protected
Instance Constructors
- new MemoryStream(id: Int, sqlContext: SQLContext, numPartitions: Option[Int] = None)(implicit arg0: Encoder[A])
Type Members
- implicit class LogStringContext extends AnyRef
- Definition Classes
- Logging
Value Members
- final def !=(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
- final def ##: Int
- Definition Classes
- AnyRef → Any
- final def ==(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
- def addData(data: IterableOnce[A]): Offset
- Definition Classes
- MemoryStream → MemoryStreamBase
- def addData(data: A*): connector.read.streaming.Offset
- Definition Classes
- MemoryStreamBase
- final def asInstanceOf[T0]: T0
- Definition Classes
- Any
- val attributes: Seq[AttributeReference]
- Attributes
- protected
- Definition Classes
- MemoryStreamBase
- val batches: ListBuffer[Array[UnsafeRow]]
All batches from
lastCommittedOffset + 1tocurrentOffset, inclusive.All batches from
lastCommittedOffset + 1tocurrentOffset, inclusive. Stored in a ListBuffer to facilitate removing committed batches.- Attributes
- protected
- def clone(): AnyRef
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.CloneNotSupportedException]) @IntrinsicCandidate() @native()
- def commit(end: connector.read.streaming.Offset): Unit
- Definition Classes
- MemoryStream → MemoryStreamBase → SparkDataStream
- def createReaderFactory(): PartitionReaderFactory
- Definition Classes
- MemoryStream → MicroBatchStream
- var currentOffset: LongOffset
- Attributes
- protected
- def deserializeOffset(json: String): connector.read.streaming.Offset
- Definition Classes
- MemoryStream → MemoryStreamBase → SparkDataStream
- val encoder: ExpressionEncoder[A]
- Definition Classes
- MemoryStreamBase
- final def eq(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
- def fullSchema(): StructType
- Definition Classes
- MemoryStreamBase
- final def getClass(): Class[_ <: AnyRef]
- Definition Classes
- AnyRef → Any
- Annotations
- @IntrinsicCandidate() @native()
- def getDefaultReadLimit(): ReadLimit
- Definition Classes
- SupportsAdmissionControl
- val id: Int
- def initialOffset(): connector.read.streaming.Offset
- Definition Classes
- MemoryStream → MemoryStreamBase → SparkDataStream
- def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean
- Attributes
- protected
- Definition Classes
- Logging
- def initializeLogIfNecessary(isInterpreter: Boolean): Unit
- Attributes
- protected
- Definition Classes
- Logging
- final def isInstanceOf[T0]: Boolean
- Definition Classes
- Any
- def isTraceEnabled(): Boolean
- Attributes
- protected
- Definition Classes
- Logging
- var lastOffsetCommitted: LongOffset
Last offset that was discarded, or -1 if no commits have occurred.
Last offset that was discarded, or -1 if no commits have occurred. Note that the value -1 is used in calculations below and isn't just an arbitrary constant.
- Attributes
- protected
- def latestOffset(startOffset: connector.read.streaming.Offset, limit: ReadLimit): connector.read.streaming.Offset
- Definition Classes
- MemoryStream → SupportsAdmissionControl
- def latestOffset(): connector.read.streaming.Offset
- Definition Classes
- MemoryStream → MicroBatchStream
- def log: Logger
- Attributes
- protected
- Definition Classes
- Logging
- def logDebug(msg: => String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logDebug(entry: LogEntry, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logDebug(entry: LogEntry): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logDebug(msg: => String): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logError(msg: => String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logError(entry: LogEntry, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logError(entry: LogEntry): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logError(msg: => String): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logInfo(msg: => String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logInfo(entry: LogEntry, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logInfo(entry: LogEntry): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logInfo(msg: => String): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logName: String
- Attributes
- protected
- Definition Classes
- Logging
- def logTrace(msg: => String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logTrace(entry: LogEntry, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logTrace(entry: LogEntry): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logTrace(msg: => String): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logWarning(msg: => String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logWarning(entry: LogEntry, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logWarning(entry: LogEntry): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logWarning(msg: => String): Unit
- Attributes
- protected
- Definition Classes
- Logging
- val logicalPlan: LogicalPlan
- Attributes
- protected
- Definition Classes
- MemoryStreamBase
- final def ne(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
- final def notify(): Unit
- Definition Classes
- AnyRef
- Annotations
- @IntrinsicCandidate() @native()
- final def notifyAll(): Unit
- Definition Classes
- AnyRef
- Annotations
- @IntrinsicCandidate() @native()
- val numPartitions: Option[Int]
- val output: Seq[Attribute]
- Attributes
- protected
- def planInputPartitions(start: connector.read.streaming.Offset, end: connector.read.streaming.Offset): Array[InputPartition]
- Definition Classes
- MemoryStream → MicroBatchStream
- def prepareForTriggerAvailableNow(): Unit
- Definition Classes
- MemoryStream → SupportsTriggerAvailableNow
- def productElementNames: Iterator[String]
- Definition Classes
- Product
- def reportLatestOffset(): connector.read.streaming.Offset
- Definition Classes
- SupportsAdmissionControl
- def reset(): Unit
- val sqlContext: SQLContext
- var startOffset: LongOffset
- Attributes
- protected
- def stop(): Unit
- Definition Classes
- MemoryStream → SparkDataStream
- final def synchronized[T0](arg0: => T0): T0
- Definition Classes
- AnyRef
- def toDF(): classic.DataFrame
- Definition Classes
- MemoryStreamBase
- def toDS(): classic.Dataset[A]
- Definition Classes
- MemoryStreamBase
- lazy val toRow: Serializer[A]
- Attributes
- protected
- Definition Classes
- MemoryStreamBase
- def toString(): String
- Definition Classes
- MemoryStream → AnyRef → Any
- final def wait(arg0: Long, arg1: Int): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.InterruptedException])
- final def wait(arg0: Long): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.InterruptedException]) @native()
- final def wait(): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.InterruptedException])
- def withLogContext(context: Map[String, String])(body: => Unit): Unit
- Attributes
- protected
- Definition Classes
- Logging
Deprecated Value Members
- def finalize(): Unit
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.Throwable]) @Deprecated
- Deprecated
(Since version 9)