Packages

o

org.apache.spark.sql.execution.streaming.state

RocksDBStateStoreProvider

object RocksDBStateStoreProvider

Linear Supertypes
AnyRef, Any
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. RocksDBStateStoreProvider
  2. AnyRef
  3. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. Protected

Value Members

  1. final def !=(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  2. final def ##: Int
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  4. val ALL_CUSTOM_METRICS: Seq[Product with StateStoreCustomMetric with Serializable { def withNewDesc(newDesc: String): Product with org.apache.spark.sql.execution.streaming.state.StateStoreCustomMetric with java.io.Serializable }]
  5. val CUSTOM_METRIC_BLOCK_CACHE_HITS: StateStoreCustomSumMetric
  6. val CUSTOM_METRIC_BLOCK_CACHE_MISS: StateStoreCustomSumMetric
  7. val CUSTOM_METRIC_BYTES_COPIED: StateStoreCustomSizeMetric
  8. val CUSTOM_METRIC_BYTES_READ: StateStoreCustomSizeMetric
  9. val CUSTOM_METRIC_BYTES_WRITTEN: StateStoreCustomSizeMetric
  10. val CUSTOM_METRIC_CHECKPOINT_TIME: StateStoreCustomTimingMetric
  11. val CUSTOM_METRIC_COMMIT_COMPACT_TIME: StateStoreCustomTimingMetric
  12. val CUSTOM_METRIC_COMPACT_READ_BYTES: StateStoreCustomSizeMetric
  13. val CUSTOM_METRIC_COMPACT_WRITTEN_BYTES: StateStoreCustomSizeMetric
  14. val CUSTOM_METRIC_FILESYNC_TIME: StateStoreCustomTimingMetric
  15. val CUSTOM_METRIC_FILES_COPIED: StateStoreCustomSumMetric
  16. val CUSTOM_METRIC_FILES_REUSED: StateStoreCustomSumMetric
  17. val CUSTOM_METRIC_FLUSH_TIME: StateStoreCustomTimingMetric
  18. val CUSTOM_METRIC_FLUSH_WRITTEN_BYTES: StateStoreCustomSizeMetric
  19. val CUSTOM_METRIC_GET_COUNT: StateStoreCustomSumMetric
  20. val CUSTOM_METRIC_GET_TIME: StateStoreCustomTimingMetric
  21. val CUSTOM_METRIC_ITERATOR_BYTES_READ: StateStoreCustomSizeMetric
  22. val CUSTOM_METRIC_NUM_EXTERNAL_COL_FAMILIES: StateStoreCustomSizeMetric
  23. val CUSTOM_METRIC_NUM_INTERNAL_COL_FAMILIES: StateStoreCustomSizeMetric
  24. val CUSTOM_METRIC_NUM_INTERNAL_COL_FAMILIES_KEYS: StateStoreCustomSizeMetric
  25. val CUSTOM_METRIC_PINNED_BLOCKS_MEM_USAGE: StateStoreCustomSizeMetric
  26. val CUSTOM_METRIC_PUT_COUNT: StateStoreCustomSumMetric
  27. val CUSTOM_METRIC_PUT_TIME: StateStoreCustomTimingMetric
  28. val CUSTOM_METRIC_SST_FILE_SIZE: StateStoreCustomSizeMetric
  29. val CUSTOM_METRIC_STALL_TIME: StateStoreCustomTimingMetric
  30. val CUSTOM_METRIC_TOTAL_COMPACT_TIME: StateStoreCustomTimingMetric
  31. val CUSTOM_METRIC_ZIP_FILE_BYTES_UNCOMPRESSED: StateStoreCustomSizeMetric
  32. val SCHEMA_ID_PREFIX_BYTES: Int
  33. val STATE_ENCODING_NUM_VERSION_BYTES: Int
  34. val STATE_ENCODING_VERSION: Byte
  35. val VIRTUAL_COL_FAMILY_PREFIX_BYTES: Int
  36. final def asInstanceOf[T0]: T0
    Definition Classes
    Any
  37. def clone(): AnyRef
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.CloneNotSupportedException]) @IntrinsicCandidate() @native()
  38. def decodeStateRowWithPrefix(data: Array[Byte]): Array[Byte]

    Function to decode state row with virtual col family id prefix

    Function to decode state row with virtual col family id prefix

    data

    - passed byte array retrieved from state store

    returns

    - pair of decoded byte array without virtual column family id prefix and name of column family

  39. def encodeStateRowWithPrefix(data: Array[Byte], vcfId: Short): Array[Byte]

    Function to encode state row with virtual col family id prefix

    Function to encode state row with virtual col family id prefix

    data

    - passed byte array to be stored in state store

    vcfId

    - virtual column family id

    returns

    - encoded byte array with virtual column family id prefix

  40. final def eq(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  41. def equals(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef → Any
  42. final def getClass(): Class[_ <: AnyRef]
    Definition Classes
    AnyRef → Any
    Annotations
    @IntrinsicCandidate() @native()
  43. def getColumnFamilyBytesAsId(data: Array[Byte]): Short

    Function to decode virtual column family id from byte array

    Function to decode virtual column family id from byte array

    data

    - passed byte array retrieved from state store

    returns

    - virtual column family id

  44. def getColumnFamilyIdAsBytes(virtualColFamilyId: Short): Array[Byte]

    Encodes a virtual column family ID into a byte array suitable for RocksDB.

    Encodes a virtual column family ID into a byte array suitable for RocksDB.

    This method creates a fixed-size byte array prefixed with the virtual column family ID, which is used to partition data within RocksDB.

    virtualColFamilyId

    The column family identifier to encode

    returns

    A byte array containing the encoded column family ID

  45. def getDataEncoder(stateStoreEncoding: String, encoderCacheKey: StateRowEncoderCacheKey, keyStateEncoderSpec: KeyStateEncoderSpec, valueSchema: StructType, stateSchemaProvider: Option[StateSchemaProvider], columnFamilyName: Option[String] = None): RocksDBDataEncoder

    Creates and returns a data encoder for the state store based on the specified encoding type.

    Creates and returns a data encoder for the state store based on the specified encoding type. This method handles caching of encoders to improve performance by reusing encoder instances when possible.

    The method supports two encoding types: - Avro: Uses Apache Avro for serialization with schema evolution support - UnsafeRow: Uses Spark's internal row format for optimal performance

    stateStoreEncoding

    The encoding type to use ("avro" or "unsaferow")

    encoderCacheKey

    A unique key for caching the encoder instance, typically combining query ID, operator ID, partition ID, and column family name

    keyStateEncoderSpec

    Specification for how to encode keys, including schema and any prefix/range scan requirements

    valueSchema

    The schema for the values to be encoded

    returns

    A RocksDBDataEncoder instance configured for the specified encoding type

  46. def hashCode(): Int
    Definition Classes
    AnyRef → Any
    Annotations
    @IntrinsicCandidate() @native()
  47. final def isInstanceOf[T0]: Boolean
    Definition Classes
    Any
  48. final def ne(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  49. final def notify(): Unit
    Definition Classes
    AnyRef
    Annotations
    @IntrinsicCandidate() @native()
  50. final def notifyAll(): Unit
    Definition Classes
    AnyRef
    Annotations
    @IntrinsicCandidate() @native()
  51. final def synchronized[T0](arg0: => T0): T0
    Definition Classes
    AnyRef
  52. def toString(): String
    Definition Classes
    AnyRef → Any
  53. final def wait(arg0: Long, arg1: Int): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException])
  54. final def wait(arg0: Long): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException]) @native()
  55. final def wait(): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException])

Deprecated Value Members

  1. def finalize(): Unit
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.Throwable]) @Deprecated
    Deprecated

    (Since version 9)

Inherited from AnyRef

Inherited from Any

Ungrouped