Packages

c

org.apache.spark.sql.execution.streaming.checkpointing

ChecksumCancellableFSDataOutputStream

class ChecksumCancellableFSDataOutputStream extends CancellableFSDataOutputStream with Logging

An implementation of CancellableFSDataOutputStream that calculates the checksum of the file that the client is writing (main file) incrementally, while it is being written. It then writes the main file and an additional checksum file, which will be used for verification by ChecksumFSDataInputStream on file read.

Linear Supertypes
Logging, CancellableFSDataOutputStream, FSDataOutputStream, Abortable, IOStatisticsSource, StreamCapabilities, CanSetDropBehind, Syncable, DataOutputStream, DataOutput, FilterOutputStream, OutputStream, Flushable, Closeable, AutoCloseable, AnyRef, Any
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. ChecksumCancellableFSDataOutputStream
  2. Logging
  3. CancellableFSDataOutputStream
  4. FSDataOutputStream
  5. Abortable
  6. IOStatisticsSource
  7. StreamCapabilities
  8. CanSetDropBehind
  9. Syncable
  10. DataOutputStream
  11. DataOutput
  12. FilterOutputStream
  13. OutputStream
  14. Flushable
  15. Closeable
  16. AutoCloseable
  17. AnyRef
  18. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. Protected

Instance Constructors

  1. new ChecksumCancellableFSDataOutputStream(mainStream: CancellableFSDataOutputStream, path: Path, checksumStream: CancellableFSDataOutputStream, uploadThreadPool: ExecutionContext)

    mainStream

    Output stream for the main file the client wants to write to

    path

    The path of the main file

    checksumStream

    Output stream for the checksum file to write the computed checksum

    uploadThreadPool

    Thread pool used to concurrently upload the main and checksum file

Type Members

  1. implicit class LogStringContext extends AnyRef
    Definition Classes
    Logging

Value Members

  1. final def !=(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  2. final def ##: Int
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  4. def MDC(key: LogKey, value: Any): MDC
    Attributes
    protected
    Definition Classes
    Logging
  5. def abort(): AbortableResult
    Definition Classes
    FSDataOutputStream → Abortable
  6. final def asInstanceOf[T0]: T0
    Definition Classes
    Any
  7. def cancel(): Unit

    Cancel the underlyingStream and ensure that the output file is not generated.

    Cancel the underlyingStream and ensure that the output file is not generated.

    Definition Classes
    ChecksumCancellableFSDataOutputStream → CancellableFSDataOutputStream
  8. def clone(): AnyRef
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.CloneNotSupportedException]) @IntrinsicCandidate() @native()
  9. def close(): Unit
    Definition Classes
    ChecksumCancellableFSDataOutputStream → FSDataOutputStream → FilterOutputStream → OutputStream → Closeable → AutoCloseable
  10. final def eq(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  11. def equals(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef → Any
  12. def flush(): Unit
    Definition Classes
    DataOutputStream → FilterOutputStream → OutputStream → Flushable
    Annotations
    @throws(classOf[java.io.IOException])
  13. final def getClass(): Class[_ <: AnyRef]
    Definition Classes
    AnyRef → Any
    Annotations
    @IntrinsicCandidate() @native()
  14. def getIOStatistics(): IOStatistics
    Definition Classes
    FSDataOutputStream → IOStatisticsSource
  15. def getPos(): Long
    Definition Classes
    FSDataOutputStream
  16. def getWrappedStream(): OutputStream
    Definition Classes
    FSDataOutputStream
    Annotations
    @LimitedPrivate()
  17. def hasCapability(arg0: String): Boolean
    Definition Classes
    FSDataOutputStream → StreamCapabilities
  18. def hashCode(): Int
    Definition Classes
    AnyRef → Any
    Annotations
    @IntrinsicCandidate() @native()
  19. def hflush(): Unit
    Definition Classes
    FSDataOutputStream → Syncable
    Annotations
    @throws(classOf[java.io.IOException])
  20. def hsync(): Unit
    Definition Classes
    FSDataOutputStream → Syncable
    Annotations
    @throws(classOf[java.io.IOException])
  21. def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean
    Attributes
    protected
    Definition Classes
    Logging
  22. def initializeLogIfNecessary(isInterpreter: Boolean): Unit
    Attributes
    protected
    Definition Classes
    Logging
  23. final def isInstanceOf[T0]: Boolean
    Definition Classes
    Any
  24. def isTraceEnabled(): Boolean
    Attributes
    protected
    Definition Classes
    Logging
  25. def log: Logger
    Attributes
    protected
    Definition Classes
    Logging
  26. def logBasedOnLevel(level: Level)(f: => MessageWithContext): Unit
    Attributes
    protected
    Definition Classes
    Logging
  27. def logDebug(msg: => String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  28. def logDebug(entry: LogEntry, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  29. def logDebug(entry: LogEntry): Unit
    Attributes
    protected
    Definition Classes
    Logging
  30. def logDebug(msg: => String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  31. def logError(msg: => String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  32. def logError(entry: LogEntry, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  33. def logError(entry: LogEntry): Unit
    Attributes
    protected
    Definition Classes
    Logging
  34. def logError(msg: => String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  35. def logInfo(msg: => String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  36. def logInfo(entry: LogEntry, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  37. def logInfo(entry: LogEntry): Unit
    Attributes
    protected
    Definition Classes
    Logging
  38. def logInfo(msg: => String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  39. def logName: String
    Attributes
    protected
    Definition Classes
    Logging
  40. def logTrace(msg: => String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  41. def logTrace(entry: LogEntry, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  42. def logTrace(entry: LogEntry): Unit
    Attributes
    protected
    Definition Classes
    Logging
  43. def logTrace(msg: => String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  44. def logWarning(msg: => String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  45. def logWarning(entry: LogEntry, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  46. def logWarning(entry: LogEntry): Unit
    Attributes
    protected
    Definition Classes
    Logging
  47. def logWarning(msg: => String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  48. final def ne(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  49. final def notify(): Unit
    Definition Classes
    AnyRef
    Annotations
    @IntrinsicCandidate() @native()
  50. final def notifyAll(): Unit
    Definition Classes
    AnyRef
    Annotations
    @IntrinsicCandidate() @native()
  51. def setDropBehind(arg0: Boolean): Unit
    Definition Classes
    FSDataOutputStream → CanSetDropBehind
    Annotations
    @throws(classOf[java.io.IOException])
  52. final def size(): Int
    Definition Classes
    DataOutputStream
  53. final def synchronized[T0](arg0: => T0): T0
    Definition Classes
    AnyRef
  54. def toString(): String
    Definition Classes
    FSDataOutputStream → AnyRef → Any
  55. val underlyingStream: OutputStream
    Attributes
    protected
    Definition Classes
    CancellableFSDataOutputStream
  56. final def wait(arg0: Long, arg1: Int): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException])
  57. final def wait(arg0: Long): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException]) @native()
  58. final def wait(): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException])
  59. def withLogContext(context: Map[String, String])(body: => Unit): Unit
    Attributes
    protected
    Definition Classes
    Logging
  60. def write(arg0: Array[Byte], arg1: Int, arg2: Int): Unit
    Definition Classes
    DataOutputStream → DataOutput → FilterOutputStream → OutputStream
    Annotations
    @throws(classOf[java.io.IOException])
  61. def write(arg0: Int): Unit
    Definition Classes
    DataOutputStream → DataOutput → FilterOutputStream → OutputStream
    Annotations
    @throws(classOf[java.io.IOException])
  62. def write(arg0: Array[Byte]): Unit
    Definition Classes
    FilterOutputStream → OutputStream
    Annotations
    @throws(classOf[java.io.IOException])
  63. final def writeBoolean(arg0: Boolean): Unit
    Definition Classes
    DataOutputStream → DataOutput
    Annotations
    @throws(classOf[java.io.IOException])
  64. final def writeByte(arg0: Int): Unit
    Definition Classes
    DataOutputStream → DataOutput
    Annotations
    @throws(classOf[java.io.IOException])
  65. final def writeBytes(arg0: String): Unit
    Definition Classes
    DataOutputStream → DataOutput
    Annotations
    @throws(classOf[java.io.IOException])
  66. final def writeChar(arg0: Int): Unit
    Definition Classes
    DataOutputStream → DataOutput
    Annotations
    @throws(classOf[java.io.IOException])
  67. final def writeChars(arg0: String): Unit
    Definition Classes
    DataOutputStream → DataOutput
    Annotations
    @throws(classOf[java.io.IOException])
  68. final def writeDouble(arg0: Double): Unit
    Definition Classes
    DataOutputStream → DataOutput
    Annotations
    @throws(classOf[java.io.IOException])
  69. final def writeFloat(arg0: Float): Unit
    Definition Classes
    DataOutputStream → DataOutput
    Annotations
    @throws(classOf[java.io.IOException])
  70. final def writeInt(arg0: Int): Unit
    Definition Classes
    DataOutputStream → DataOutput
    Annotations
    @throws(classOf[java.io.IOException])
  71. final def writeLong(arg0: Long): Unit
    Definition Classes
    DataOutputStream → DataOutput
    Annotations
    @throws(classOf[java.io.IOException])
  72. final def writeShort(arg0: Int): Unit
    Definition Classes
    DataOutputStream → DataOutput
    Annotations
    @throws(classOf[java.io.IOException])
  73. final def writeUTF(arg0: String): Unit
    Definition Classes
    DataOutputStream → DataOutput
    Annotations
    @throws(classOf[java.io.IOException])

Deprecated Value Members

  1. def finalize(): Unit
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.Throwable]) @Deprecated
    Deprecated

    (Since version 9)

Inherited from Logging

Inherited from CancellableFSDataOutputStream

Inherited from FSDataOutputStream

Inherited from Abortable

Inherited from IOStatisticsSource

Inherited from StreamCapabilities

Inherited from CanSetDropBehind

Inherited from Syncable

Inherited from DataOutputStream

Inherited from DataOutput

Inherited from FilterOutputStream

Inherited from OutputStream

Inherited from Flushable

Inherited from Closeable

Inherited from AutoCloseable

Inherited from AnyRef

Inherited from Any

Ungrouped