c

org.apache.spark.sql.delta.files

DelayedCommitProtocol

class DelayedCommitProtocol extends FileCommitProtocol with Serializable with Logging

Writes out the files to path and returns a list of them in addedStatuses. Includes special handling for partitioning on CDC_PARTITION_COL for compatibility between enabled and disabled CDC; partitions with a value of false in this column produce no corresponding partitioning directory.

Linear Supertypes
Serializable, FileCommitProtocol, Logging, AnyRef, Any
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. DelayedCommitProtocol
  2. Serializable
  3. FileCommitProtocol
  4. Logging
  5. AnyRef
  6. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. Protected

Instance Constructors

  1. new DelayedCommitProtocol(jobId: String, path: String, randomPrefixLength: Option[Int], subdir: Option[String])

    path

    The base path files will be written

    randomPrefixLength

    The length of random subdir name under 'path' that files been written

    subdir

    The immediate subdir under path; If randomPrefixLength and subdir both exist, file path will be path/subdir/[rand str of randomPrefixLength]/file

Type Members

  1. implicit class LogStringContext extends AnyRef
    Definition Classes
    Logging

Value Members

  1. final def !=(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  2. final def ##: Int
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  4. val FILE_NAME_PREFIX: String

    Prefix added in testing mode to all filenames to test special chars that need URL-encoding.

  5. def abortJob(jobContext: JobContext): Unit
    Definition Classes
    DelayedCommitProtocol → FileCommitProtocol
  6. def abortTask(taskContext: TaskAttemptContext): Unit
    Definition Classes
    DelayedCommitProtocol → FileCommitProtocol
  7. var addedFiles: ArrayBuffer[(Map[String, String], String)]
    Attributes
    protected
  8. val addedStatuses: ArrayBuffer[AddFile]
  9. final def asInstanceOf[T0]: T0
    Definition Classes
    Any
  10. def buildActionFromAddedFile(f: (Map[String, String], String), stat: FileStatus, taskContext: TaskAttemptContext): FileAction
    Attributes
    protected
  11. val cdcPartitionFalse: String
    Attributes
    protected
  12. val cdcPartitionTrue: String
    Attributes
    protected
  13. val cdcPartitionTrueRegex: Regex
    Attributes
    protected
  14. val changeFiles: ArrayBuffer[AddCDCFile]
  15. def clone(): AnyRef
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.CloneNotSupportedException]) @IntrinsicCandidate() @native()
  16. def commitJob(jobContext: JobContext, taskCommits: Seq[TaskCommitMessage]): Unit

    Commits a job after the writes succeed.

    Commits a job after the writes succeed. Must be called on the driver. Partitions the written files into AddFiles and AddCDCFiles as these metadata actions are treated differently by TransactionalWrite (i.e. AddFile's may have additional statistics injected)

    Definition Classes
    DelayedCommitProtocol → FileCommitProtocol
  17. def commitTask(taskContext: TaskAttemptContext): TaskCommitMessage
    Definition Classes
    DelayedCommitProtocol → FileCommitProtocol
  18. def deleteWithJob(fs: FileSystem, path: Path, recursive: Boolean): Boolean
    Definition Classes
    FileCommitProtocol
  19. final def eq(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  20. def equals(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef → Any
  21. final def getClass(): Class[_ <: AnyRef]
    Definition Classes
    AnyRef → Any
    Annotations
    @IntrinsicCandidate() @native()
  22. def getFileName(taskContext: TaskAttemptContext, ext: String, partitionValues: Map[String, String]): String
    Attributes
    protected
  23. def hashCode(): Int
    Definition Classes
    AnyRef → Any
    Annotations
    @IntrinsicCandidate() @native()
  24. def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean
    Attributes
    protected
    Definition Classes
    Logging
  25. def initializeLogIfNecessary(isInterpreter: Boolean): Unit
    Attributes
    protected
    Definition Classes
    Logging
  26. final def isInstanceOf[T0]: Boolean
    Definition Classes
    Any
  27. def isTraceEnabled(): Boolean
    Attributes
    protected
    Definition Classes
    Logging
  28. def log: Logger
    Attributes
    protected
    Definition Classes
    Logging
  29. def logDebug(msg: => String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  30. def logDebug(entry: LogEntry, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  31. def logDebug(entry: LogEntry): Unit
    Attributes
    protected
    Definition Classes
    Logging
  32. def logDebug(msg: => String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  33. def logError(msg: => String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  34. def logError(entry: LogEntry, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  35. def logError(entry: LogEntry): Unit
    Attributes
    protected
    Definition Classes
    Logging
  36. def logError(msg: => String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  37. def logInfo(msg: => String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  38. def logInfo(entry: LogEntry, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  39. def logInfo(entry: LogEntry): Unit
    Attributes
    protected
    Definition Classes
    Logging
  40. def logInfo(msg: => String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  41. def logName: String
    Attributes
    protected
    Definition Classes
    Logging
  42. def logTrace(msg: => String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  43. def logTrace(entry: LogEntry, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  44. def logTrace(entry: LogEntry): Unit
    Attributes
    protected
    Definition Classes
    Logging
  45. def logTrace(msg: => String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  46. def logWarning(msg: => String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  47. def logWarning(entry: LogEntry, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  48. def logWarning(entry: LogEntry): Unit
    Attributes
    protected
    Definition Classes
    Logging
  49. def logWarning(msg: => String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  50. final def ne(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  51. def newTaskTempFile(taskContext: TaskAttemptContext, dir: Option[String], ext: String): String

    Notifies the commit protocol to add a new file, and gets back the full path that should be used.

    Notifies the commit protocol to add a new file, and gets back the full path that should be used.

    Includes special logic for CDC files and paths. Specifically, if the directory dir contains the CDC partition is_cdc=true then - the file name begins with cdc- instead of part- - the directory has the is_cdc=true partition removed and is placed in the _changed_data folder

    Definition Classes
    DelayedCommitProtocol → FileCommitProtocol
  52. def newTaskTempFile(taskContext: TaskAttemptContext, dir: Option[String], spec: FileNameSpec): String
    Definition Classes
    FileCommitProtocol
  53. def newTaskTempFileAbsPath(taskContext: TaskAttemptContext, absoluteDir: String, ext: String): String
    Definition Classes
    DelayedCommitProtocol → FileCommitProtocol
  54. def newTaskTempFileAbsPath(taskContext: TaskAttemptContext, absoluteDir: String, spec: FileNameSpec): String
    Definition Classes
    FileCommitProtocol
  55. final def notify(): Unit
    Definition Classes
    AnyRef
    Annotations
    @IntrinsicCandidate() @native()
  56. final def notifyAll(): Unit
    Definition Classes
    AnyRef
    Annotations
    @IntrinsicCandidate() @native()
  57. def onTaskCommit(taskCommit: TaskCommitMessage): Unit
    Definition Classes
    FileCommitProtocol
  58. def parsePartitions(dir: String, taskContext: TaskAttemptContext): Map[String, String]
    Attributes
    protected
  59. def setupJob(jobContext: JobContext): Unit
    Definition Classes
    DelayedCommitProtocol → FileCommitProtocol
  60. def setupTask(taskContext: TaskAttemptContext): Unit
    Definition Classes
    DelayedCommitProtocol → FileCommitProtocol
  61. final def synchronized[T0](arg0: => T0): T0
    Definition Classes
    AnyRef
  62. def toString(): String
    Definition Classes
    AnyRef → Any
  63. final def wait(arg0: Long, arg1: Int): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException])
  64. final def wait(arg0: Long): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException]) @native()
  65. final def wait(): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException])
  66. def withLogContext(context: Map[String, String])(body: => Unit): Unit
    Attributes
    protected
    Definition Classes
    Logging

Deprecated Value Members

  1. def finalize(): Unit
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.Throwable]) @Deprecated
    Deprecated

    (Since version 9)

Inherited from Serializable

Inherited from FileCommitProtocol

Inherited from Logging

Inherited from AnyRef

Inherited from Any

Ungrouped