com.lancedb.lance.spark.internal

Class LanceDatasetAdapter

java.lang.Object

com.lancedb.lance.spark.internal.LanceDatasetAdapter

public class LanceDatasetAdapter
extends Object

Field Summary

Fields

Modifier and Type	Field and Description
static org.apache.arrow.memory.BufferAllocator	allocator

Constructor Summary

Constructors

Constructor and Description
LanceDatasetAdapter()

Method Summary

Modifier and Type	Method and Description
static void	appendFragments(LanceConfig config, List<FragmentMetadata> fragments)
static void	createDataset(String datasetUri, org.apache.spark.sql.types.StructType sparkSchema, WriteParams params)
static List<FragmentMetadata>	createFragment(String datasetUri, org.apache.arrow.vector.ipc.ArrowReader reader, WriteParams params)
static void	dropDataset(LanceConfig config)
static LanceArrowWriter	getArrowWriter(org.apache.spark.sql.types.StructType sparkSchema, int batchSize)
static Optional<Long>	getDatasetDataSize(LanceConfig config)
static Optional<Long>	getDatasetRowCount(LanceConfig config)
static List<Integer>	getFragmentIds(LanceConfig config)
static LanceFragmentScanner	getFragmentScanner(int fragmentId, LanceInputPartition inputPartition)
static Optional<org.apache.spark.sql.types.StructType>	getSchema(LanceConfig config)
static Optional<org.apache.spark.sql.types.StructType>	getSchema(String datasetUri)
static void	overwriteFragments(LanceConfig config, List<FragmentMetadata> fragments, org.apache.spark.sql.types.StructType sparkSchema)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

allocator

public static final org.apache.arrow.memory.BufferAllocator allocator

Constructor Detail

LanceDatasetAdapter

public LanceDatasetAdapter()

Method Detail

getSchema

public static Optional<org.apache.spark.sql.types.StructType> getSchema(LanceConfig config)

getSchema

public static Optional<org.apache.spark.sql.types.StructType> getSchema(String datasetUri)

getDatasetRowCount

public static Optional<Long> getDatasetRowCount(LanceConfig config)

getDatasetDataSize

public static Optional<Long> getDatasetDataSize(LanceConfig config)

getFragmentIds

public static List<Integer> getFragmentIds(LanceConfig config)

getFragmentScanner

public static LanceFragmentScanner getFragmentScanner(int fragmentId,
                                                      LanceInputPartition inputPartition)

appendFragments

public static void appendFragments(LanceConfig config,
                                   List<FragmentMetadata> fragments)

overwriteFragments

public static void overwriteFragments(LanceConfig config,
                                      List<FragmentMetadata> fragments,
                                      org.apache.spark.sql.types.StructType sparkSchema)

getArrowWriter

public static LanceArrowWriter getArrowWriter(org.apache.spark.sql.types.StructType sparkSchema,
                                              int batchSize)

createFragment

public static List<FragmentMetadata> createFragment(String datasetUri,
                                                    org.apache.arrow.vector.ipc.ArrowReader reader,
                                                    WriteParams params)

createDataset

public static void createDataset(String datasetUri,
                                 org.apache.spark.sql.types.StructType sparkSchema,
                                 WriteParams params)

dropDataset

public static void dropDataset(LanceConfig config)