Package com.lancedb.lance.spark.internal

Class LanceDatasetAdapter

java.lang.Object

com.lancedb.lance.spark.internal.LanceDatasetAdapter

public class LanceDatasetAdapter
extends Object

Field Summary

Fields

Modifier and Type	Field	Description
static final org.apache.arrow.memory.BufferAllocator	allocator

Constructor Summary

Constructors

Constructor	Description
LanceDatasetAdapter()

Method Summary

Modifier and Type	Method	Description
static void	appendFragments(LanceConfig config, List<com.lancedb.lance.FragmentMetadata> fragments)
static void	createDataset(String datasetUri, org.apache.spark.sql.types.StructType sparkSchema, com.lancedb.lance.WriteParams params)
static List<com.lancedb.lance.FragmentMetadata>	createFragment(String datasetUri, org.apache.arrow.vector.ipc.ArrowReader reader, com.lancedb.lance.WriteParams params)
static com.lancedb.lance.FragmentMetadata	deleteRows(LanceConfig config, int fragmentId, List<Integer> rowIndexes)
static void	dropDataset(LanceConfig config)
static LanceArrowWriter	getArrowWriter(org.apache.spark.sql.types.StructType sparkSchema, int batchSize)
static Optional<Long>	getDatasetDataSize(LanceConfig config)
static Optional<Long>	getDatasetRowCount(LanceConfig config)
static List<Integer>	getFragmentIds(LanceConfig config)
static LanceFragmentScanner	getFragmentScanner(int fragmentId, LanceInputPartition inputPartition)
static Optional<org.apache.spark.sql.types.StructType>	getSchema(LanceConfig config)
static Optional<org.apache.spark.sql.types.StructType>	getSchema(String datasetUri)
static void	overwriteFragments(LanceConfig config, List<com.lancedb.lance.FragmentMetadata> fragments, org.apache.spark.sql.types.StructType sparkSchema)
static void	updateFragments(LanceConfig config, List<Long> removedFragmentIds, List<com.lancedb.lance.FragmentMetadata> updatedFragments, List<com.lancedb.lance.FragmentMetadata> newFragments)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

allocator

public static final org.apache.arrow.memory.BufferAllocator allocator

Constructor Details

LanceDatasetAdapter

public LanceDatasetAdapter()

Method Details

getSchema

public static Optional<org.apache.spark.sql.types.StructType> getSchema(LanceConfig config)

getSchema

public static Optional<org.apache.spark.sql.types.StructType> getSchema(String datasetUri)

getDatasetRowCount

public static Optional<Long> getDatasetRowCount(LanceConfig config)

getDatasetDataSize

public static Optional<Long> getDatasetDataSize(LanceConfig config)

getFragmentIds

public static List<Integer> getFragmentIds(LanceConfig config)

getFragmentScanner

public static LanceFragmentScanner getFragmentScanner(int fragmentId,
 LanceInputPartition inputPartition)

appendFragments

public static void appendFragments(LanceConfig config,
 List<com.lancedb.lance.FragmentMetadata> fragments)

overwriteFragments

public static void overwriteFragments(LanceConfig config,
 List<com.lancedb.lance.FragmentMetadata> fragments,
 org.apache.spark.sql.types.StructType sparkSchema)

updateFragments

public static void updateFragments(LanceConfig config,
 List<Long> removedFragmentIds,
 List<com.lancedb.lance.FragmentMetadata> updatedFragments,
 List<com.lancedb.lance.FragmentMetadata> newFragments)

deleteRows

public static com.lancedb.lance.FragmentMetadata deleteRows(LanceConfig config,
 int fragmentId,
 List<Integer> rowIndexes)

getArrowWriter

public static LanceArrowWriter getArrowWriter(org.apache.spark.sql.types.StructType sparkSchema,
 int batchSize)

createFragment

public static List<com.lancedb.lance.FragmentMetadata> createFragment(String datasetUri,
 org.apache.arrow.vector.ipc.ArrowReader reader,
 com.lancedb.lance.WriteParams params)

createDataset

public static void createDataset(String datasetUri,
 org.apache.spark.sql.types.StructType sparkSchema,
 com.lancedb.lance.WriteParams params)

dropDataset

public static void dropDataset(LanceConfig config)