Package ai.djl.huggingface.tokenizers.jni

Class TokenizersLibrary

java.lang.Object

ai.djl.huggingface.tokenizers.jni.TokenizersLibrary

public final class TokenizersLibrary
extends java.lang.Object

A class containing utilities to interact with the Tokenizer JNI layer.

Field Summary

Fields

Modifier and Type	Field	Description
static TokenizersLibrary	LIB

Method Summary

Modifier and Type	Method	Description
java.lang.String[]	batchDecode(long tokenizer, long[][] batchIds, boolean addSpecialTokens)
long[]	batchEncode(long tokenizer, java.lang.String[] inputs, boolean addSpecialTokens)
long[]	batchEncodePair(long tokenizer, java.lang.String[] text, java.lang.String[] textPair, boolean addSpecialTokens)
long	createBpeTokenizer(java.lang.String vocabulary, java.lang.String merges)
long	createTokenizer(java.lang.String identifier)
long	createTokenizerFromString(java.lang.String json)
java.lang.String	decode(long tokenizer, long[] ids, boolean addSpecialTokens)
void	deleteEncoding(long encoding)
void	deleteTokenizer(long handle)
void	disablePadding(long tokenizer)
void	disableTruncation(long tokenizer)
long	encode(long tokenizer, java.lang.String text, boolean addSpecialTokens)
long	encodeDual(long tokenizer, java.lang.String text, java.lang.String textPair, boolean addSpecialTokens)
long	encodeList(long tokenizer, java.lang.String[] inputs, boolean addSpecialTokens)
long[]	getAttentionMask(long encoding)
int	getMaxLength(long tokenizer)
long[]	getOverflowing(long encoding)
java.lang.String	getPaddingStrategy(long tokenizer)
int	getPadToMultipleOf(long tokenizer)
long[]	getSpecialTokenMask(long encoding)
int	getStride(long tokenizer)
CharSpan[]	getTokenCharSpans(long encoding)
long[]	getTokenIds(long encoding)
java.lang.String[]	getTokens(long encoding)
java.lang.String	getTruncationStrategy(long tokenizer)
long[]	getTypeIds(long encoding)
long[]	getWordIds(long encoding)
void	setPadding(long tokenizer, int maxLength, java.lang.String paddingStrategy, int padToMultipleOf)
void	setTruncation(long tokenizer, int maxLength, java.lang.String truncationStrategy, int stride)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

LIB

public static final TokenizersLibrary LIB

Method Detail

createTokenizer

public long createTokenizer(java.lang.String identifier)

createTokenizerFromString

public long createTokenizerFromString(java.lang.String json)

createBpeTokenizer

public long createBpeTokenizer(java.lang.String vocabulary,
                               java.lang.String merges)

deleteTokenizer

public void deleteTokenizer(long handle)

encode

public long encode(long tokenizer,
                   java.lang.String text,
                   boolean addSpecialTokens)

encodeDual

public long encodeDual(long tokenizer,
                       java.lang.String text,
                       java.lang.String textPair,
                       boolean addSpecialTokens)

encodeList

public long encodeList(long tokenizer,
                       java.lang.String[] inputs,
                       boolean addSpecialTokens)

batchEncode

public long[] batchEncode(long tokenizer,
                          java.lang.String[] inputs,
                          boolean addSpecialTokens)

batchEncodePair

public long[] batchEncodePair(long tokenizer,
                              java.lang.String[] text,
                              java.lang.String[] textPair,
                              boolean addSpecialTokens)

batchDecode

public java.lang.String[] batchDecode(long tokenizer,
                                      long[][] batchIds,
                                      boolean addSpecialTokens)

deleteEncoding

public void deleteEncoding(long encoding)

getTokenIds

public long[] getTokenIds(long encoding)

getTypeIds

public long[] getTypeIds(long encoding)

getWordIds

public long[] getWordIds(long encoding)

getTokens

public java.lang.String[] getTokens(long encoding)

getAttentionMask

public long[] getAttentionMask(long encoding)

getSpecialTokenMask

public long[] getSpecialTokenMask(long encoding)

getTokenCharSpans

public CharSpan[] getTokenCharSpans(long encoding)

getOverflowing

public long[] getOverflowing(long encoding)

decode

public java.lang.String decode(long tokenizer,
                               long[] ids,
                               boolean addSpecialTokens)

getTruncationStrategy

public java.lang.String getTruncationStrategy(long tokenizer)

getPaddingStrategy

public java.lang.String getPaddingStrategy(long tokenizer)

getMaxLength

public int getMaxLength(long tokenizer)

getStride

public int getStride(long tokenizer)

getPadToMultipleOf

public int getPadToMultipleOf(long tokenizer)

disablePadding

public void disablePadding(long tokenizer)

setPadding

public void setPadding(long tokenizer,
                       int maxLength,
                       java.lang.String paddingStrategy,
                       int padToMultipleOf)

disableTruncation

public void disableTruncation(long tokenizer)

setTruncation

public void setTruncation(long tokenizer,
                          int maxLength,
                          java.lang.String truncationStrategy,
                          int stride)