Interface TokenizationOptions

interface TokenizationOptions {
    type?: "word" | "sentence" | "subword_bpe";
    toLowerCase?: boolean;
    removePunctuation?: boolean;
    language?: string;
    modelId?: string;
}

Properties

type? toLowerCase? removePunctuation? language? modelId?

Properties

Optional type

type?: "word" | "sentence" | "subword_bpe"

Optional toLowerCase

toLowerCase?: boolean

Default true. StatisticalUtilityAI keeps a sentence's case unless it is true.

Optional removePunctuation

removePunctuation?: boolean

LLMUtilityAI: default true for words, false for sentences. StatisticalUtilityAI's words never hold punctuation.

Optional language

language?: string

Optional modelId

modelId?: string

Settings

Member Visibility

Theme

On This Page

typetoLowerCaseremovePunctuationlanguagemodelId