[][src]Struct sentencepiece::SentencePieceProcessor

pub struct SentencePieceProcessor { /* fields omitted */ }

Sentence piece tokenizer.

Instances of SentencePieceProcessor can be used to tokenizer a sentence using a sentencepiece model.

Methods

impl SentencePieceProcessor[src]

pub fn from_serialized_proto(data: &[u8]) -> Result<Self, SentencePieceError>[src]

pub fn load(filename: &str) -> Result<Self, SentencePieceError>[src]

Load a sentencepiece model.

pub fn encode(
    &self,
    sentence: &str
) -> Result<Vec<PieceWithId>, SentencePieceError>
[src]

Tokenizer a sentence.

Trait Implementations

impl Debug for SentencePieceProcessor[src]

impl Drop for SentencePieceProcessor[src]

Auto Trait Implementations

impl RefUnwindSafe for SentencePieceProcessor

impl !Send for SentencePieceProcessor

impl !Sync for SentencePieceProcessor

impl Unpin for SentencePieceProcessor

impl UnwindSafe for SentencePieceProcessor

Blanket Implementations

impl<T> Any for T where
    T: 'static + ?Sized
[src]

impl<T> Borrow<T> for T where
    T: ?Sized
[src]

impl<T> BorrowMut<T> for T where
    T: ?Sized
[src]

impl<T> From<T> for T[src]

impl<T, U> Into<U> for T where
    U: From<T>, 
[src]

impl<T, U> TryFrom<U> for T where
    U: Into<T>, 
[src]

type Error = Infallible

The type returned in the event of a conversion error.

impl<T, U> TryInto<U> for T where
    U: TryFrom<T>, 
[src]

type Error = <U as TryFrom<T>>::Error

The type returned in the event of a conversion error.