Struct llamapun::tokenizer::Tokenizer

source · [−]

pub struct Tokenizer {
    pub stopwords: HashSet<&'static str>,
    pub abbreviations: Regex,
}

Expand description

Stores auxiliary resources required by the tokenizer so that they need to be initialized only once

Fields

stopwords: HashSet<&'static str>

set of stopwords

abbreviations: Regex

regular expression for abbreviations

Implementations

source

impl Tokenizer

source

pub fn sentences<'a>(&self, dnm: &'a DNM) -> Vec<DNMRange<'a>>ⓘNotable traits for Vec<u8, A>`impl<A> Write for Vec<u8, A>where A: Allocator,`

gets the sentences from a dnm

source

pub fn words<'b>(&'b self, sentence_range: &DNMRange<'b>) -> Vec<DNMRange<'_>>ⓘNotable traits for Vec<u8, A>`impl<A> Write for Vec<u8, A>where A: Allocator,`

returns the words of a sentence using simple heuristics

source

pub fn words_and_punct<'b>(&'b self, range: &DNMRange<'b>) -> Vec<DNMRange<'_>>ⓘNotable traits for Vec<u8, A>`impl<A> Write for Vec<u8, A>where A: Allocator,`

returns the words and punctuation of a sentence, using simple heuristics

Trait Implementations

source

impl Default for Tokenizer

source

fn default() -> Tokenizer

Returns the “default value” for a type. Read more

Auto Trait Implementations

impl RefUnwindSafe for Tokenizer

impl Send for Tokenizer

impl Sync for Tokenizer

impl Unpin for Tokenizer

impl UnwindSafe for Tokenizer

Blanket Implementations

source

impl<T> Any for Twhere
T: 'static + ?Sized,

source

fn type_id(&self) -> TypeId

Gets the TypeId of self. Read more

source

impl<T> Borrow<T> for Twhere
T: ?Sized,

const: unstable · source

fn borrow(&self) -> &T

Immutably borrows from an owned value. Read more

source

impl<T> BorrowMut<T> for Twhere
T: ?Sized,

const: unstable · source

fn borrow_mut(&mut self) -> &mut T

Mutably borrows from an owned value. Read more

source

impl<T> From<T> for T

const: unstable · source

fn from(t: T) -> T

Returns the argument unchanged.

source

impl<T, U> Into for Twhere
U: From<T>,

const: unstable · source

fn into(self) -> U

Calls U::from(self).

That is, this conversion is whatever the implementation of From<T> for U chooses to do.

impl<T> Pointable for T

const ALIGN: usize = mem::align_of::<T>()

The alignment of pointer.

type Init = T

The type for initializers.

unsafe fn init(init: <T as Pointable>::Init) -> usize

Initializes a with the given initializer. Read more

unsafe fn deref<'a>(ptr: usize) -> &'a T

Dereferences the given pointer. Read more

unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut T

Mutably dereferences the given pointer. Read more

unsafe fn drop(ptr: usize)

Drops the object pointed to by the given pointer. Read more

source

impl<T, U> TryFrom for Twhere
U: Into<T>,

type Error = Infallible

The type returned in the event of a conversion error.

const: unstable · source

fn try_from(value: U) -> Result<T, <T as TryFrom>::Error>

Performs the conversion.

source

impl<T, U> TryInto for Twhere
U: TryFrom<T>,

type Error = >::Error

The type returned in the event of a conversion error.

const: unstable · source

fn try_into(self) -> Result<U, >::Error>

Performs the conversion.

Struct llamapun::tokenizer::Tokenizer

Fields

Implementations

impl Tokenizer

pub fn sentences<'a>(&self, dnm: &'a DNM) -> Vec<DNMRange<'a>>ⓘNotable traits for Vec<u8, A>`impl<A> Write for Vec<u8, A>where A: Allocator,`

pub fn words<'b>(&'b self, sentence_range: &DNMRange<'b>) -> Vec<DNMRange<'_>>ⓘNotable traits for Vec<u8, A>`impl<A> Write for Vec<u8, A>where A: Allocator,`

pub fn words_and_punct<'b>(&'b self, range: &DNMRange<'b>) -> Vec<DNMRange<'_>>ⓘNotable traits for Vec<u8, A>`impl<A> Write for Vec<u8, A>where A: Allocator,`

Trait Implementations

impl Default for Tokenizer

fn default() -> Tokenizer

Auto Trait Implementations

impl RefUnwindSafe for Tokenizer

impl Send for Tokenizer

impl Sync for Tokenizer

impl Unpin for Tokenizer

impl UnwindSafe for Tokenizer

Blanket Implementations

impl<T> Any for Twhere
T: 'static + ?Sized,

fn type_id(&self) -> TypeId

impl<T> Borrow<T> for Twhere
T: ?Sized,

fn borrow(&self) -> &T

impl<T> BorrowMut<T> for Twhere
T: ?Sized,

fn borrow_mut(&mut self) -> &mut T

impl<T> From<T> for T

fn from(t: T) -> T

impl<T, U> Into<U> for Twhere
U: From<T>,

fn into(self) -> U

impl<T> Pointable for T

const ALIGN: usize = mem::align_of::<T>()

type Init = T

unsafe fn init(init: <T as Pointable>::Init) -> usize

unsafe fn deref<'a>(ptr: usize) -> &'a T

unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut T

unsafe fn drop(ptr: usize)

impl<T, U> TryFrom<U> for Twhere
U: Into<T>,

type Error = Infallible

fn try_from(value: U) -> Result<T, <T as TryFrom<U>>::Error>

impl<T, U> TryInto<U> for Twhere
U: TryFrom<T>,

type Error = <U as TryFrom<T>>::Error

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

Struct llamapun::tokenizer::Tokenizer

Fields

Implementations

impl Tokenizer

pub fn sentences<'a>(&self, dnm: &'a DNM) -> Vec<DNMRange<'a>>ⓘNotable traits for Vec<u8, A>impl<A> Write for Vec<u8, A>where A: Allocator,

pub fn words<'b>(&'b self, sentence_range: &DNMRange<'b>) -> Vec<DNMRange<'_>>ⓘNotable traits for Vec<u8, A>impl<A> Write for Vec<u8, A>where A: Allocator,

pub fn words_and_punct<'b>(&'b self, range: &DNMRange<'b>) -> Vec<DNMRange<'_>>ⓘNotable traits for Vec<u8, A>impl<A> Write for Vec<u8, A>where A: Allocator,

Trait Implementations

impl Default for Tokenizer

fn default() -> Tokenizer

Auto Trait Implementations

impl RefUnwindSafe for Tokenizer

impl Send for Tokenizer

impl Sync for Tokenizer

impl Unpin for Tokenizer

impl UnwindSafe for Tokenizer

Blanket Implementations

impl<T> Any for Twhere T: 'static + ?Sized,

fn type_id(&self) -> TypeId

impl<T> Borrow<T> for Twhere T: ?Sized,

fn borrow(&self) -> &T

impl<T> BorrowMut<T> for Twhere T: ?Sized,

fn borrow_mut(&mut self) -> &mut T

impl<T> From<T> for T

fn from(t: T) -> T

impl<T, U> Into<U> for Twhere U: From<T>,

fn into(self) -> U

impl<T> Pointable for T

const ALIGN: usize = mem::align_of::<T>()

type Init = T

unsafe fn init(init: <T as Pointable>::Init) -> usize

unsafe fn deref<'a>(ptr: usize) -> &'a T

unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut T

unsafe fn drop(ptr: usize)

impl<T, U> TryFrom<U> for Twhere U: Into<T>,

type Error = Infallible

fn try_from(value: U) -> Result<T, <T as TryFrom<U>>::Error>

impl<T, U> TryInto<U> for Twhere U: TryFrom<T>,

type Error = <U as TryFrom<T>>::Error

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

pub fn sentences<'a>(&self, dnm: &'a DNM) -> Vec<DNMRange<'a>>ⓘNotable traits for Vec<u8, A>`impl<A> Write for Vec<u8, A>where A: Allocator,`

pub fn words<'b>(&'b self, sentence_range: &DNMRange<'b>) -> Vec<DNMRange<'_>>ⓘNotable traits for Vec<u8, A>`impl<A> Write for Vec<u8, A>where A: Allocator,`

pub fn words_and_punct<'b>(&'b self, range: &DNMRange<'b>) -> Vec<DNMRange<'_>>ⓘNotable traits for Vec<u8, A>`impl<A> Write for Vec<u8, A>where A: Allocator,`

impl<T> Any for Twhere
T: 'static + ?Sized,

impl<T> Borrow<T> for Twhere
T: ?Sized,

impl<T> BorrowMut<T> for Twhere
T: ?Sized,

impl<T, U> Into<U> for Twhere
U: From<T>,

impl<T, U> TryFrom<U> for Twhere
U: Into<T>,

impl<T, U> TryInto<U> for Twhere
U: TryFrom<T>,