Struct llamapun::parallel_data::corpus::Corpus

source · [−]

pub struct Corpus {
    pub path: String,
    pub xml_parser: Parser,
    pub html_parser: Parser,
    pub tokenizer: Tokenizer,
    pub dnm_parameters: DNMParameters,
    pub extension: Option<String>,
}

Expand description

A parallel iterable Corpus of HTML5 documents

Fields

path: String

root directory

xml_parser: Parser

document XHTML5 parser

html_parser: Parser

document HTML5 parser

tokenizer: Tokenizer

DNM-aware sentence and word tokenizer

dnm_parameters: DNMParameters

Default setting for DNM generation

extension: Option<String>

Extension of corpus files (for specially tailored resources such as DLMF’s .html5) defaults to selecting .html AND .xhtml files

Implementations

source

impl Corpus

source

pub fn new(dirpath: String) -> Self

Create a new parallel-processing corpus with the base directory dirpath

source

pub fn catalog_with_parallel_walk<F>(&self, closure: F) -> HashMap<String, u64>where
F: Fn(Document<'_>) -> HashMap<String, u64> + Send + Sync,

Get a parallel iterator over the documents

Trait Implementations

source

impl Default for Corpus

source

fn default() -> Corpus

Returns the “default value” for a type. Read more

Auto Trait Implementations

impl RefUnwindSafe for Corpus

impl Send for Corpus

impl Sync for Corpus

impl Unpin for Corpus

impl UnwindSafe for Corpus

Blanket Implementations

source

impl<T> Any for Twhere
T: 'static + ?Sized,

source

fn type_id(&self) -> TypeId

Gets the TypeId of self. Read more

source

impl<T> Borrow<T> for Twhere
T: ?Sized,

const: unstable · source

fn borrow(&self) -> &T

Immutably borrows from an owned value. Read more

source

impl<T> BorrowMut<T> for Twhere
T: ?Sized,

const: unstable · source

fn borrow_mut(&mut self) -> &mut T

Mutably borrows from an owned value. Read more

source

impl<T> From<T> for T

const: unstable · source

fn from(t: T) -> T

Returns the argument unchanged.

source

impl<T, U> Into for Twhere
U: From<T>,

const: unstable · source

fn into(self) -> U

Calls U::from(self).

That is, this conversion is whatever the implementation of From<T> for U chooses to do.

impl<T> Pointable for T

const ALIGN: usize = mem::align_of::<T>()

The alignment of pointer.

type Init = T

The type for initializers.

unsafe fn init(init: <T as Pointable>::Init) -> usize

Initializes a with the given initializer. Read more

unsafe fn deref<'a>(ptr: usize) -> &'a T

Dereferences the given pointer. Read more

unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut T

Mutably dereferences the given pointer. Read more

unsafe fn drop(ptr: usize)

Drops the object pointed to by the given pointer. Read more

source

impl<T, U> TryFrom for Twhere
U: Into<T>,

type Error = Infallible

The type returned in the event of a conversion error.

const: unstable · source

fn try_from(value: U) -> Result<T, <T as TryFrom>::Error>

Performs the conversion.

source

impl<T, U> TryInto for Twhere
U: TryFrom<T>,

type Error = >::Error

The type returned in the event of a conversion error.

const: unstable · source

fn try_into(self) -> Result<U, >::Error>

Performs the conversion.

Struct llamapun::parallel_data::corpus::Corpus

Fields

Implementations

impl Corpus

pub fn new(dirpath: String) -> Self

pub fn catalog_with_parallel_walk<F>(&self, closure: F) -> HashMap<String, u64>where
F: Fn(Document<'_>) -> HashMap<String, u64> + Send + Sync,

Trait Implementations

impl Default for Corpus

fn default() -> Corpus

Auto Trait Implementations

impl RefUnwindSafe for Corpus

impl Send for Corpus

impl Sync for Corpus

impl Unpin for Corpus

impl UnwindSafe for Corpus

Blanket Implementations

impl<T> Any for Twhere
T: 'static + ?Sized,

fn type_id(&self) -> TypeId

impl<T> Borrow<T> for Twhere
T: ?Sized,

fn borrow(&self) -> &T

impl<T> BorrowMut<T> for Twhere
T: ?Sized,

fn borrow_mut(&mut self) -> &mut T

impl<T> From<T> for T

fn from(t: T) -> T

impl<T, U> Into<U> for Twhere
U: From<T>,

fn into(self) -> U

impl<T> Pointable for T

const ALIGN: usize = mem::align_of::<T>()

type Init = T

unsafe fn init(init: <T as Pointable>::Init) -> usize

unsafe fn deref<'a>(ptr: usize) -> &'a T

unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut T

unsafe fn drop(ptr: usize)

impl<T, U> TryFrom<U> for Twhere
U: Into<T>,

type Error = Infallible

fn try_from(value: U) -> Result<T, <T as TryFrom<U>>::Error>

impl<T, U> TryInto<U> for Twhere
U: TryFrom<T>,

type Error = <U as TryFrom<T>>::Error

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

Struct llamapun::parallel_data::corpus::Corpus

Fields

Implementations

impl Corpus

pub fn new(dirpath: String) -> Self

pub fn catalog_with_parallel_walk<F>(&self, closure: F) -> HashMap<String, u64>where F: Fn(Document<'_>) -> HashMap<String, u64> + Send + Sync,

Trait Implementations

impl Default for Corpus

fn default() -> Corpus

Auto Trait Implementations

impl RefUnwindSafe for Corpus

impl Send for Corpus

impl Sync for Corpus

impl Unpin for Corpus

impl UnwindSafe for Corpus

Blanket Implementations

impl<T> Any for Twhere T: 'static + ?Sized,

fn type_id(&self) -> TypeId

impl<T> Borrow<T> for Twhere T: ?Sized,

fn borrow(&self) -> &T

impl<T> BorrowMut<T> for Twhere T: ?Sized,

fn borrow_mut(&mut self) -> &mut T

impl<T> From<T> for T

fn from(t: T) -> T

impl<T, U> Into<U> for Twhere U: From<T>,

fn into(self) -> U

impl<T> Pointable for T

const ALIGN: usize = mem::align_of::<T>()

type Init = T

unsafe fn init(init: <T as Pointable>::Init) -> usize

unsafe fn deref<'a>(ptr: usize) -> &'a T

unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut T

unsafe fn drop(ptr: usize)

impl<T, U> TryFrom<U> for Twhere U: Into<T>,

type Error = Infallible

fn try_from(value: U) -> Result<T, <T as TryFrom<U>>::Error>

impl<T, U> TryInto<U> for Twhere U: TryFrom<T>,

type Error = <U as TryFrom<T>>::Error

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

pub fn catalog_with_parallel_walk<F>(&self, closure: F) -> HashMap<String, u64>where
F: Fn(Document<'_>) -> HashMap<String, u64> + Send + Sync,

impl<T> Any for Twhere
T: 'static + ?Sized,

impl<T> Borrow<T> for Twhere
T: ?Sized,

impl<T> BorrowMut<T> for Twhere
T: ?Sized,

impl<T, U> Into<U> for Twhere
U: From<T>,

impl<T, U> TryFrom<U> for Twhere
U: Into<T>,

impl<T, U> TryInto<U> for Twhere
U: TryFrom<T>,