medspacy.preprocess.preprocessor

Preprocessor

This is the medspacy Preprocessor class. It is designed as a wrapper for destructive preprocessing rules such as stripping or replacing text in a document before the text enters a spaCy pipeline.

This is NOT a spaCy component and cannot be added to a spaCy pipeline. Please use the preprocessor before calling nlp("your text here"). SpaCy only allows for non-destructive processing on the text, but that is not always advisable for every project, so this enables destructive preprocessing when required.

Source code in medspacy/preprocess/preprocessor.py

class Preprocessor:
    """
    This is the medspacy Preprocessor class. It is designed as a wrapper for destructive preprocessing rules such as
    stripping or replacing text in a document before the text enters a spaCy pipeline.

    This is NOT a spaCy component and cannot be added to a spaCy pipeline. Please use the preprocessor before
    calling `nlp("your text here")`. SpaCy only allows for non-destructive processing on the text, but that is not
    always advisable for every project, so this enables destructive preprocessing when required.
    """

    def __init__(self, tokenizer):
        """

        Args:
            tokenizer:
        """
        self.tokenizer = tokenizer
        self._rules = []

    def add(self, rules: Union[PreprocessingRule, Iterable[PreprocessingRule]]):
        """
        Adds a PreprocessingRule or collection of PreprocessingRules to the Preprocessor.

        Args:
            rules: A single PreprocessingRule or a collection of PreprocessingRules to add.
        """
        if isinstance(rules, PreprocessingRule):
            rules = [rules]
        for rule in rules:
            if not isinstance(rule, PreprocessingRule):
                raise TypeError(
                    f"Each rule must be an instance of PreprocessingRule, not {type(rule)}."
                )
        self._rules += rules

    def __call__(self, text, tokenize=True) -> Union[str, Doc]:
        """

        Args:
            text:
            tokenize:

        Returns:

        """
        for rule in self._rules:
            text = rule(text)

        if not tokenize:
            return text

        return self.tokenizer(text)

__call__(text, tokenize=True)

Parameters:

Name	Type	Description	Default
text			required
tokenize			True

Returns:

Source code in medspacy/preprocess/preprocessor.py

def __call__(self, text, tokenize=True) -> Union[str, Doc]:
    """

    Args:
        text:
        tokenize:

    Returns:

    """
    for rule in self._rules:
        text = rule(text)

    if not tokenize:
        return text

    return self.tokenizer(text)

__init__(tokenizer)

Parameters:

Name	Type	Description	Default
tokenizer			required

Source code in medspacy/preprocess/preprocessor.py

def __init__(self, tokenizer):
    """

    Args:
        tokenizer:
    """
    self.tokenizer = tokenizer
    self._rules = []

add(rules)

Adds a PreprocessingRule or collection of PreprocessingRules to the Preprocessor.

Parameters:

Name	Type	Description	Default
rules	Union[PreprocessingRule, Iterable[PreprocessingRule]]	A single PreprocessingRule or a collection of PreprocessingRules to add.	required

Source code in medspacy/preprocess/preprocessor.py

def add(self, rules: Union[PreprocessingRule, Iterable[PreprocessingRule]]):
    """
    Adds a PreprocessingRule or collection of PreprocessingRules to the Preprocessor.

    Args:
        rules: A single PreprocessingRule or a collection of PreprocessingRules to add.
    """
    if isinstance(rules, PreprocessingRule):
        rules = [rules]
    for rule in rules:
        if not isinstance(rule, PreprocessingRule):
            raise TypeError(
                f"Each rule must be an instance of PreprocessingRule, not {type(rule)}."
            )
    self._rules += rules