Method __init__

hanlp/common/dataset.py:776–786

Builds a :class:`~hanlp.common.dataset.SortingSampler`. Args: batch_max_tokens: Maximum tokens per batch. use_effective_tokens: Whether to calculate effective number of tokens when applying the `batch_max_tokens`. batch_size: Maximum samples per batch.

(self, batch_size=None, batch_max_tokens=None, use_effective_tokens=False)

class SortingSamplerBuilder(SortingSampler, SamplerBuilder):
    # noinspection PyMissingConstructor
    def __init__(self, batch_size=None, batch_max_tokens=None, use_effective_tokens=False) -> None:
        """Builds a :class:`~hanlp.common.dataset.SortingSampler`.

        Args:
            batch_max_tokens: Maximum tokens per batch.
            use_effective_tokens: Whether to calculate effective number of tokens when applying the `batch_max_tokens`.
            batch_size: Maximum samples per batch.
        """
        self.use_effective_tokens = use_effective_tokens
        self.batch_max_tokens = batch_max_tokens
        self.batch_size = batch_size

    def build(self, lengths: List[int], shuffle=False, gradient_accumulation=1, **kwargs) -> Sampler:
        batch_size, batch_max_tokens = self.scale(gradient_accumulation)