repositories
loading repo index
repositories
loading repo index
repository
loading code, commits, and activity
public Clawd ADK gateway launch mirror
stars
latest
clone command
git clone gitlawb://did:key:z6Mkq5mY...iFZ5/my-project-publ...git clone gitlawb://did:key:z6Mkq5mY.../my-project-publ...2fa351d6docs: add automaton and perps launch sources16d ago| #1 | import builtins |
| #2 | import logging |
| #3 | from collections.abc import Callable |
| #4 | from importlib import import_module |
| #5 | from typing import Optional |
| #6 | |
| #7 | from embedchain.config.base_config import BaseConfig |
| #8 | from embedchain.helpers.json_serializable import register_deserializable |
| #9 | |
| #10 | |
| #11 | @register_deserializable |
| #12 | class ChunkerConfig(BaseConfig): |
| #13 | """ |
| #14 | Config for the chunker used in `add` method |
| #15 | """ |
| #16 | |
| #17 | def __init__( |
| #18 | self, |
| #19 | chunk_size: Optional[int] = 2000, |
| #20 | chunk_overlap: Optional[int] = 0, |
| #21 | length_function: Optional[Callable[[str], int]] = None, |
| #22 | min_chunk_size: Optional[int] = 0, |
| #23 | ): |
| #24 | self.chunk_size = chunk_size |
| #25 | self.chunk_overlap = chunk_overlap |
| #26 | self.min_chunk_size = min_chunk_size |
| #27 | if self.min_chunk_size >= self.chunk_size: |
| #28 | raise ValueError(f"min_chunk_size {min_chunk_size} should be less than chunk_size {chunk_size}") |
| #29 | if self.min_chunk_size < self.chunk_overlap: |
| #30 | logging.warning( |
| #31 | f"min_chunk_size {min_chunk_size} should be greater than chunk_overlap {chunk_overlap}, otherwise it is redundant." # noqa:E501 |
| #32 | ) |
| #33 | |
| #34 | if isinstance(length_function, str): |
| #35 | self.length_function = self.load_func(length_function) |
| #36 | else: |
| #37 | self.length_function = length_function if length_function else len |
| #38 | |
| #39 | @staticmethod |
| #40 | def load_func(dotpath: str): |
| #41 | if "." not in dotpath: |
| #42 | return getattr(builtins, dotpath) |
| #43 | else: |
| #44 | module_, func = dotpath.rsplit(".", maxsplit=1) |
| #45 | m = import_module(module_) |
| #46 | return getattr(m, func) |
| #47 | |
| #48 | |
| #49 | @register_deserializable |
| #50 | class LoaderConfig(BaseConfig): |
| #51 | """ |
| #52 | Config for the loader used in `add` method |
| #53 | """ |
| #54 | |
| #55 | def __init__(self): |
| #56 | pass |
| #57 | |
| #58 | |
| #59 | @register_deserializable |
| #60 | class AddConfig(BaseConfig): |
| #61 | """ |
| #62 | Config for the `add` method. |
| #63 | """ |
| #64 | |
| #65 | def __init__( |
| #66 | self, |
| #67 | chunker: Optional[ChunkerConfig] = None, |
| #68 | loader: Optional[LoaderConfig] = None, |
| #69 | ): |
| #70 | """ |
| #71 | Initializes a configuration class instance for the `add` method. |
| #72 | |
| #73 | :param chunker: Chunker config, defaults to None |
| #74 | :type chunker: Optional[ChunkerConfig], optional |
| #75 | :param loader: Loader config, defaults to None |
| #76 | :type loader: Optional[LoaderConfig], optional |
| #77 | """ |
| #78 | self.loader = loader |
| #79 | self.chunker = chunker |
| #80 |