repositories
loading repo index
repositories
loading repo index
repository
loading code, commits, and activity
public Clawd ADK gateway launch mirror
stars
latest
clone command
git clone gitlawb://did:key:z6Mkq5mY...iFZ5/my-project-publ...git clone gitlawb://did:key:z6Mkq5mY.../my-project-publ...2fa351d6docs: add automaton and perps launch sources16d ago| #1 | import hashlib |
| #2 | import os |
| #3 | |
| #4 | from embedchain.helpers.json_serializable import register_deserializable |
| #5 | from embedchain.loaders.base_loader import BaseLoader |
| #6 | |
| #7 | |
| #8 | @register_deserializable |
| #9 | class TextFileLoader(BaseLoader): |
| #10 | def load_data(self, url: str): |
| #11 | """Load data from a text file located at a local path.""" |
| #12 | if not os.path.exists(url): |
| #13 | raise FileNotFoundError(f"The file at {url} does not exist.") |
| #14 | |
| #15 | with open(url, "r", encoding="utf-8") as file: |
| #16 | content = file.read() |
| #17 | |
| #18 | doc_id = hashlib.sha256((content + url).encode()).hexdigest() |
| #19 | |
| #20 | metadata = {"url": url, "file_size": os.path.getsize(url), "file_type": url.split(".")[-1]} |
| #21 | |
| #22 | return { |
| #23 | "doc_id": doc_id, |
| #24 | "data": [ |
| #25 | { |
| #26 | "content": content, |
| #27 | "meta_data": metadata, |
| #28 | } |
| #29 | ], |
| #30 | } |
| #31 |