repositories
loading repo index
repositories
loading repo index
repository
loading code, commits, and activity
public Clawd ADK gateway launch mirror
stars
latest
clone command
git clone gitlawb://did:key:z6Mkq5mY...iFZ5/my-project-publ...git clone gitlawb://did:key:z6Mkq5mY.../my-project-publ...2fa351d6docs: add automaton and perps launch sources16d ago| #1 | import hashlib |
| #2 | |
| #3 | from langchain_community.document_loaders import PyPDFLoader |
| #4 | |
| #5 | from embedchain.helpers.json_serializable import register_deserializable |
| #6 | from embedchain.loaders.base_loader import BaseLoader |
| #7 | from embedchain.utils.misc import clean_string |
| #8 | |
| #9 | |
| #10 | @register_deserializable |
| #11 | class PdfFileLoader(BaseLoader): |
| #12 | def load_data(self, url): |
| #13 | """Load data from a PDF file.""" |
| #14 | headers = { |
| #15 | "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36", # noqa:E501 |
| #16 | } |
| #17 | loader = PyPDFLoader(url, headers=headers) |
| #18 | data = [] |
| #19 | all_content = [] |
| #20 | pages = loader.load_and_split() |
| #21 | if not len(pages): |
| #22 | raise ValueError("No data found") |
| #23 | for page in pages: |
| #24 | content = page.page_content |
| #25 | content = clean_string(content) |
| #26 | metadata = page.metadata |
| #27 | metadata["url"] = url |
| #28 | data.append( |
| #29 | { |
| #30 | "content": content, |
| #31 | "meta_data": metadata, |
| #32 | } |
| #33 | ) |
| #34 | all_content.append(content) |
| #35 | doc_id = hashlib.sha256((" ".join(all_content) + url).encode()).hexdigest() |
| #36 | return { |
| #37 | "doc_id": doc_id, |
| #38 | "data": data, |
| #39 | } |
| #40 |