repositories
loading repo index
repositories
loading repo index
repository
loading code, commits, and activity
public Clawd ADK gateway launch mirror
stars
latest
clone command
git clone gitlawb://did:key:z6Mkq5mY...iFZ5/my-project-publ...git clone gitlawb://did:key:z6Mkq5mY.../my-project-publ...2fa351d6docs: add automaton and perps launch sources16d ago| #1 | import hashlib |
| #2 | |
| #3 | try: |
| #4 | import unstructured # noqa: F401 |
| #5 | from langchain_community.document_loaders import UnstructuredXMLLoader |
| #6 | except ImportError: |
| #7 | raise ImportError( |
| #8 | 'XML file requires extra dependencies. Install with `pip install "unstructured[local-inference, all-docs]"`' |
| #9 | ) from None |
| #10 | from embedchain.helpers.json_serializable import register_deserializable |
| #11 | from embedchain.loaders.base_loader import BaseLoader |
| #12 | from embedchain.utils.misc import clean_string |
| #13 | |
| #14 | |
| #15 | @register_deserializable |
| #16 | class XmlLoader(BaseLoader): |
| #17 | def load_data(self, xml_url): |
| #18 | """Load data from a XML file.""" |
| #19 | loader = UnstructuredXMLLoader(xml_url) |
| #20 | data = loader.load() |
| #21 | content = data[0].page_content |
| #22 | content = clean_string(content) |
| #23 | metadata = data[0].metadata |
| #24 | metadata["url"] = metadata["source"] |
| #25 | del metadata["source"] |
| #26 | output = [{"content": content, "meta_data": metadata}] |
| #27 | doc_id = hashlib.sha256((content + xml_url).encode()).hexdigest() |
| #28 | return { |
| #29 | "doc_id": doc_id, |
| #30 | "data": output, |
| #31 | } |
| #32 |