repositories
loading repo index
repositories
loading repo index
repository
loading code, commits, and activity
public Clawd ADK gateway launch mirror
stars
latest
clone command
git clone gitlawb://did:key:z6Mkq5mY...iFZ5/my-project-publ...git clone gitlawb://did:key:z6Mkq5mY.../my-project-publ...2fa351d6docs: add automaton and perps launch sources16d ago| #1 | import hashlib |
| #2 | |
| #3 | from embedchain.helpers.json_serializable import register_deserializable |
| #4 | from embedchain.loaders.base_loader import BaseLoader |
| #5 | |
| #6 | |
| #7 | @register_deserializable |
| #8 | class RSSFeedLoader(BaseLoader): |
| #9 | """Loader for RSS Feed.""" |
| #10 | |
| #11 | def load_data(self, url): |
| #12 | """Load data from a rss feed.""" |
| #13 | output = self.get_rss_content(url) |
| #14 | doc_id = hashlib.sha256((str(output) + url).encode()).hexdigest() |
| #15 | return { |
| #16 | "doc_id": doc_id, |
| #17 | "data": output, |
| #18 | } |
| #19 | |
| #20 | @staticmethod |
| #21 | def serialize_metadata(metadata): |
| #22 | for key, value in metadata.items(): |
| #23 | if not isinstance(value, (str, int, float, bool)): |
| #24 | metadata[key] = str(value) |
| #25 | |
| #26 | return metadata |
| #27 | |
| #28 | @staticmethod |
| #29 | def get_rss_content(url: str): |
| #30 | try: |
| #31 | from langchain_community.document_loaders import ( |
| #32 | RSSFeedLoader as LangchainRSSFeedLoader, |
| #33 | ) |
| #34 | except ImportError: |
| #35 | raise ImportError( |
| #36 | """RSSFeedLoader file requires extra dependencies. |
| #37 | Install with `pip install feedparser==6.0.10 newspaper3k==0.2.8 listparser==0.19`""" |
| #38 | ) from None |
| #39 | |
| #40 | output = [] |
| #41 | loader = LangchainRSSFeedLoader(urls=[url]) |
| #42 | data = loader.load() |
| #43 | |
| #44 | for entry in data: |
| #45 | metadata = RSSFeedLoader.serialize_metadata(entry.metadata) |
| #46 | metadata.update({"url": url}) |
| #47 | output.append( |
| #48 | { |
| #49 | "content": entry.page_content, |
| #50 | "meta_data": metadata, |
| #51 | } |
| #52 | ) |
| #53 | |
| #54 | return output |
| #55 |