my-project-public

repository

loading code, commits, and activity

repositories

loading repo index

#1	import hashlib
#2	import re
#3
#4	try:
#5	from googleapiclient.errors import HttpError
#6	except ImportError:
#7	raise ImportError(
#8	"Google Drive requires extra dependencies. Install with `pip install embedchain[googledrive]`"
#9	) from None
#10
#11	from langchain_community.document_loaders import GoogleDriveLoader as Loader
#12
#13	try:
#14	import unstructured # noqa: F401
#15	from langchain_community.document_loaders import UnstructuredFileIOLoader
#16	except ImportError:
#17	raise ImportError(
#18	'Unstructured file requires extra dependencies. Install with `pip install "unstructured[local-inference, all-docs]"`' # noqa: E501
#19	) from None
#20
#21	from embedchain.helpers.json_serializable import register_deserializable
#22	from embedchain.loaders.base_loader import BaseLoader
#23
#24
#25	@register_deserializable
#26	class GoogleDriveLoader(BaseLoader):
#27	@staticmethod
#28	def _get_drive_id_from_url(url: str):
#29	regex = r"^https:\/\/drive\.google\.com\/drive\/(?:u\/\d+\/)folders\/([a-zA-Z0-9_-]+)$"
#30	if re.match(regex, url):
#31	return url.split("/")[-1]
#32	raise ValueError(
#33	f"The url provided {url} does not match a google drive folder url. Example drive url: "
#34	f"https://drive.google.com/drive/u/0/folders/xxxx"
#35	)
#36
#37	def load_data(self, url: str):
#38	"""Load data from a Google drive folder."""
#39	folder_id: str = self._get_drive_id_from_url(url)
#40
#41	try:
#42	loader = Loader(
#43	folder_id=folder_id,
#44	recursive=True,
#45	file_loader_cls=UnstructuredFileIOLoader,
#46	)
#47
#48	data = []
#49	all_content = []
#50
#51	docs = loader.load()
#52	for doc in docs:
#53	all_content.append(doc.page_content)
#54	# renames source to url for later use.
#55	doc.metadata["url"] = doc.metadata.pop("source")
#56	data.append({"content": doc.page_content, "meta_data": doc.metadata})
#57
#58	doc_id = hashlib.sha256((" ".join(all_content) + url).encode()).hexdigest()
#59	return {"doc_id": doc_id, "data": data}
#60
#61	except HttpError:
#62	raise FileNotFoundError("Unable to locate folder or files, check provided drive URL and try again")
#63

z6Mkq5mY3JWtxoxUobWcfNHm7AkRubgSWEZTkBVqZXJviFZ5/my-project-public