my-project-public

repository

loading code, commits, and activity

repositories

loading repo index

my-project-public — gitlawb

#1	"""
#2	Mnemosyne Token Counter
#3	Lightweight token estimation for memory context benchmarking.
#4	Uses tiktoken if available, falls back to chars/4 heuristic.
#5	"""
#6
#7	import os
#8	from typing import Optional
#9
#10	# Try to import tiktoken, but don't fail if missing
#11	try:
#12	import tiktoken
#13	_TIKTOKEN_AVAILABLE = True
#14	_ENCODING = tiktoken.get_encoding("cl100k_base")
#15	except Exception:
#16	_TIKTOKEN_AVAILABLE = False
#17	_ENCODING = None
#18
#19
#20	def estimate_tokens(text: str, model: str = "default") -> int:
#21	"""
#22	Estimate token count for a given text.
#23
#24	Args:
#25	text: The text to estimate
#26	model: Optional model hint (claude, gpt4, etc.)
#27
#28	Returns:
#29	Estimated token count
#30	"""
#31	if not text:
#32	return 0
#33
#34	if _TIKTOKEN_AVAILABLE and _ENCODING is not None:
#35	try:
#36	return len(_ENCODING.encode(text))
#37	except Exception:
#38	pass
#39
#40	# Fallback: chars/4 is a decent approximation for English/Spanish
#41	return len(text) // 4
#42
#43
#44	def estimate_cost(tokens: int, model: str = "claude-sonnet-4") -> dict:
#45	"""
#46	Estimate API cost for injected prompt tokens.
#47
#48	Args:
#49	tokens: Number of prompt tokens
#50	model: Model identifier for pricing lookup
#51
#52	Returns:
#53	Dictionary with cost estimates for common models
#54	"""
#55	# Pricing per 1M tokens (input) - update as needed
#56	PRICING = {
#57	"claude-sonnet-4": 3.00,
#58	"claude-haiku": 0.80,
#59	"gpt-4o": 2.50,
#60	"gpt-4o-mini": 0.15,
#61	"default": 3.00,
#62	}
#63
#64	rate = PRICING.get(model, PRICING["default"])
#65	cost = (tokens / 1_000_000) * rate
#66
#67	return {
#68	"tokens": tokens,
#69	"model": model,
#70	"cost_usd": round(cost, 6),
#71	"rate_per_1m": rate,
#72	}
#73

z6Mkq5mY3JWtxoxUobWcfNHm7AkRubgSWEZTkBVqZXJviFZ5/my-project-public