Thorsten
419f5e3a89
RAG: switch Postgres to pgvector, chunk and embed the three D&D rulebooks locally via sentence-transformers, and retrieve relevant excerpts per DM turn (query = latest player message) to ground the system prompt. Retrieval runs off the event loop and is capped by a relevance threshold and a max character budget so it can't blow up context size or cost. Game setup wizard: creating a game now opens a short chat where the DM asks about genre, length, and the player's experience level, then proposes a name and description via a tool call. The player can edit both before creating the game. Stateless endpoint — the frontend carries the conversation, no DB needed since the game doesn't exist yet. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
34 lines
925 B
Python
34 lines
925 B
Python
import re
|
|
|
|
# Rough word-count targets standing in for the 300-800 token / 10-20% overlap guideline —
|
|
# English prose runs ~0.75 words per token, so ~380 words ≈ 500 tokens.
|
|
WORDS_PER_CHUNK = 380
|
|
OVERLAP_WORDS = 60
|
|
|
|
_WHITESPACE_RE = re.compile(r"[ \t]+")
|
|
_BLANK_LINES_RE = re.compile(r"\n{3,}")
|
|
|
|
|
|
def _normalize(text: str) -> str:
|
|
text = _WHITESPACE_RE.sub(" ", text)
|
|
text = _BLANK_LINES_RE.sub("\n\n", text)
|
|
return text.strip()
|
|
|
|
|
|
def chunk_text(
|
|
text: str, words_per_chunk: int = WORDS_PER_CHUNK, overlap_words: int = OVERLAP_WORDS
|
|
) -> list[str]:
|
|
words = _normalize(text).split()
|
|
if not words:
|
|
return []
|
|
|
|
step = words_per_chunk - overlap_words
|
|
chunks = []
|
|
start = 0
|
|
while start < len(words):
|
|
chunks.append(" ".join(words[start : start + words_per_chunk]))
|
|
if start + words_per_chunk >= len(words):
|
|
break
|
|
start += step
|
|
return chunks
|