# ---- Part 0.1: Imports, configuration, logging ----
from __future__ import annotations

import hashlib
import json
import logging
import os
import re
import shutil
import subprocess
import sys
import time
import uuid
from dataclasses import dataclass, asdict, field
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable, Sequence
from urllib.parse import urlparse
from urllib.robotparser import RobotFileParser

import requests
from dotenv import load_dotenv

# --- structured logging (rubric C: "sensible logging / progress") -------------
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s | %(levelname)-7s | %(message)s",
    datefmt="%H:%M:%S",
    stream=sys.stdout,
    force=True,
)
log = logging.getLogger("cs8")

# The root logger at INFO also switches on every third-party library that logs at
# INFO — httpx emits one line per ChromaDB call, huggingface_hub one per metadata
# HEAD request. On the first run that buried the graded output (40 of the 66 lines
# in the storage-verification cell were HTTP chatter). The pipeline's own logging
# stays at INFO; the libraries are raised to WARNING so failures still surface.
for _noisy in ("httpx", "httpcore", "urllib3", "requests", "filelock",
               "huggingface_hub", "transformers", "sentence_transformers",
               "chromadb", "chromadb.telemetry", "PIL", "matplotlib",
               "matplotlib.font_manager", "asyncio"):
    logging.getLogger(_noisy).setLevel(logging.WARNING)

import warnings
warnings.filterwarnings("ignore", category=FutureWarning, module="sentence_transformers")

# --- configuration ------------------------------------------------------------
load_dotenv()

NOTEBOOK_DIR = Path.cwd()
EVIDENCE_DIR = NOTEBOOK_DIR / "evidence"
EVIDENCE_DIR.mkdir(exist_ok=True)

CHROMA_HOST     = os.getenv("CHROMA_HOST", "localhost")
CHROMA_PORT     = int(os.getenv("CHROMA_PORT", "8000"))
COLLECTION_NAME = os.getenv("COLLECTION_NAME", "web_scraping_collection")

EMBEDDING_MODEL     = os.getenv("EMBEDDING_MODEL", "all-MiniLM-L6-v2")
ALT_EMBEDDING_MODEL = os.getenv("ALT_EMBEDDING_MODEL", "all-mpnet-base-v2")

CHUNK_SIZE    = int(os.getenv("CHUNK_SIZE", "1000"))
CHUNK_OVERLAP = int(os.getenv("CHUNK_OVERLAP", "200"))
SPACE_TYPE    = os.getenv("SPACE_TYPE", "cosine")          # cosine | l2 | ip

OLLAMA_BASE_URL = os.getenv("OLLAMA_BASE_URL", "http://localhost:11434")
OLLAMA_MODEL    = os.getenv("OLLAMA_MODEL", "llama3.2:3b")
WEBUI_URL       = os.getenv("WEBUI_URL", "http://localhost:3000")

REQUEST_DELAY = float(os.getenv("REQUEST_DELAY", "1.5"))   # polite rate limiting
USER_AGENT    = os.getenv("USER_AGENT", "IT721-CaseStudy8-EducationalScraper/1.0")
FAST_MODE     = os.getenv("FAST_MODE", "0") == "1"         # 1 = reduced experiment grid

RUN_ID = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")

def save_evidence(name: str, payload: Any) -> Path:
    """Persist a JSON evidence artifact (used by the written report)."""
    path = EVIDENCE_DIR / name
    path.write_text(json.dumps(payload, indent=2, ensure_ascii=False, default=str), encoding="utf-8")
    log.info("evidence written -> %s", path.name)
    return path

log.info("Run ID              : %s", RUN_ID)
log.info("ChromaDB            : http://%s:%s (space=%s)", CHROMA_HOST, CHROMA_PORT, SPACE_TYPE)
log.info("Collection          : %s", COLLECTION_NAME)
log.info("Embedding (primary) : %s", EMBEDDING_MODEL)
log.info("Embedding (alt)     : %s", ALT_EMBEDDING_MODEL)
log.info("Chunking default    : size=%d overlap=%d", CHUNK_SIZE, CHUNK_OVERLAP)
log.info("Fast mode           : %s", FAST_MODE)

23:30:54 | INFO    | Run ID              : 20260831T053054Z
23:30:54 | INFO    | ChromaDB            : http://localhost:8000 (space=cosine)
23:30:54 | INFO    | Collection          : web_scraping_collection
23:30:54 | INFO    | Embedding (primary) : all-MiniLM-L6-v2
23:30:54 | INFO    | Embedding (alt)     : all-mpnet-base-v2
23:30:54 | INFO    | Chunking default    : size=1000 overlap=200
23:30:54 | INFO    | Fast mode           : False
23:30:54 | INFO    | ChromaDB            : http://localhost:8000 (space=cosine)
23:30:54 | INFO    | Collection          : web_scraping_collection
23:30:54 | INFO    | Embedding (primary) : all-MiniLM-L6-v2
23:30:54 | INFO    | Embedding (alt)     : all-mpnet-base-v2
23:30:54 | INFO    | Chunking default    : size=1000 overlap=200
23:30:54 | INFO    | Fast mode           : False

# ---- Part 0.2: Container & service verification (deliverable: screenshot evidence) ----
# Proves the three required Docker containers are up BEFORE any pipeline work starts.

def _docker_bin() -> str | None:
    """Docker Desktop on macOS is not always on the Jupyter kernel PATH."""
    for cand in ("docker", "/usr/local/bin/docker", "/opt/homebrew/bin/docker",
                 str(Path.home() / ".docker/bin/docker")):
        if shutil.which(cand) or Path(cand).exists():
            return shutil.which(cand) or cand
    return None

def docker_ps() -> tuple[str, list[dict]]:
    """Return (raw table, parsed rows) for running containers."""
    docker = _docker_bin()
    if not docker:
        log.warning("docker CLI not found on PATH — capture the Docker Desktop screenshot manually")
        return "", []
    fmt = "{{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}"
    try:
        raw = subprocess.run([docker, "ps", "--format", fmt],
                             capture_output=True, text=True, timeout=30, check=True).stdout.strip()
    except Exception as exc:                                    # noqa: BLE001
        log.error("docker ps failed: %s", exc)
        return "", []
    rows = []
    for line in raw.splitlines():
        parts = line.split("\t")
        if len(parts) >= 3:
            rows.append({"name": parts[0], "image": parts[1], "status": parts[2],
                         "ports": parts[3] if len(parts) > 3 else ""})
    return raw, rows

def probe(name: str, url: str, timeout: float = 8.0) -> dict:
    """HTTP liveness probe with a compact, report-friendly result."""
    started = time.perf_counter()
    try:
        r = requests.get(url, timeout=timeout)
        ok = r.status_code < 400
        body = r.text[:180].replace("\n", " ")
    except Exception as exc:                                    # noqa: BLE001
        ok, r, body = False, None, f"{type(exc).__name__}: {exc}"
    return {"service": name, "url": url, "ok": ok,
            "status_code": getattr(r, "status_code", None),
            "latency_ms": round((time.perf_counter() - started) * 1000, 1),
            "sample": body}

REQUIRED_CONTAINERS = ["chromadb-server", "ollama", "open-webui"]

raw_ps, ps_rows = docker_ps()
running = {r["name"] for r in ps_rows}

print("=" * 96)
print("DOCKER CONTAINERS")
print("=" * 96)
print(raw_ps or "(docker CLI unavailable from the kernel — see screenshots/docker_containers.png)")
print()
for c in REQUIRED_CONTAINERS:
    mark = "OK    " if c in running else "MISSING"
    print(f"  [{mark}] {c}")

probes = [
    probe("ChromaDB heartbeat", f"http://{CHROMA_HOST}:{CHROMA_PORT}/api/v2/heartbeat"),
    probe("Ollama tags",        f"{OLLAMA_BASE_URL}/api/tags"),
    probe("OpenWebUI",          WEBUI_URL),
]
# ChromaDB v1 API fallback for older server images
if not probes[0]["ok"]:
    probes[0] = probe("ChromaDB heartbeat", f"http://{CHROMA_HOST}:{CHROMA_PORT}/api/v1/heartbeat")

print()
print("=" * 96)
print("SERVICE PROBES")
print("=" * 96)
for p in probes:
    print(f"  [{'OK ' if p['ok'] else 'DOWN'}] {p['service']:<20} {p['url']:<46} "
          f"{p['latency_ms']:>7.1f} ms  {str(p['sample'])[:60]}")

env_evidence = {"run_id": RUN_ID, "captured_at": datetime.now(timezone.utc).isoformat(),
                "docker_ps_raw": raw_ps, "containers": ps_rows,
                "required_containers": {c: (c in running) for c in REQUIRED_CONTAINERS},
                "probes": probes,
                "python": sys.version.split()[0], "platform": sys.platform}
save_evidence("00_environment.json", env_evidence)

if not all(p["ok"] for p in probes[:1]):
    log.error("ChromaDB is not reachable — start it with ./setup_containers.sh before continuing.")

================================================================================================
DOCKER CONTAINERS
================================================================================================
open-webui	ghcr.io/open-webui/open-webui:main	Up 6 hours (healthy)	0.0.0.0:3000->8080/tcp, [::]:3000->8080/tcp
ollama	ollama/ollama	Up 6 hours	0.0.0.0:11434->11434/tcp, [::]:11434->11434/tcp
chromadb-server	chromadb/chroma:latest	Up 6 hours	0.0.0.0:8000->8000/tcp, [::]:8000->8000/tcp

  [OK    ] chromadb-server
  [OK    ] ollama
  [OK    ] open-webui

================================================================================================
SERVICE PROBES
================================================================================================
  [OK ] ChromaDB heartbeat   http://localhost:8000/api/v2/heartbeat             7.9 ms  {"nanosecond heartbeat":1788154254648722590}
  [OK ] Ollama tags          http://localhost:11434/api/tags                    3.2 ms  {"models":[{"name":"llama3.2:3b","model":"llama3.2:3b","modi
  [OK ] OpenWebUI            http://localhost:3000                              4.2 ms  <!doctype html> <html lang="en"> 	<head> 		<meta charset="ut
23:30:54 | INFO    | evidence written -> 00_environment.json

# ---- Part 1.1: Source registry with documented fallbacks ----

@dataclass
class Source:
    slug: str
    url: str
    category: str
    register: str
    fallback: str | None = None
    blocked: bool = False          # set by the robots.txt gate in Part 1.2

SOURCES: list[Source] = [
    Source(
        slug="wikipedia",
        url="https://en.wikipedia.org/wiki/Retrieval-augmented_generation",
        category="Educational / encyclopedic",
        register="neutral, citation-dense",
        # cross-domain fallback: a same-domain fallback cannot rescue a
        # domain-level robots.txt block
        fallback="https://www.cloudflare.com/learning/ai/retrieval-augmented-generation-rag/",
    ),
    Source(
        slug="aws",
        url="https://aws.amazon.com/what-is/retrieval-augmented-generation/",
        category="Cloud-vendor documentation",
        register="instructional, architectural",
        fallback="https://www.ibm.com/think/topics/retrieval-augmented-generation",
    ),
    Source(
        slug="nvidia",
        url="https://blogs.nvidia.com/blog/what-is-retrieval-augmented-generation/",
        category="Industry blog",
        register="narrative, analogy-driven",
        fallback="https://en.wikipedia.org/wiki/Large_language_model",
    ),
]

# Rubric requirement: 3 URLs from 3 DIFFERENT domains — asserted, not assumed.
domains = [urlparse(s.url).netloc for s in SOURCES]
assert len(set(domains)) == 3, f"URLs must come from 3 distinct domains, got {domains}"

print(f"{'#':<3}{'DOMAIN':<24}{'CATEGORY':<32}URL")
print("-" * 120)
for i, s in enumerate(SOURCES, 1):
    print(f"{i:<3}{urlparse(s.url).netloc:<24}{s.category:<32}{s.url}")

#  DOMAIN                  CATEGORY                        URL
------------------------------------------------------------------------------------------------------------------------
1  en.wikipedia.org        Educational / encyclopedic      https://en.wikipedia.org/wiki/Retrieval-augmented_generation
2  aws.amazon.com          Cloud-vendor documentation      https://aws.amazon.com/what-is/retrieval-augmented-generation/
3  blogs.nvidia.com        Industry blog                   https://blogs.nvidia.com/blog/what-is-retrieval-augmented-generation/

# ---- Part 1.2: robots.txt compliance gate ----
#
# NOTE — a real defect found on the first run and corrected here.
# `RobotFileParser.read()` fetches robots.txt with urllib's default
# `Python-urllib/3.x` User-Agent. Several large sites (Wikimedia among them)
# answer that UA with HTTP 403. On 401/403 the parser sets `disallow_all = True`,
# so the first run reported en.wikipedia.org as "disallowed" for a page that is
# in fact perfectly crawlable — and, worse, the scraper went on to fetch it
# anyway, so the log and the behaviour disagreed.
#
# Two corrections:
#   1. robots.txt is fetched with `requests` using the SAME identifying
#      User-Agent that the page requests use, so the verdict reflects reality.
#   2. The verdict is now ENFORCED: a source that stays disallowed after the
#      fallback is marked `blocked` and is never fetched.

_robots_cache: dict[str, tuple[RobotFileParser | None, str]] = {}

def _load_robots(base: str) -> tuple[RobotFileParser | None, str]:
    """Fetch and parse robots.txt with our own User-Agent."""
    if base in _robots_cache:
        return _robots_cache[base]
    try:
        r = requests.get(f"{base}/robots.txt",
                         headers={"User-Agent": USER_AGENT}, timeout=15)
    except Exception as exc:                                     # noqa: BLE001
        result = (None, f"unreachable ({type(exc).__name__}) — proceeding with rate limiting")
    else:
        if r.status_code in (401, 403):
            # RFC 9309: an access-restricted robots.txt means "assume disallowed".
            result = (None, f"HTTP {r.status_code} — access restricted, treated as DISALLOW")
        elif r.status_code >= 400:
            result = (None, f"HTTP {r.status_code} — no robots.txt, no restrictions declared")
        else:
            rp = RobotFileParser()
            rp.parse(r.text.splitlines())
            result = (rp, f"parsed OK (HTTP {r.status_code}, {len(r.text.splitlines())} lines)")
    _robots_cache[base] = result
    return result

def robots_allows(url: str, agent: str = USER_AGENT) -> tuple[bool, str]:
    """Return (allowed, auditable explanation)."""
    parsed = urlparse(url)
    base = f"{parsed.scheme}://{parsed.netloc}"
    rp, detail = _load_robots(base)
    if rp is None:
        # restricted robots.txt -> deny; absent/unreachable -> allow, rate-limited
        allowed = "treated as DISALLOW" not in detail
        return allowed, detail
    for ua in (agent, "*"):
        if not rp.can_fetch(ua, url):
            return False, f"{detail}; disallowed for user-agent '{ua}'"
    delay = rp.crawl_delay(agent) or rp.crawl_delay("*")
    return True, f"{detail}; allowed (declared crawl-delay: {delay or 'none'})"

robots_report = []
for s in SOURCES:
    allowed, why = robots_allows(s.url)
    entry = {"slug": s.slug, "url": s.url, "allowed": allowed, "detail": why}
    log.info("robots %-9s %-6s %s", s.slug, "ALLOW" if allowed else "DENY", why)

    if not allowed and s.fallback:
        alt_ok, alt_why = robots_allows(s.fallback)
        entry.update({"switched_to": s.fallback, "fallback_allowed": alt_ok,
                      "fallback_detail": alt_why})
        if alt_ok:
            log.warning("%s -> documented fallback %s (%s)", s.slug, s.fallback, alt_why)
            s.url = s.fallback
            allowed = True
        else:
            log.error("%s: primary AND fallback disallowed — source will be SKIPPED", s.slug)

    s.blocked = not allowed
    entry["final_url"] = s.url
    entry["scraped"] = not s.blocked
    robots_report.append(entry)

print()
print(f"{'SOURCE':<11}{'VERDICT':<9}{'WILL SCRAPE':<13}URL")
print("-" * 118)
for e in robots_report:
    print(f"{e['slug']:<11}{'ALLOW' if e['allowed'] else 'DENY':<9}"
          f"{'yes' if e['scraped'] else 'NO':<13}{e['final_url']}")

allowed_sources = [s for s in SOURCES if not s.blocked]
assert len(allowed_sources) == 3, (
    "the assignment requires 3 sources; a blocked source must be replaced in the "
    "registry rather than scraped in violation of robots.txt")

save_evidence("01_robots_compliance.json", robots_report)

23:30:54 | INFO    | robots wikipedia ALLOW  parsed OK (HTTP 200, 712 lines); allowed (declared crawl-delay: none)
23:30:55 | INFO    | robots aws       ALLOW  parsed OK (HTTP 200, 314 lines); allowed (declared crawl-delay: none)
23:30:55 | INFO    | robots nvidia    ALLOW  parsed OK (HTTP 200, 180 lines); allowed (declared crawl-delay: none)

SOURCE     VERDICT  WILL SCRAPE  URL
----------------------------------------------------------------------------------------------------------------------
wikipedia  ALLOW    yes          https://en.wikipedia.org/wiki/Retrieval-augmented_generation
aws        ALLOW    yes          https://aws.amazon.com/what-is/retrieval-augmented-generation/
nvidia     ALLOW    yes          https://blogs.nvidia.com/blog/what-is-retrieval-augmented-generation/
23:30:55 | INFO    | evidence written -> 01_robots_compliance.json

PosixPath('~/Documents/DBA/Master AI&ML/Applied Research Topics in Deep Learning- Theory & Practical Applications/Week 8/Case/Antonio_Gonzalez_WebScrapingChallenge/evidence/01_robots_compliance.json')

# ---- Part 1.3: Fetch, clean, extract ----
BOILERPLATE_TAGS = ["script", "style", "noscript", "header", "footer", "nav",
                    "aside", "svg", "form", "iframe", "figure"]
# Wikipedia/vendor page furniture that adds no semantic value to the vector store
NOISE_PATTERNS = [
    r"^\s*(Jump to content|Toggle .*|Retrieved from|Categories:|Hidden categories:).*$",
    r"^\s*(Cookie|Privacy|Contact Us|Sign In|Sign up|Subscribe|Share this|Follow us).*$",
    r"^\s*\[\d+\]\s*$",
]
_noise_re = re.compile("|".join(NOISE_PATTERNS), re.IGNORECASE | re.MULTILINE)

def fetch_html(url: str, retries: int = 3, timeout: int = 25) -> str:
    """GET with identifying UA, exponential backoff and explicit status handling."""
    headers = {"User-Agent": USER_AGENT,
               "Accept": "text/html,application/xhtml+xml",
               "Accept-Language": "en-US,en;q=0.9"}
    last: Exception | None = None
    for attempt in range(1, retries + 1):
        try:
            r = requests.get(url, headers=headers, timeout=timeout)
            r.raise_for_status()
            r.encoding = r.apparent_encoding or r.encoding
            return r.text
        except Exception as exc:                                # noqa: BLE001
            last = exc
            wait = 2 ** attempt
            log.warning("fetch attempt %d/%d failed for %s (%s) — retrying in %ss",
                        attempt, retries, url, type(exc).__name__, wait)
            time.sleep(wait)
    raise RuntimeError(f"could not fetch {url}: {last}")

def extract_text(html: str) -> tuple[str, str]:
    """Strip boilerplate and return (page_title, normalised visible text)."""
    from bs4 import BeautifulSoup
    try:
        soup = BeautifulSoup(html, "lxml")
    except Exception:                                            # lxml missing
        soup = BeautifulSoup(html, "html.parser")
    title = (soup.title.get_text(strip=True) if soup.title else "").strip()
    for tag in soup(BOILERPLATE_TAGS):
        tag.decompose()
    main = soup.find("main") or soup.find("article") or soup.find(id="mw-content-text") or soup.body or soup
    text = main.get_text(separator="\n")
    text = _noise_re.sub("", text)
    text = re.sub(r"[ \t\xa0]+", " ", text)
    lines = [ln.strip() for ln in text.splitlines()]
    lines = [ln for ln in lines if len(ln) > 2]                  # drop orphan characters
    return title, "\n".join(lines)

@dataclass
class Document:
    slug: str; url: str; domain: str; category: str; register: str
    title: str; text: str; n_chars: int; n_words: int
    scraped_at: str; sha1: str

def scrape(sources: Sequence[Source], delay: float = REQUEST_DELAY) -> list[Document]:
    docs: list[Document] = []
    for s in sources:
        try:
            t0 = time.perf_counter()
            html = fetch_html(s.url)
            title, text = extract_text(html)
            if len(text) < 500:                                  # quality gate
                log.warning("thin content for %s (%d chars) — check the selector", s.url, len(text))
            docs.append(Document(
                slug=s.slug, url=s.url, domain=urlparse(s.url).netloc,
                category=s.category, register=s.register, title=title, text=text,
                n_chars=len(text), n_words=len(text.split()),
                scraped_at=datetime.now(timezone.utc).isoformat(),
                sha1=hashlib.sha1(text.encode("utf-8")).hexdigest(),
            ))
            log.info("scraped %-9s %6d chars %5d words in %.2fs — %s",
                     s.slug, len(text), len(text.split()), time.perf_counter() - t0, title[:60])
        except Exception as exc:                                 # noqa: BLE001
            log.error("FAILED %s: %s", s.url, exc)
        time.sleep(delay)                                        # polite rate limiting
    return docs

documents = scrape(allowed_sources)   # robots.txt gate applied in Part 1.2
assert len(documents) == 3, "all three sources must scrape successfully"

print()
print(f"{'SLUG':<10}{'CHARS':>8}{'WORDS':>8}  TITLE")
print("-" * 110)
for d in documents:
    print(f"{d.slug:<10}{d.n_chars:>8}{d.n_words:>8}  {d.title[:70]}")
print("-" * 110)
print(f"{'TOTAL':<10}{sum(d.n_chars for d in documents):>8}{sum(d.n_words for d in documents):>8}")

save_evidence("02_scrape_summary.json",
              [{k: v for k, v in asdict(d).items() if k != "text"} for d in documents])

23:30:55 | INFO    | scraped wikipedia  21024 chars  2938 words in 0.31s — Retrieval-augmented generation - Wikipedia
23:30:57 | INFO    | scraped aws        10781 chars  1574 words in 0.24s — What is RAG? - Retrieval-Augmented Generation AI Explained -
23:30:59 | INFO    | scraped nvidia     10698 chars  1712 words in 0.36s — What Is Retrieval-Augmented Generation aka RAG | NVIDIA Blog

SLUG         CHARS   WORDS  TITLE
--------------------------------------------------------------------------------------------------------------
wikipedia    21024    2938  Retrieval-augmented generation - Wikipedia
aws          10781    1574  What is RAG? - Retrieval-Augmented Generation AI Explained - AWS
nvidia       10698    1712  What Is Retrieval-Augmented Generation aka RAG | NVIDIA Blogs
--------------------------------------------------------------------------------------------------------------
TOTAL        42503    6224
23:31:00 | INFO    | evidence written -> 02_scrape_summary.json

PosixPath('~/Documents/DBA/Master AI&ML/Applied Research Topics in Deep Learning- Theory & Practical Applications/Week 8/Case/Antonio_Gonzalez_WebScrapingChallenge/evidence/02_scrape_summary.json')

# ---- Part 2.1: Chunking strategies ----
_SENT_SPLIT = re.compile(r"(?<=[.!?;:])\s+|\n+")

def chunk_fixed(text: str, size: int = CHUNK_SIZE, overlap: int = CHUNK_OVERLAP) -> list[str]:
    """Baseline: raw character window with overlap (the template's approach)."""
    step = max(1, size - overlap)
    return [text[i:i + size] for i in range(0, len(text), step) if text[i:i + size].strip()]

def chunk_sentence_aware(text: str, size: int = CHUNK_SIZE, overlap: int = CHUNK_OVERLAP,
                         min_chars: int = 120) -> list[str]:
    """Pack whole sentences up to `size`; carry a sentence-aligned tail as overlap."""
    sentences = [s.strip() for s in _SENT_SPLIT.split(text) if s and s.strip()]
    chunks, buf, buf_len = [], [], 0
    for sent in sentences:
        s_len = len(sent) + 1
        if buf and buf_len + s_len > size:
            chunks.append(" ".join(buf))
            tail, tail_len = [], 0                               # rebuild overlap from the tail
            for prev in reversed(buf):
                if tail_len + len(prev) + 1 > overlap:
                    break
                tail.insert(0, prev); tail_len += len(prev) + 1
            buf, buf_len = tail, tail_len
        buf.append(sent); buf_len += s_len
    if buf:
        chunks.append(" ".join(buf))
    return [c for c in chunks if len(c) >= min_chars]            # drop navigational scraps

CHUNKERS = {"fixed_char": chunk_fixed, "sentence_aware": chunk_sentence_aware}

# ---- Part 2.2: Build the chunk corpus with rich metadata (bonus: enhanced metadata) ----

def build_corpus(docs: Sequence[Document], strategy: str = "sentence_aware",
                 size: int = CHUNK_SIZE, overlap: int = CHUNK_OVERLAP) -> list[dict]:
    chunker = CHUNKERS[strategy]
    corpus: list[dict] = []
    for d in docs:
        for i, ch in enumerate(chunker(d.text, size, overlap)):
            corpus.append({
                "id": f"{d.slug}-{strategy}-{size}-{overlap}-{i:04d}",
                "text": ch,
                "metadata": {
                    "url": d.url, "domain": d.domain, "source_slug": d.slug,
                    "title": d.title[:180], "category": d.category, "register": d.register,
                    "chunk_index": i, "chunk_strategy": strategy,
                    "chunk_size": size, "chunk_overlap": overlap,
                    "char_len": len(ch), "word_count": len(ch.split()),
                    "scraped_at": d.scraped_at, "run_id": RUN_ID,
                    "content_sha1": hashlib.sha1(ch.encode("utf-8")).hexdigest()[:16],
                },
            })
    return corpus

corpus = build_corpus(documents)

by_domain: dict[str, list[int]] = {}
for item in corpus:
    by_domain.setdefault(item["metadata"]["domain"], []).append(item["metadata"]["char_len"])

print(f"{'DOMAIN':<24}{'CHUNKS':>8}{'MEAN CHARS':>12}{'MIN':>8}{'MAX':>8}")
print("-" * 62)
for dom, lens in by_domain.items():
    print(f"{dom:<24}{len(lens):>8}{sum(lens)/len(lens):>12.0f}{min(lens):>8}{max(lens):>8}")
print("-" * 62)
print(f"{'TOTAL':<24}{len(corpus):>8}")

assert len(corpus) >= 50, f"rubric requires >= 50 chunks, produced {len(corpus)}"
print(f"\n[OK] Rubric check — chunk count {len(corpus)} >= 50")

save_evidence("03_corpus_stats.json", {
    "total_chunks": len(corpus), "strategy": "sentence_aware",
    "chunk_size": CHUNK_SIZE, "chunk_overlap": CHUNK_OVERLAP,
    "per_domain": {k: {"chunks": len(v), "mean_chars": round(sum(v)/len(v), 1)} for k, v in by_domain.items()},
})

print("\nSample chunk:")
print("-" * 96)
print(corpus[len(corpus)//2]["text"][:600], "...")

DOMAIN                    CHUNKS  MEAN CHARS     MIN     MAX
--------------------------------------------------------------
en.wikipedia.org              26         952     725     999
aws.amazon.com                14         907     297     999
blogs.nvidia.com              14         900     510     985
--------------------------------------------------------------
TOTAL                         54

[OK] Rubric check — chunk count 54 >= 50
23:31:01 | INFO    | evidence written -> 03_corpus_stats.json

Sample chunk:
------------------------------------------------------------------------------------------------
RAG extends the already powerful capabilities of LLMs to specific domains or an organization's internal knowledge base, all without the need to retrain the model. It is a cost-effective approach to improving LLM output so it remains relevant, accurate, and useful in various contexts. Why is Retrieval-Augmented Generation important? LLMs are a key artificial intelligence (AI) technology powering intelligent chatbots and other natural language processing (NLP) applications. The goal is to create bots that can answer user questions in various contexts by cross-referencing authoritative knowledge  ...

Sample chunk:
------------------------------------------------------------------------------------------------
RAG extends the already powerful capabilities of LLMs to specific domains or an organization's internal knowledge base, all without the need to retrain the model. It is a cost-effective approach to improving LLM output so it remains relevant, accurate, and useful in various contexts. Why is Retrieval-Augmented Generation important? LLMs are a key artificial intelligence (AI) technology powering intelligent chatbots and other natural language processing (NLP) applications. The goal is to create bots that can answer user questions in various contexts by cross-referencing authoritative knowledge  ...

# ---- Part 3.1: Embedding backend + ChromaDB helpers ----
import chromadb
from sentence_transformers import SentenceTransformer

def pick_device() -> str:
    try:
        import torch
        if torch.backends.mps.is_available():                    # Apple Silicon
            return "mps"
        if torch.cuda.is_available():
            return "cuda"
    except Exception:                                            # noqa: BLE001
        pass
    return "cpu"

DEVICE = pick_device()
log.info("embedding device: %s", DEVICE)

_model_cache: dict[str, SentenceTransformer] = {}

def get_model(name: str) -> SentenceTransformer:
    if name not in _model_cache:
        t0 = time.perf_counter()
        _model_cache[name] = SentenceTransformer(name, device=DEVICE)
        log.info("loaded %s (dim=%d) in %.1fs", name,
                 embedding_dim(_model_cache[name]), time.perf_counter() - t0)
    return _model_cache[name]

def embedding_dim(model: SentenceTransformer) -> int:
    """Version-tolerant accessor: sentence-transformers renamed
    `get_sentence_embedding_dimension()` to `get_embedding_dimension()`."""
    getter = getattr(model, "get_embedding_dimension", None) or \
             getattr(model, "get_sentence_embedding_dimension")
    return int(getter())

def embed(texts: Sequence[str], model_name: str = EMBEDDING_MODEL,
          batch_size: int = 64, progress: bool = False) -> list[list[float]]:
    model = get_model(model_name)
    vecs = model.encode(list(texts), batch_size=batch_size, convert_to_numpy=True,
                        normalize_embeddings=(SPACE_TYPE == "cosine"),
                        show_progress_bar=progress)
    return vecs.tolist()

# --- Chroma client -----------------------------------------------------------
chroma = chromadb.HttpClient(host=CHROMA_HOST, port=CHROMA_PORT)
print("ChromaDB heartbeat (ns):", chroma.heartbeat())

def get_collection(name: str, space: str = SPACE_TYPE, reset: bool = False):
    """Version-tolerant collection factory.

    Chroma moved the distance metric from `metadata={'hnsw:space': ...}` to
    `configuration={'hnsw': {'space': ...}}`; both spellings are attempted so the
    notebook runs against either server image. `embedding_function=None` keeps the
    client from instantiating its bundled ONNX model — every vector is supplied
    explicitly by the SentenceTransformer above.
    """
    if reset:
        try:
            chroma.delete_collection(name)
            log.info("dropped existing collection '%s'", name)
        except Exception:                                        # noqa: BLE001
            pass
    attempts = [
        {"configuration": {"hnsw": {"space": space}}},
        {"metadata": {"hnsw:space": space}},
        {},
    ]
    last: Exception | None = None
    for kw in attempts:
        for with_ef in (True, False):
            try:
                extra = {"embedding_function": None} if with_ef else {}
                return chroma.get_or_create_collection(name=name, **kw, **extra)
            except Exception as exc:                             # noqa: BLE001
                last = exc
    raise RuntimeError(f"could not create collection '{name}': {last}")

def add_corpus(collection, items: Sequence[dict], model_name: str = EMBEDDING_MODEL,
               batch_size: int = 100, progress: bool = False) -> int:
    """Batched upsert of documents + pre-computed embeddings."""
    total = 0
    for i in range(0, len(items), batch_size):
        block = items[i:i + batch_size]
        collection.add(
            ids=[b["id"] for b in block],
            documents=[b["text"] for b in block],
            metadatas=[b["metadata"] for b in block],
            embeddings=embed([b["text"] for b in block], model_name, progress=progress),
        )
        total += len(block)
        log.info("indexed %d/%d chunks", total, len(items))
    return total

23:31:06 | INFO    | embedding device: mps
ChromaDB heartbeat (ns): 1788154266063375929

# ---- Part 3.2: Build the production collection & verify storage ----
collection = get_collection(COLLECTION_NAME, reset=True)   # deterministic, re-runnable

t0 = time.perf_counter()
n_indexed = add_corpus(collection, corpus, EMBEDDING_MODEL, progress=True)
index_seconds = time.perf_counter() - t0

count = collection.count()
probe_vec = collection.get(limit=1, include=["embeddings", "metadatas", "documents"])
vector_dim = len(probe_vec["embeddings"][0])

print()
print("=" * 96)
print("CHROMADB STORAGE VERIFICATION")
print("=" * 96)
print(f"  Collection name        : {COLLECTION_NAME}")
print(f"  Distance space         : {SPACE_TYPE}")
print(f"  Documents indexed      : {n_indexed}")
print(f"  collection.count()     : {count}")
print(f"  Embedding model        : {EMBEDDING_MODEL}")
print(f"  Vector dimensionality  : {vector_dim}")
print(f"  Index build time       : {index_seconds:.1f}s ({n_indexed/index_seconds:.1f} chunks/s on {DEVICE})")

# Show the stored embedding itself, not just its shape. The L2 norm doubles as a
# validation check: cosine space requires unit-length vectors, so a norm of ~1.0
# confirms that normalize_embeddings actually took effect at encode time.
_vec = probe_vec["embeddings"][0]
_norm = sum(float(x) * float(x) for x in _vec) ** 0.5
print(f"  Embedding (first 8/{vector_dim})   : {[round(float(x), 4) for x in _vec[:8]]}")
print(f"  Embedding L2 norm      : {_norm:.4f}  (expected ~1.0 for cosine space)")
print(f"  Server collections     : {[c.name if hasattr(c, 'name') else c for c in chroma.list_collections()]}")

assert count == len(corpus), "indexed count must match the corpus size"
assert vector_dim == embedding_dim(get_model(EMBEDDING_MODEL))

print("\n  Sample stored record")
print("  " + "-" * 92)
md0 = probe_vec["metadatas"][0]
print(f"  id       : {probe_vec['ids'][0]}")
print(f"  source   : {md0['domain']}  |  chunk {md0['chunk_index']}  |  {md0['char_len']} chars")
print(f"  document : {probe_vec['documents'][0][:220]} ...")

save_evidence("04_chroma_status.json", {
    "collection": COLLECTION_NAME, "space": SPACE_TYPE, "count": count,
    "embedding_model": EMBEDDING_MODEL, "vector_dim": vector_dim,
    "index_seconds": round(index_seconds, 2), "device": DEVICE,
    "chunks_per_second": round(n_indexed / index_seconds, 2),
})

23:31:06 | INFO    | dropped existing collection 'web_scraping_collection'

Loading weights:   0%|          | 0/103 [00:00<?, ?it/s]

23:31:09 | INFO    | loaded all-MiniLM-L6-v2 (dim=384) in 3.2s

Batches:   0%|          | 0/1 [00:00<?, ?it/s]

23:31:10 | INFO    | indexed 54/54 chunks

================================================================================================
CHROMADB STORAGE VERIFICATION
================================================================================================
  Collection name        : web_scraping_collection
  Distance space         : cosine
  Documents indexed      : 54
  collection.count()     : 54
  Embedding model        : all-MiniLM-L6-v2
  Vector dimensionality  : 384
  Index build time       : 4.1s (13.3 chunks/s on mps)
  Embedding (first 8/384)   : [-0.0792, -0.0275, -0.0111, 0.014, 0.0157, 0.0175, 0.0075, 0.0194]
  Embedding L2 norm      : 1.0000  (expected ~1.0 for cosine space)
  Server collections     : ['web_scraping_collection']

  Sample stored record
  --------------------------------------------------------------------------------------------
  id       : wikipedia-sentence_aware-1000-200-0000
  source   : en.wikipedia.org  |  chunk 0  |  989 chars
  document : From Wikipedia, the free encyclopedia Type of information retrieval using LLMs Retrieval-augmented generation RAG ) is a technique that enables large language models (LLMs) to retrieve and incorporate new information fro ...
23:31:10 | INFO    | evidence written -> 04_chroma_status.json

PosixPath('~/Documents/DBA/Master AI&ML/Applied Research Topics in Deep Learning- Theory & Practical Applications/Week 8/Case/Antonio_Gonzalez_WebScrapingChallenge/evidence/04_chroma_status.json')

# ---- Part 3B.1: Evaluation query set with gold lexical anchors ----
EVAL_QUERIES: list[dict] = [
    {"q": "How does retrieval-augmented generation reduce hallucinations in large language models?",
     "gold": ["hallucin", "inaccur", "ground", "factual", "false"]},
    {"q": "What is a vector database and how are embeddings stored for search?",
     "gold": ["vector", "embedding", "database", "index"]},
    {"q": "How is a long document split into passages before it is indexed?",
     "gold": ["chunk", "split", "segment", "passage", "document"]},
    {"q": "Why would an organisation connect a foundation model to its own internal data?",
     "gold": ["organization", "organisation", "enterprise", "internal", "propriet", "domain-specific", "own data"]},
    {"q": "What role does semantic similarity play when ranking candidate passages?",
     "gold": ["similar", "semantic", "relevan", "rank", "nearest"]},
    {"q": "How is the retrieved context combined with the user's original prompt?",
     "gold": ["prompt", "context", "augment", "input"]},
    {"q": "What are the cost and effort trade-offs compared with fine-tuning or retraining a model?",
     "gold": ["fine-tun", "retrain", "cost", "expensive", "comput"]},
    {"q": "How can users verify an answer by checking the cited sources?",
     "gold": ["source", "citation", "cite", "attribut", "verif", "reference"]},
]

def is_relevant(text: str, gold: Sequence[str]) -> bool:
    low = text.lower()
    return any(g in low for g in gold)

def evaluate_collection(coll, model_name: str, queries: Sequence[dict], k: int = 5) -> dict:
    """Score a collection on the fixed evaluation set."""
    hits, precisions, rrs, top1_sims, coverages = [], [], [], [], []
    q_vecs = embed([q["q"] for q in queries], model_name)
    for q, qv in zip(queries, q_vecs):
        res = coll.query(query_embeddings=[qv], n_results=k,
                         include=["documents", "metadatas", "distances"])
        docs = res["documents"][0]; metas = res["metadatas"][0]; dists = res["distances"][0]
        flags = [is_relevant(d, q["gold"]) for d in docs]
        hits.append(1.0 if any(flags) else 0.0)
        precisions.append(sum(flags) / max(1, len(flags)))
        rrs.append(1.0 / (flags.index(True) + 1) if any(flags) else 0.0)
        top1_sims.append(1.0 - dists[0] if dists else 0.0)
        coverages.append(len({m["domain"] for m in metas}))
    n = len(queries)
    return {"hit@5": sum(hits)/n, "precision@5": sum(precisions)/n, "mrr": sum(rrs)/n,
            "mean_top1_sim": sum(top1_sims)/n, "domain_coverage@5": sum(coverages)/n}

# ---- Part 3B.2: Run the configuration matrix ----
# Each configuration is materialised in its own temporary Chroma collection, scored,
# then dropped — so the experiment exercises the real vector store, not an in-memory mock.

SIZES     = [500, 1000, 1500] if not FAST_MODE else [500, 1000, 1500]
OVERLAPS  = [100, 200, 300]   if not FAST_MODE else [200]
MODELS    = [EMBEDDING_MODEL, ALT_EMBEDDING_MODEL]
STRATEGY_GRID = ["sentence_aware", "fixed_char"]

experiments: list[dict] = []
total_runs = len(MODELS) * len(SIZES) * len(OVERLAPS)
run_no = 0

for model_name in MODELS:
    get_model(model_name)                                    # warm the model once
    for size in SIZES:
        for overlap in OVERLAPS:
            run_no += 1
            tag = f"exp_{model_name.split('-')[1]}_{size}_{overlap}".lower().replace(".", "")
            exp_corpus = build_corpus(documents, "sentence_aware", size, overlap)
            coll = get_collection(tag, reset=True)
            t0 = time.perf_counter()
            add_corpus(coll, exp_corpus, model_name, batch_size=128)
            build_s = time.perf_counter() - t0
            scores = evaluate_collection(coll, model_name, EVAL_QUERIES)
            row = {"model": model_name, "chunk_size": size, "overlap": overlap,
                   "strategy": "sentence_aware", "chunks": len(exp_corpus),
                   "index_s": round(build_s, 2), **{k: round(v, 4) for k, v in scores.items()}}
            experiments.append(row)
            log.info("[%02d/%02d] %-20s size=%-5d ovl=%-4d chunks=%-4d hit@5=%.2f mrr=%.3f p@5=%.2f",
                     run_no, total_runs, model_name, size, overlap, len(exp_corpus),
                     row["hit@5"], row["mrr"], row["precision@5"])
            try:
                chroma.delete_collection(tag)
            except Exception:                                # noqa: BLE001
                pass

# --- chunking-strategy ablation at the default geometry ----------------------
for strategy in STRATEGY_GRID:
    tag = f"exp_strategy_{strategy}"
    exp_corpus = build_corpus(documents, strategy, CHUNK_SIZE, CHUNK_OVERLAP)
    coll = get_collection(tag, reset=True)
    add_corpus(coll, exp_corpus, EMBEDDING_MODEL, batch_size=128)
    scores = evaluate_collection(coll, EMBEDDING_MODEL, EVAL_QUERIES)
    experiments.append({"model": EMBEDDING_MODEL, "chunk_size": CHUNK_SIZE,
                        "overlap": CHUNK_OVERLAP, "strategy": strategy,
                        "chunks": len(exp_corpus), "index_s": None,
                        **{k: round(v, 4) for k, v in scores.items()}})
    log.info("strategy ablation %-15s chunks=%-4d hit@5=%.2f mrr=%.3f",
             strategy, len(exp_corpus), scores["hit@5"], scores["mrr"])
    try:
        chroma.delete_collection(tag)
    except Exception:                                        # noqa: BLE001
        pass

import pandas as pd
results = pd.DataFrame(experiments)
results.to_csv(EVIDENCE_DIR / "05_experiment_results.csv", index=False)
save_evidence("05_experiment_results.json", experiments)

grid = results[results.strategy == "sentence_aware"].copy()
print("\nCONFIGURATION MATRIX (sorted by MRR, then hit@5)")
print(grid.sort_values(["mrr", "hit@5"], ascending=False).to_string(index=False))

best = grid.sort_values(["mrr", "hit@5", "precision@5"], ascending=False).iloc[0]
print(f"\nBest configuration -> model={best['model']} size={int(best['chunk_size'])} "
      f"overlap={int(best['overlap'])} | mrr={best['mrr']:.3f} hit@5={best['hit@5']:.2f}")

23:31:10 | INFO    | indexed 106/106 chunks
23:31:10 | INFO    | [01/18] all-MiniLM-L6-v2     size=500   ovl=100  chunks=106  hit@5=1.00 mrr=0.729 p@5=0.68
23:31:10 | INFO    | indexed 128/137 chunks
23:31:10 | INFO    | indexed 137/137 chunks
23:31:10 | INFO    | [02/18] all-MiniLM-L6-v2     size=500   ovl=200  chunks=137  hit@5=0.88 mrr=0.775 p@5=0.70
23:31:11 | INFO    | indexed 128/208 chunks
23:31:11 | INFO    | indexed 208/208 chunks
23:31:11 | INFO    | [03/18] all-MiniLM-L6-v2     size=500   ovl=300  chunks=208  hit@5=1.00 mrr=0.900 p@5=0.68
23:31:11 | INFO    | indexed 49/49 chunks
23:31:11 | INFO    | [04/18] all-MiniLM-L6-v2     size=1000  ovl=100  chunks=49   hit@5=1.00 mrr=0.854 p@5=0.70
23:31:11 | INFO    | indexed 54/54 chunks
23:31:11 | INFO    | [05/18] all-MiniLM-L6-v2     size=1000  ovl=200  chunks=54   hit@5=1.00 mrr=0.817 p@5=0.68
23:31:12 | INFO    | indexed 60/60 chunks
23:31:12 | INFO    | [06/18] all-MiniLM-L6-v2     size=1000  ovl=300  chunks=60   hit@5=1.00 mrr=0.906 p@5=0.78
23:31:12 | INFO    | indexed 31/31 chunks
23:31:12 | INFO    | [07/18] all-MiniLM-L6-v2     size=1500  ovl=100  chunks=31   hit@5=1.00 mrr=0.854 p@5=0.72
23:31:12 | INFO    | indexed 34/34 chunks
23:31:12 | INFO    | [08/18] all-MiniLM-L6-v2     size=1500  ovl=200  chunks=34   hit@5=1.00 mrr=0.844 p@5=0.68
23:31:12 | INFO    | indexed 36/36 chunks
23:31:12 | INFO    | [09/18] all-MiniLM-L6-v2     size=1500  ovl=300  chunks=36   hit@5=1.00 mrr=0.823 p@5=0.80

Loading weights:   0%|          | 0/199 [00:00<?, ?it/s]

23:31:15 | INFO    | loaded all-mpnet-base-v2 (dim=768) in 3.0s
23:31:16 | INFO    | indexed 106/106 chunks
23:31:16 | INFO    | [10/18] all-mpnet-base-v2    size=500   ovl=100  chunks=106  hit@5=1.00 mrr=0.906 p@5=0.70
23:31:18 | INFO    | indexed 128/137 chunks
23:31:18 | INFO    | indexed 137/137 chunks
23:31:18 | INFO    | [11/18] all-mpnet-base-v2    size=500   ovl=200  chunks=137  hit@5=0.88 mrr=0.812 p@5=0.75
23:31:19 | INFO    | indexed 128/208 chunks
23:31:20 | INFO    | indexed 208/208 chunks
23:31:20 | INFO    | [12/18] all-mpnet-base-v2    size=500   ovl=300  chunks=208  hit@5=0.88 mrr=0.875 p@5=0.78
23:31:21 | INFO    | indexed 49/49 chunks
23:31:21 | INFO    | [13/18] all-mpnet-base-v2    size=1000  ovl=100  chunks=49   hit@5=1.00 mrr=0.917 p@5=0.80
23:31:22 | INFO    | indexed 54/54 chunks
23:31:22 | INFO    | [14/18] all-mpnet-base-v2    size=1000  ovl=200  chunks=54   hit@5=1.00 mrr=0.823 p@5=0.70
23:31:23 | INFO    | indexed 60/60 chunks
23:31:23 | INFO    | [15/18] all-mpnet-base-v2    size=1000  ovl=300  chunks=60   hit@5=1.00 mrr=0.833 p@5=0.82
23:31:24 | INFO    | indexed 31/31 chunks
23:31:24 | INFO    | [16/18] all-mpnet-base-v2    size=1500  ovl=100  chunks=31   hit@5=1.00 mrr=0.875 p@5=0.85
23:31:25 | INFO    | indexed 34/34 chunks
23:31:25 | INFO    | [17/18] all-mpnet-base-v2    size=1500  ovl=200  chunks=34   hit@5=1.00 mrr=0.875 p@5=0.90
23:31:26 | INFO    | indexed 36/36 chunks
23:31:26 | INFO    | [18/18] all-mpnet-base-v2    size=1500  ovl=300  chunks=36   hit@5=1.00 mrr=0.938 p@5=0.93
23:31:26 | INFO    | indexed 54/54 chunks
23:31:26 | INFO    | strategy ablation sentence_aware  chunks=54   hit@5=1.00 mrr=0.817
23:31:27 | INFO    | indexed 55/55 chunks
23:31:27 | INFO    | strategy ablation fixed_char      chunks=55   hit@5=1.00 mrr=0.844
23:31:27 | INFO    | evidence written -> 05_experiment_results.json

CONFIGURATION MATRIX (sorted by MRR, then hit@5)
            model  chunk_size  overlap       strategy  chunks  index_s  hit@5  precision@5    mrr  mean_top1_sim  domain_coverage@5
all-mpnet-base-v2        1500      300 sentence_aware      36     0.90  1.000        0.925 0.9375         0.4231              2.375
all-mpnet-base-v2        1000      100 sentence_aware      49     1.09  1.000        0.800 0.9167         0.4523              2.375
 all-MiniLM-L6-v2        1000      300 sentence_aware      60     0.21  1.000        0.775 0.9062         0.4513              2.250
all-mpnet-base-v2         500      100 sentence_aware     106     1.10  1.000        0.700 0.9062         0.4808              2.250
 all-MiniLM-L6-v2         500      300 sentence_aware     208     0.44  1.000        0.675 0.9000         0.5305              2.375
all-mpnet-base-v2        1500      100 sentence_aware      31     0.79  1.000        0.850 0.8750         0.4416              2.500
all-mpnet-base-v2        1500      200 sentence_aware      34     0.88  1.000        0.900 0.8750         0.4223              2.250
all-mpnet-base-v2         500      300 sentence_aware     208     1.77  0.875        0.775 0.8750         0.5027              2.250
 all-MiniLM-L6-v2        1000      100 sentence_aware      49     0.15  1.000        0.700 0.8542         0.4397              2.375
 all-MiniLM-L6-v2        1500      100 sentence_aware      31     0.13  1.000        0.725 0.8542         0.4125              2.500
 all-MiniLM-L6-v2        1500      200 sentence_aware      34     0.13  1.000        0.675 0.8438         0.4232              2.250
all-mpnet-base-v2        1000      300 sentence_aware      60     1.27  1.000        0.825 0.8333         0.4535              2.250
 all-MiniLM-L6-v2        1500      300 sentence_aware      36     0.14  1.000        0.800 0.8229         0.4059              2.375
all-mpnet-base-v2        1000      200 sentence_aware      54     1.20  1.000        0.700 0.8229         0.4562              2.250
 all-MiniLM-L6-v2        1000      200 sentence_aware      54     0.16  1.000        0.675 0.8167         0.4349              2.125
 all-MiniLM-L6-v2        1000      200 sentence_aware      54      NaN  1.000        0.675 0.8167         0.4349              2.125
all-mpnet-base-v2         500      200 sentence_aware     137     1.30  0.875        0.750 0.8125         0.4833              2.000
 all-MiniLM-L6-v2         500      200 sentence_aware     137     0.33  0.875        0.700 0.7750         0.4892              2.125
 all-MiniLM-L6-v2         500      100 sentence_aware     106     0.27  1.000        0.675 0.7292         0.4796              2.250

Best configuration -> model=all-mpnet-base-v2 size=1500 overlap=300 | mrr=0.938 hit@5=1.00

# ---- Part 3B.3: Charts ----
import matplotlib as mpl
import matplotlib.pyplot as plt
from matplotlib.colors import LinearSegmentedColormap

# Fixed categorical order (CVD-validated: adjacent-pair dE >= 9.9 protan, >= 24 normal vision)
CAT = ["#2F6FE0", "#D4700A", "#0E8A72"]
INK, INK_MUTED, GRID = "#1c1c1a", "#5f5f5a", "#e3e3df"
SEQ = LinearSegmentedColormap.from_list("seq_blue", ["#eef3fd", "#9dbdf2", "#2F6FE0", "#123E7F"])

mpl.rcParams.update({
    "figure.dpi": 120, "savefig.dpi": 200, "font.size": 10,
    "axes.edgecolor": GRID, "axes.labelcolor": INK_MUTED, "axes.titlecolor": INK,
    "axes.titlesize": 11.5, "axes.titleweight": "600", "axes.titlelocation": "left",
    "axes.spines.top": False, "axes.spines.right": False,
    "xtick.color": INK_MUTED, "ytick.color": INK_MUTED,
    "grid.color": GRID, "grid.linewidth": 0.8,
    "figure.facecolor": "#fcfcfb", "axes.facecolor": "#fcfcfb",
    "legend.frameon": False,
})

def _finish(ax, ylabel=None):
    ax.set_axisbelow(True)
    ax.yaxis.grid(True); ax.xaxis.grid(False)
    if ylabel: ax.set_ylabel(ylabel)

# --- Chart 1: retrieval quality by chunk size, one facet per embedding model -----
fig, axes = plt.subplots(1, len(MODELS), figsize=(11.5, 4.2), sharey=True)
axes = axes if len(MODELS) > 1 else [axes]
width = 0.26
for ax, model_name in zip(axes, MODELS):
    sub = grid[grid.model == model_name]
    xs = range(len(SIZES))
    for j, ovl in enumerate(OVERLAPS):
        vals = [float(sub[(sub.chunk_size == s) & (sub.overlap == ovl)]["mrr"].iloc[0]) for s in SIZES]
        pos = [x + (j - (len(OVERLAPS) - 1) / 2) * width for x in xs]
        bars = ax.bar(pos, vals, width * 0.92, color=CAT[j], label=f"overlap {ovl}",
                      edgecolor="#fcfcfb", linewidth=2)          # 2px surface gap
        for b, v in zip(bars, vals):                              # selective direct labels
            ax.text(b.get_x() + b.get_width() / 2, v + 0.015, f"{v:.2f}",
                    ha="center", va="bottom", fontsize=8, color=INK_MUTED)
    ax.set_xticks(list(xs)); ax.set_xticklabels([f"{s} chars" for s in SIZES])
    ax.set_title(model_name)
    ax.set_ylim(0, 1.12)
    _finish(ax)
axes[0].set_ylabel("Mean reciprocal rank (higher is better)")
handles, labels = axes[0].get_legend_handles_labels()
fig.legend(handles, labels, loc="upper right", bbox_to_anchor=(0.995, 0.995), ncols=3)
fig.suptitle("Retrieval quality by chunk geometry and embedding model",
             x=0.008, y=0.985, ha="left", fontsize=13, fontweight="600", color=INK)
fig.tight_layout(rect=(0, 0, 1, 0.90))
fig.savefig(EVIDENCE_DIR / "chart_1_mrr_by_config.png", bbox_inches="tight")
plt.show()

# --- Chart 2: quality vs. index cost (efficiency frontier) ----------------------
fig, ax = plt.subplots(figsize=(8.6, 5.0))
_lo, _hi = float(grid["hit@5"].min()), float(grid["hit@5"].max())
_pad = max(0.06, (_hi - _lo) * 0.35)
for j, model_name in enumerate(MODELS):
    sub = grid[(grid.model == model_name) & grid.index_s.notna()].sort_values("index_s")
    ax.scatter(sub.index_s, sub["hit@5"], s=95, color=CAT[j], label=model_name,
               edgecolor="#fcfcfb", linewidth=2, zorder=3)        # 2px surface ring
    for i, (_, r) in enumerate(sub.iterrows()):                   # stagger to avoid collisions
        dy = 11 if (i + j) % 2 == 0 else -15
        ax.annotate(f"{int(r.chunk_size)}/{int(r.overlap)}", (r.index_s, r["hit@5"]),
                    textcoords="offset points", xytext=(0, dy), ha="center",
                    fontsize=7.5, color=INK_MUTED, zorder=4)
ax.set_xlabel("Index build time (s) — lower is cheaper")
ax.set_title("Quality vs. indexing cost   ·   labels = chunk size / overlap")
ax.set_ylim(max(0.0, _lo - _pad), min(1.06, _hi + _pad))
_finish(ax, "hit@5 (higher is better)")
ax.legend(loc="lower right")
fig.tight_layout()
fig.savefig(EVIDENCE_DIR / "chart_2_quality_vs_cost.png", bbox_inches="tight")
plt.show()

# --- Chart 3: precision@5 heatmap (sequential, single hue) ----------------------
fig, axes = plt.subplots(1, len(MODELS), figsize=(10.5, 3.6))
axes = axes if len(MODELS) > 1 else [axes]
vmin = float(grid["precision@5"].min()); vmax = float(grid["precision@5"].max())
for ax, model_name in zip(axes, MODELS):
    sub = grid[grid.model == model_name]
    m = [[float(sub[(sub.chunk_size == s) & (sub.overlap == o)]["precision@5"].iloc[0])
          for s in SIZES] for o in OVERLAPS]
    im = ax.imshow(m, cmap=SEQ, vmin=vmin, vmax=vmax, aspect="auto")
    ax.set_xticks(range(len(SIZES)), [str(s) for s in SIZES])
    ax.set_yticks(range(len(OVERLAPS)), [str(o) for o in OVERLAPS])
    ax.set_xlabel("chunk size (chars)"); ax.set_ylabel("overlap (chars)")
    ax.set_title(model_name)
    ax.grid(False)
    for r in range(len(OVERLAPS)):
        for c in range(len(SIZES)):
            shade = "#ffffff" if m[r][c] > (vmin + vmax) / 2 else INK
            ax.text(c, r, f"{m[r][c]:.2f}", ha="center", va="center", fontsize=9, color=shade)
fig.colorbar(im, ax=axes, shrink=0.86, label="precision@5")
fig.suptitle("Context-window efficiency (precision@5)", x=0.008, y=1.10, ha="left",
             fontsize=13, fontweight="600", color=INK)
fig.savefig(EVIDENCE_DIR / "chart_3_precision_heatmap.png", bbox_inches="tight")
plt.show()

print("Charts saved to evidence/: chart_1_mrr_by_config.png, chart_2_quality_vs_cost.png, chart_3_precision_heatmap.png")

23:31:27 | WARNING | findfont: Failed to find font weight 600, now using 700.
23:31:27 | WARNING | findfont: Failed to find font weight 600, now using 700.

Charts saved to evidence/: chart_1_mrr_by_config.png, chart_2_quality_vs_cost.png, chart_3_precision_heatmap.png

# ---- Part 4.1: Semantic search over the production collection ----

def semantic_search(query: str, k: int = 5, model_name: str = EMBEDDING_MODEL,
                    where: dict | None = None) -> list[dict]:
    res = collection.query(query_embeddings=embed([query], model_name), n_results=k,
                           where=where, include=["documents", "metadatas", "distances"])
    return [{"rank": i + 1, "similarity": round(1.0 - d, 4), "distance": round(d, 4),
             "domain": m["domain"], "url": m["url"], "chunk_index": m["chunk_index"],
             "text": doc}
            for i, (doc, m, d) in enumerate(zip(res["documents"][0], res["metadatas"][0], res["distances"][0]))]

def show(query: str, k: int = 5, note: str = "") -> list[dict]:
    hits = semantic_search(query, k)
    print("=" * 118)
    print(f"QUERY: {query}")
    if note:
        print(f"DESIGN: {note}")
    print("=" * 118)
    for h in hits:
        print(f"  #{h['rank']}  sim={h['similarity']:.4f}  [{h['domain']}]  chunk {h['chunk_index']}")
        print(f"      {h['text'][:300].strip()} ...")
    print(f"  -> distinct sources in top-{k}: {len({h['domain'] for h in hits})}")
    print()
    return hits

SEARCH_SET = [
    ("How does grounding a chatbot in external documents stop it from making things up?",
     "Paraphrase test — the corpus says 'hallucination'; the query says 'making things up'."),
    ("What infrastructure is needed to turn a company's internal documents into a searchable knowledge base?",
     "Architecture intent — should favour the vendor/implementation sources over the encyclopedic one."),
    ("Is it cheaper to retrain a model on new data or to look the data up at answer time?",
     "Comparative trade-off — tests reasoning-style phrasing with no single anchor term."),
    ("How does the system decide which passages are the most relevant to a question?",
     "Mechanism question — targets the similarity/ranking portion of the corpus."),
    ("Why can a user trust the answer and check where it came from?",
     "Trust and attribution — tests the citation/verification sub-topic."),
]

search_evidence = []
for q, note in SEARCH_SET:
    hits = show(q, k=5, note=note)
    search_evidence.append({"query": q, "design_note": note,
                            "results": [{k: v for k, v in h.items() if k != "text"} | {"snippet": h["text"][:400]}
                                        for h in hits]})

save_evidence("06_semantic_searches.json", search_evidence)

# Human-readable transcript for the written report / screenshot substitute
lines = [f"# Semantic search transcript — run {RUN_ID}", ""]
for e in search_evidence:
    lines += [f"## {e['query']}", f"*{e['design_note']}*", ""]
    for r in e["results"]:
        lines += [f"- **#{r['rank']}** sim `{r['similarity']}` — `{r['domain']}` (chunk {r['chunk_index']})",
                  f"  > {r['snippet'][:260]} ..."]
    lines.append("")
(EVIDENCE_DIR / "06_semantic_searches.md").write_text("\n".join(lines), encoding="utf-8")
print("Transcript written -> evidence/06_semantic_searches.md")

======================================================================================================================
QUERY: How does grounding a chatbot in external documents stop it from making things up?
DESIGN: Paraphrase test — the corpus says 'hallucination'; the query says 'making things up'.
======================================================================================================================
  #1  sim=0.4035  [en.wikipedia.org]  chunk 0
      From Wikipedia, the free encyclopedia Type of information retrieval using LLMs Retrieval-augmented generation RAG ) is a technique that enables large language models (LLMs) to retrieve and incorporate new information from external data sources. With RAG, LLMs first refer to a specified set of docume ...
  #2  sim=0.3733  [aws.amazon.com]  chunk 2
      Known challenges of LLMs include: Presenting false information when it does not have the answer. Presenting out-of-date or generic information when the user expects a specific, current response. Creating a response from non-authoritative sources. Creating inaccurate responses due to terminology conf ...
  #3  sim=0.3608  [aws.amazon.com]  chunk 5
      This can increase trust and confidence in your generative AI solution. More developer control With RAG, developers can test and improve their chat applications more efficiently. They can control and change the LLM's information sources to adapt to changing requirements or cross-functional usage. Dev ...
  #4  sim=0.3516  [aws.amazon.com]  chunk 3
      RAG is one approach to solving some of these challenges. It redirects the LLM to retrieve relevant information from authoritative, pre-determined knowledge sources. Organizations have greater control over the generated text output, and users gain insights into how the LLM generates the response. Wha ...
  #5  sim=0.3478  [en.wikipedia.org]  chunk 1
      information retrieval before generating responses. Unlike LLMs that rely on static training data, RAG pulls relevant text from databases, uploaded documents, or web sources. According to Ars Technica , "RAG is a way of improving LLM performance, in essence by blending the LLM process with a web sear ...
  -> distinct sources in top-5: 2

======================================================================================================================
QUERY: What infrastructure is needed to turn a company's internal documents into a searchable knowledge base?
DESIGN: Architecture intent — should favour the vendor/implementation sources over the encyclopedic one.
======================================================================================================================
  #1  sim=0.5780  [aws.amazon.com]  chunk 9
      Semantic search enhances RAG results for organizations wanting to add vast external knowledge sources to their LLM applications. Modern enterprises store vast amounts of information like manuals, FAQs, research reports, customer service guides, and human resource document repositories across various ...
  #2  sim=0.5169  [aws.amazon.com]  chunk 11
      With knowledge bases for Amazon Bedrock, you can connect FMs to your data sources for RAG in just a few clicks. Vector conversions, retrievals, and improved output generation are all handled automatically. For organizations managing their own RAG, Amazon Kendra is a highly-accurate enterprise search ...
  #3  sim=0.4759  [aws.amazon.com]  chunk 10
      Developers can then use that answer to provide more context to the LLM. Conventional or keyword search solutions in RAG produce limited results for knowledge-intensive tasks. Developers must also deal with word embeddings, document chunking, and other complexities as they manually prepare their data ...
  #4  sim=0.4709  [aws.amazon.com]  chunk 6
      With RAG, an information retrieval component is introduced that utilizes the user input to first pull information from a new data source. The user query and the relevant information are both given to the LLM. The LLM uses the new knowledge and its training data to create better responses. The follow ...
  #5  sim=0.4667  [en.wikipedia.org]  chunk 0
      From Wikipedia, the free encyclopedia Type of information retrieval using LLMs Retrieval-augmented generation RAG ) is a technique that enables large language models (LLMs) to retrieve and incorporate new information from external data sources. With RAG, LLMs first refer to a specified set of docume ...
  -> distinct sources in top-5: 2

======================================================================================================================
QUERY: Is it cheaper to retrain a model on new data or to look the data up at answer time?
DESIGN: Comparative trade-off — tests reasoning-style phrasing with no single anchor term.
======================================================================================================================
  #1  sim=0.3942  [aws.amazon.com]  chunk 6
      With RAG, an information retrieval component is introduced that utilizes the user input to first pull information from a new data source. The user query and the relevant information are both given to the LLM. The LLM uses the new knowledge and its training data to create better responses. The follow ...
  #2  sim=0.3795  [aws.amazon.com]  chunk 8
      Next, the RAG model augments the user input (or prompts) by adding the relevant retrieved data in context. This step uses prompt engineering techniques to communicate effectively with the LLM. The augmented prompt allows the large language models to generate an accurate answer to user queries. Updat ...
  #3  sim=0.3730  [en.wikipedia.org]  chunk 4
      with prompt stuffing, additional relevant context is added to this input to guide the model's response. This approach provides the LLM with key information early in the prompt, encouraging it to prioritize the supplied data over pre-existing training knowledge. Process edit Retrieval-augmented gener ...
  #4  sim=0.3571  [blogs.nvidia.com]  chunk 4
      Another great advantage of RAG is it’s relatively easy. A blog by Lewis and three of the paper’s coauthors said developers can implement the process with as few as five lines of code That makes the method faster and less expensive than retraining a model with additional datasets. And it lets users h ...
  #5  sim=0.3326  [en.wikipedia.org]  chunk 9
      perplexity , and minimizing KL divergence between the retriever's selections and the model's likelihoods to refine retrieval. Reranking techniques can refine retriever performance by prioritizing the most relevant retrieved documents during training. Language model edit Retro language model for RAG. ...
  -> distinct sources in top-5: 3

======================================================================================================================
QUERY: How does the system decide which passages are the most relevant to a question?
DESIGN: Mechanism question — targets the similarity/ranking portion of the corpus.
======================================================================================================================
  #1  sim=0.3535  [blogs.nvidia.com]  chunk 11
      How RAG Works At a high level, here’s how RAG works When users ask an LLM a question, the AI model sends the query to another model that converts it into a numeric format so machines can read it. The numeric version of the query is sometimes called an embedding or a vector. The embedding model then ...
  #2  sim=0.3414  [en.wikipedia.org]  chunk 3
      For example, LLMs can generate misinformation even when pulling from factually correct sources if they misinterpret the context. MIT Technology Review gives the example of an AI-generated response stating, "The United States has had one Muslim president, Barack Hussein Obama." The model retrieved th ...
  #3  sim=0.3312  [en.wikipedia.org]  chunk 12
      Additionally, LLMs may struggle to recognize when they lack sufficient information to provide a reliable response. Without specific training, models may generate answers even when they should indicate uncertainty. According to IBM , this issue can arise when the model lacks the ability to assess its ...
  #4  sim=0.3249  [en.wikipedia.org]  chunk 15
      Transactions of the Association for Computational Linguistics 329– 345. arXiv 2005.00181 doi 10.1162/tacl_a_00369 . Retrieved 15 March 2025 "Information retrieval" Microsoft . 10 January 2025 . Retrieved 15 March 2025 Khattab, Omar; Zaharia, Matei (2020). "ColBERT: Efficient and Effective Passage Se ...
  #5  sim=0.3207  [blogs.nvidia.com]  chunk 0
      Editor’s note: This article, originally published on Nov. 15, 2023, has been updated. To understand the latest advancements in generative AI , imagine a courtroom. Judges hear and decide cases based on their general understanding of the law. Sometimes a case — like a malpractice suit or a labor disp ...
  -> distinct sources in top-5: 2

======================================================================================================================
QUERY: Why can a user trust the answer and check where it came from?
DESIGN: Trust and attribution — tests the citation/verification sub-topic.
======================================================================================================================
  #1  sim=0.3765  [blogs.nvidia.com]  chunk 3
      Lewis and colleagues developed retrieval-augmented generation to link generative AI services to external resources, especially ones rich in the latest technical details. The paper, with coauthors from the former Facebook AI Research (now Meta AI), University College London and New York University, c ...
  #2  sim=0.3355  [en.wikipedia.org]  chunk 3
      For example, LLMs can generate misinformation even when pulling from factually correct sources if they misinterpret the context. MIT Technology Review gives the example of an AI-generated response stating, "The United States has had one Muslim president, Barack Hussein Obama." The model retrieved th ...
  #3  sim=0.3159  [en.wikipedia.org]  chunk 12
      Additionally, LLMs may struggle to recognize when they lack sufficient information to provide a reliable response. Without specific training, models may generate answers even when they should indicate uncertainty. According to IBM , this issue can arise when the model lacks the ability to assess its ...
  #4  sim=0.2864  [aws.amazon.com]  chunk 5
      This can increase trust and confidence in your generative AI solution. More developer control With RAG, developers can test and improve their chat applications more efficiently. They can control and change the LLM's information sources to adapt to changing requirements or cross-functional usage. Dev ...
  #5  sim=0.2781  [en.wikipedia.org]  chunk 1
      information retrieval before generating responses. Unlike LLMs that rely on static training data, RAG pulls relevant text from databases, uploaded documents, or web sources. According to Ars Technica , "RAG is a way of improving LLM performance, in essence by blending the LLM process with a web sear ...
  -> distinct sources in top-5: 3

23:31:27 | INFO    | evidence written -> 06_semantic_searches.json
Transcript written -> evidence/06_semantic_searches.md
Transcript written -> evidence/06_semantic_searches.md

# ---- Part 4.2: Semantic vs. lexical baseline (does the vector store earn its keep?) ----
# A deliberately simple keyword retriever over the same corpus, so the comparison
# isolates the retrieval METHOD rather than the data.

STOP = {"the","a","an","is","are","was","were","of","to","in","on","for","and","or","it","its",
        "how","what","why","does","do","can","with","that","this","from","by","at","as","be",
        "when","which","who","into","over","most","more","than","so","if","not","you","your",
        "they","them","their","there","have","has","been","would","could","should","about"}

def keyword_search(query: str, items: Sequence[dict], k: int = 5) -> list[dict]:
    terms = [t for t in re.findall(r"[a-z]+", query.lower()) if t not in STOP and len(t) > 2]
    scored = []
    for it in items:
        low = it["text"].lower()
        score = sum(low.count(t) for t in terms)
        if score:
            scored.append((score, it))
    scored.sort(key=lambda x: -x[0])
    return [{"rank": i + 1, "score": s, "domain": it["metadata"]["domain"], "text": it["text"]}
            for i, (s, it) in enumerate(scored[:k])]

def _rr(hits: Sequence[dict], gold: Sequence[str]) -> float:
    """Reciprocal rank of the first relevant hit (0.0 if none)."""
    for i, h in enumerate(hits):
        if is_relevant(h["text"], gold):
            return 1.0 / (i + 1)
    return 0.0

def compare(queries: Sequence[dict], label: str, k: int = 5) -> dict:
    """Score both retrievers on the same query set."""
    s_hit = k_hit = 0
    s_rr = k_rr = 0.0
    rows = []
    for q in queries:
        sem = semantic_search(q["q"], k=k)
        kw  = keyword_search(q["q"], corpus, k=k)
        srr, krr = _rr(sem, q["gold"]), _rr(kw, q["gold"])
        s_hit += srr > 0; k_hit += krr > 0
        s_rr += srr; k_rr += krr
        rows.append({"query": q["q"], "semantic_rr": round(srr, 3), "keyword_rr": round(krr, 3),
                     "keyword_candidates": len(kw)})
        print(f"  {q['q'][:64]:<66}{srr:>8.2f}{krr:>10.2f}{len(kw):>8}")
    n = len(queries)
    summary = {"set": label, "n": n,
               "semantic_hit_rate": round(s_hit / n, 3), "keyword_hit_rate": round(k_hit / n, 3),
               "semantic_mrr": round(s_rr / n, 3), "keyword_mrr": round(k_rr / n, 3),
               "per_query": rows}
    print(f"  {'-'*92}")
    print(f"  {label:<66}{s_rr/n:>8.2f}{k_rr/n:>10.2f}   <- MRR")
    print()
    return summary

# Probe set B: information needs phrased so that they share almost NO content words
# with the corpus. This is where an embedding index is supposed to beat string matching,
# and it is the blind spot of the lexical relevance proxy used in Part 3B.
PARAPHRASE_PROBES = [
    {"q": "how can a machine avoid inventing things it was never taught",
     "gold": ["hallucin", "inaccur", "false", "ground", "made up", "misinform"]},
    {"q": "keeping an assistant current without teaching it all over again",
     "gold": ["retrain", "fine-tun", "up-to-date", "up to date", "current", "update"]},
    {"q": "letting a reader trace a statement back to where it came from",
     "gold": ["source", "citation", "cite", "attribut", "reference", "footnote"]},
    {"q": "turning words into coordinates so that close ideas sit near each other",
     "gold": ["vector", "embedding", "numeric", "mathematic", "similar"]},
]

print(f"  {'QUERY':<66}{'SEM RR':>8}{'KW RR':>10}{'KW cand':>8}")
print("=" * 96)
print("SET A — literal information needs (anchor vocabulary present in the corpus)")
set_a = compare(EVAL_QUERIES, "SET A — literal (n=8)")
print("SET B — paraphrase probes (query vocabulary deliberately absent from the corpus)")
set_b = compare(PARAPHRASE_PROBES, "SET B — paraphrase (n=4)")

print("=" * 96)
print(f"{'':<28}{'semantic':>12}{'keyword':>12}{'delta':>10}")
for s in (set_a, set_b):
    d = s["semantic_mrr"] - s["keyword_mrr"]
    print(f"{s['set']:<28}{s['semantic_mrr']:>12.3f}{s['keyword_mrr']:>12.3f}{d:>+10.3f}")
print("=" * 96)

save_evidence("07_semantic_vs_keyword.json", {"literal": set_a, "paraphrase": set_b})

  QUERY                                                               SEM RR     KW RR KW cand
================================================================================================
SET A — literal information needs (anchor vocabulary present in the corpus)
  How does retrieval-augmented generation reduce hallucinations in      0.33      0.25       5
  What is a vector database and how are embeddings stored for sear      1.00      1.00       5
  How is a long document split into passages before it is indexed?      1.00      1.00       5
  Why would an organisation connect a foundation model to its own       1.00      0.50       5
  What role does semantic similarity play when ranking candidate p      0.20      1.00       5
  How is the retrieved context combined with the user's original p      1.00      1.00       5
  What are the cost and effort trade-offs compared with fine-tunin      1.00      1.00       5
  How can users verify an answer by checking the cited sources?         1.00      1.00       5
  --------------------------------------------------------------------------------------------
  SET A — literal (n=8)                                                 0.82      0.84   <- MRR

SET B — paraphrase probes (query vocabulary deliberately absent from the corpus)
  how can a machine avoid inventing things it was never taught          1.00      1.00       5
  keeping an assistant current without teaching it all over again       1.00      1.00       5
  letting a reader trace a statement back to where it came from         1.00      1.00       5
  turning words into coordinates so that close ideas sit near each      1.00      1.00       5
  --------------------------------------------------------------------------------------------
  SET B — paraphrase (n=4)                                              1.00      1.00   <- MRR

================================================================================================
                                semantic     keyword     delta
SET A — literal (n=8)              0.817       0.844    -0.027
SET B — paraphrase (n=4)           1.000       1.000    +0.000
================================================================================================
23:31:28 | INFO    | evidence written -> 07_semantic_vs_keyword.json

PosixPath('~/Documents/DBA/Master AI&ML/Applied Research Topics in Deep Learning- Theory & Practical Applications/Week 8/Case/Antonio_Gonzalez_WebScrapingChallenge/evidence/07_semantic_vs_keyword.json')

# ---- Part 4.3: Retrieval-set disagreement (diagnosing the tie in Part 4.2) ----

def _sem_ids(q: str, k: int = 5) -> list[str]:
    return collection.query(query_embeddings=embed([q]), n_results=k)["ids"][0]

def _kw_ids(q: str, k: int = 5) -> list[str]:
    terms = [t for t in re.findall(r"[a-z]+", q.lower()) if t not in STOP and len(t) > 2]
    scored = [(sum(it["text"].lower().count(t) for t in terms), it["id"]) for it in corpus]
    scored = sorted([s for s in scored if s[0] > 0], key=lambda x: -x[0])
    return [i for _, i in scored[:k]]

def disagreement(queries: Sequence[dict], label: str, k: int = 5) -> dict:
    jac, same_top1, rows = [], 0, []
    for q in queries:
        a, b = _sem_ids(q["q"], k), _kw_ids(q["q"], k)
        sa, sb = set(a), set(b)
        j = len(sa & sb) / len(sa | sb) if (sa | sb) else 0.0
        t1 = bool(a and b and a[0] == b[0])
        jac.append(j); same_top1 += t1
        rows.append({"query": q["q"], "jaccard@5": round(j, 3), "same_top1": t1,
                     "shared": len(sa & sb)})
        print(f"  {q['q'][:60]:<62}{len(sa & sb):>4}/5{j:>9.2f}{'  yes' if t1 else '   no':>7}")
    n = len(queries)
    print(f"  {'-'*88}")
    print(f"  {label:<62}{'':>4}   {sum(jac)/n:>7.2f}{same_top1:>5}/{n} agree on rank 1\n")
    return {"set": label, "n": n, "mean_jaccard@5": round(sum(jac)/n, 3),
            "top1_agreement": round(same_top1/n, 3), "per_query": rows}

print(f"  {'QUERY':<62}{'SHARED':>8}{'JACCARD':>9}{'TOP-1':>7}")
print("=" * 92)
dis_a = disagreement(EVAL_QUERIES, "SET A — literal (n=8)")
dis_b = disagreement(PARAPHRASE_PROBES, "SET B — paraphrase (n=4)")

overall = (dis_a["mean_jaccard@5"] * dis_a["n"] + dis_b["mean_jaccard@5"] * dis_b["n"]) / (dis_a["n"] + dis_b["n"])
print("=" * 92)
print("INTERPRETATION")
print("-" * 92)
print(f"  Mean top-5 overlap between the two retrievers : {overall:.2f}")
print(f"  Queries where both rank the SAME chunk first  : "
      f"{int(dis_a['top1_agreement']*dis_a['n'] + dis_b['top1_agreement']*dis_b['n'])}/{dis_a['n']+dis_b['n']}")
print()
if overall < 0.5:
    print("  The retrievers return substantially DIFFERENT passages, yet the lexical")
    print("  relevance proxy scores them equally. The tie in Part 4.2 is therefore a")
    print("  property of the METRIC, not of the methods: on a 54-chunk corpus where")
    print("  every document explains the same concept, 'contains a gold anchor' is")
    print("  satisfied by almost any ranking. Separating the two would require either")
    print("  human relevance judgements or a larger, topically heterogeneous corpus.")
else:
    print("  The retrievers largely agree on which passages are relevant, which is the")
    print("  expected behaviour on a small, single-topic corpus: with 54 chunks the")
    print("  candidate pool is too small for the two ranking functions to diverge.")

save_evidence("10_retriever_disagreement.json",
              {"literal": dis_a, "paraphrase": dis_b, "mean_jaccard_overall": round(overall, 3)})

  QUERY                                                           SHARED  JACCARD  TOP-1
============================================================================================
  How does retrieval-augmented generation reduce hallucination     3/5     0.43     no
  What is a vector database and how are embeddings stored for      3/5     0.43     no
  How is a long document split into passages before it is inde     1/5     0.11     no
  Why would an organisation connect a foundation model to its      0/5     0.00     no
  What role does semantic similarity play when ranking candida     1/5     0.11     no
  How is the retrieved context combined with the user's origin     4/5     0.67    yes
  What are the cost and effort trade-offs compared with fine-t     2/5     0.25    yes
  How can users verify an answer by checking the cited sources     3/5     0.43    yes
  ----------------------------------------------------------------------------------------
  SET A — literal (n=8)                                                   0.30    3/8 agree on rank 1

  how can a machine avoid inventing things it was never taught     0/5     0.00     no
  keeping an assistant current without teaching it all over ag     1/5     0.11     no
  letting a reader trace a statement back to where it came fro     1/5     0.11     no
  turning words into coordinates so that close ideas sit near      1/5     0.11     no
  ----------------------------------------------------------------------------------------
  SET B — paraphrase (n=4)                                                0.08    0/4 agree on rank 1

============================================================================================
INTERPRETATION
--------------------------------------------------------------------------------------------
  Mean top-5 overlap between the two retrievers : 0.23
  Queries where both rank the SAME chunk first  : 3/12

  The retrievers return substantially DIFFERENT passages, yet the lexical
  relevance proxy scores them equally. The tie in Part 4.2 is therefore a
  property of the METRIC, not of the methods: on a 54-chunk corpus where
  every document explains the same concept, 'contains a gold anchor' is
  satisfied by almost any ranking. Separating the two would require either
  human relevance judgements or a larger, topically heterogeneous corpus.
23:31:28 | INFO    | evidence written -> 10_retriever_disagreement.json

PosixPath('~/Documents/DBA/Master AI&ML/Applied Research Topics in Deep Learning- Theory & Practical Applications/Week 8/Case/Antonio_Gonzalez_WebScrapingChallenge/evidence/10_retriever_disagreement.json')

# ---- Part 4C.1: Grounded generation ----

def ollama_available() -> tuple[bool, list[str]]:
    try:
        r = requests.get(f"{OLLAMA_BASE_URL}/api/tags", timeout=6)
        r.raise_for_status()
        return True, [m["name"] for m in r.json().get("models", [])]
    except Exception as exc:                                     # noqa: BLE001
        log.warning("Ollama unavailable (%s) — skipping generation", type(exc).__name__)
        return False, []

def ollama_generate(prompt: str, model: str = OLLAMA_MODEL, timeout: int = 180) -> str:
    r = requests.post(f"{OLLAMA_BASE_URL}/api/generate",
                      json={"model": model, "prompt": prompt, "stream": False,
                            "options": {"temperature": 0.1, "num_ctx": 4096}},
                      timeout=timeout)
    r.raise_for_status()
    return r.json().get("response", "").strip()

RAG_PROMPT = """You are a technical assistant. Answer the question using ONLY the context below.
If the context does not contain the answer, say exactly: "The indexed sources do not cover this."
Cite the source URL in square brackets after each claim.

CONTEXT
{context}

QUESTION: {question}

ANSWER:"""

def rag_answer(question: str, k: int = 4) -> dict:
    hits = semantic_search(question, k=k)
    context = "\n\n".join(f"[{h['url']}]\n{h['text']}" for h in hits)
    answer = ollama_generate(RAG_PROMPT.format(context=context, question=question))
    return {"question": question, "answer": answer,
            "sources": sorted({h["url"] for h in hits}),
            "top_similarity": hits[0]["similarity"] if hits else None}

available, models = ollama_available()
rag_evidence = []
if available:
    print("Ollama models present:", models or "(none pulled)")
    have = any(OLLAMA_MODEL.split(":")[0] in m for m in models)
    if not have:
        log.warning("model '%s' not pulled — run: docker exec -it ollama ollama pull %s",
                    OLLAMA_MODEL, OLLAMA_MODEL)
    else:
        for question in [
            "According to the indexed sources, what problem does retrieval-augmented generation solve?",
            "What are the main components of a RAG pipeline, end to end?",
            "What is the population of Saltillo?",   # negative control: not in the corpus
        ]:
            t0 = time.perf_counter()
            out = rag_answer(question)
            out["latency_s"] = round(time.perf_counter() - t0, 2)
            rag_evidence.append(out)
            print("=" * 112)
            print("Q:", question)
            print("-" * 112)
            print(out["answer"])
            print(f"\nsources: {out['sources']}   top_sim={out['top_similarity']}   {out['latency_s']}s")
            print()
else:
    print("Ollama not reachable — generation skipped (retrieval results above remain valid).")

# OpenWebUI reachability evidence (the chat UI shares the same Ollama backend)
webui = probe("OpenWebUI", WEBUI_URL)
print(f"OpenWebUI: {'reachable' if webui['ok'] else 'unreachable'} at {WEBUI_URL} "
      f"({webui['latency_ms']} ms)")

save_evidence("08_rag_generation.json",
              {"ollama_available": available, "models": models,
               "openwebui": webui, "answers": rag_evidence})

Ollama models present: ['llama3.2:3b']
================================================================================================================
Q: According to the indexed sources, what problem does retrieval-augmented generation solve?
----------------------------------------------------------------------------------------------------------------
Retrieval-augmented generation solves the problem of models giving a very plausible but incorrect answer, a phenomenon called hallucination. [https://blogs.nvidia.com/blog/what-is-retrieval-augmented-generation/]

Additionally, it also reduces the possibility of ambiguity in a user query and builds trust by giving models sources they can cite, like footnotes in a research paper, so users can check any claims. [https://blogs.nvidia.com/blog/what-is-retrieval-augmented-generation/]

It also helps to clear up ambiguity in a user query. [https://aws.amazon.com/what-is/retrieval-augmented-generation/]

Retrieval-augmented generation solves the problem of models making stuff up. [https://ars-technica.com/2024/06/can-a-technology-called-rag-keep-ai-models-from-making-stuff-up/]

It solves the problem of models giving incorrect answers. [https://en.wikipedia.org/wiki/Retrieval-augmented_generation]

sources: ['https://aws.amazon.com/what-is/retrieval-augmented-generation/', 'https://blogs.nvidia.com/blog/what-is-retrieval-augmented-generation/', 'https://en.wikipedia.org/wiki/Retrieval-augmented_generation']   top_sim=0.6325   44.68s

================================================================================================================
Q: What are the main components of a RAG pipeline, end to end?
----------------------------------------------------------------------------------------------------------------
The main components of a RAG pipeline, end to end, are:

1. User query
2. Query conversion to numeric format (embedding or vector)
3. Comparison of numeric values to vectors in a machine-readable index of an available knowledge base
4. Retrieval of related data
5. Conversion of retrieved data to human-readable words
6. Combination of retrieved data and the LLM's response to the query
7. Presentation of the final answer to the user, potentially citing sources.

[https://blogs.nvidia.com/blog/what-is-retrieval-augmented-generation/]

Note that the Wikipedia article does not provide a detailed end-to-end pipeline, but rather focuses on the key stages of RAG, which are:

1. Data to be referenced is converted into LLM embeddings
2. The embedding model compares the numeric values to vectors in a machine-readable index
3. The embedding model retrieves the related data
4. The embedding model converts the retrieved data to human-readable words
5. The LLM combines the retrieved words and its own response to the query

[https://en.wikipedia.org/wiki/Retrieval-augmented_generation]

sources: ['https://aws.amazon.com/what-is/retrieval-augmented-generation/', 'https://blogs.nvidia.com/blog/what-is-retrieval-augmented-generation/', 'https://en.wikipedia.org/wiki/Retrieval-augmented_generation']   top_sim=0.3462   16.29s

================================================================================================================
Q: What is the population of Saltillo?
----------------------------------------------------------------------------------------------------------------
The indexed sources do not cover this.

sources: ['https://aws.amazon.com/what-is/retrieval-augmented-generation/', 'https://blogs.nvidia.com/blog/what-is-retrieval-augmented-generation/', 'https://en.wikipedia.org/wiki/Retrieval-augmented_generation']   top_sim=0.0652   5.1s

OpenWebUI: reachable at http://localhost:3000 (16.4 ms)
23:32:34 | INFO    | evidence written -> 08_rag_generation.json

PosixPath('~/Documents/DBA/Master AI&ML/Applied Research Topics in Deep Learning- Theory & Practical Applications/Week 8/Case/Antonio_Gonzalez_WebScrapingChallenge/evidence/08_rag_generation.json')

# ---- Part 4D: Validate emitted citations against the indexed source allow-list ----
_URL_RE = re.compile(r"https?://[^\s\]\)>,;\"']+")
ALLOWED_URLS = {d.url for d in documents}

citation_report = []
if rag_evidence:
    print(f"{'QUESTION':<52}{'CITED':>7}{'VALID':>7}{'LEAKED':>8}")
    print("-" * 78)
    for a in rag_evidence:
        cited = set(_URL_RE.findall(a["answer"]))
        valid = {u for u in cited if any(u.startswith(src) for src in ALLOWED_URLS)}
        leaked = sorted(cited - valid)
        citation_report.append({"question": a["question"], "cited": len(cited),
                                "valid": len(valid), "leaked": leaked})
        print(f"{a['question'][:50]:<52}{len(cited):>7}{len(valid):>7}{len(leaked):>8}")
        for u in leaked:
            print(f"    LEAK -> {u[:96]}")
    total_leaks = sum(len(r["leaked"]) for r in citation_report)
    print("-" * 78)
    print(f"Total out-of-corpus citations: {total_leaks}")
    print()
    print("MITIGATION (production fix, not applied here so the defect stays visible):")
    print("  1. Cite from chunk METADATA (metadatas[i]['url']), never from text the")
    print("     model produced — provenance is a property of the retrieval step.")
    print("  2. Strip bare URLs and reference markers from chunk bodies at ingest time,")
    print("     so reference lists inside a source cannot be mistaken for provenance.")
    print("  3. Post-validate every emitted URL against the allow-list and reject or")
    print("     rewrite the answer when a citation falls outside it (this check).")
else:
    print("No generated answers to validate (Ollama was unavailable).")

save_evidence("11_citation_integrity.json",
              {"allowed_urls": sorted(ALLOWED_URLS), "per_answer": citation_report})

QUESTION                                              CITED  VALID  LEAKED
------------------------------------------------------------------------------
According to the indexed sources, what problem doe        4      3       1
    LEAK -> https://ars-technica.com/2024/06/can-a-technology-called-rag-keep-ai-models-from-making-stuff-up
What are the main components of a RAG pipeline, en        2      2       0
What is the population of Saltillo?                       0      0       0
------------------------------------------------------------------------------
Total out-of-corpus citations: 1

MITIGATION (production fix, not applied here so the defect stays visible):
  1. Cite from chunk METADATA (metadatas[i]['url']), never from text the
     model produced — provenance is a property of the retrieval step.
  2. Strip bare URLs and reference markers from chunk bodies at ingest time,
     so reference lists inside a source cannot be mistaken for provenance.
  3. Post-validate every emitted URL against the allow-list and reject or
     rewrite the answer when a citation falls outside it (this check).
23:32:34 | INFO    | evidence written -> 11_citation_integrity.json

PosixPath('~/Documents/DBA/Master AI&ML/Applied Research Topics in Deep Learning- Theory & Practical Applications/Week 8/Case/Antonio_Gonzalez_WebScrapingChallenge/evidence/11_citation_integrity.json')

# ---- Part 5.1: Automated submission checklist ----
checks = [
    ("All 3 required containers running",
     all(env_evidence["required_containers"].values())),
    ("ChromaDB heartbeat verified",
     env_evidence["probes"][0]["ok"]),
    ("3 URLs from 3 distinct domains",
     len({d.domain for d in documents}) == 3),
    ("robots.txt evaluated for every source",
     len(robots_report) == 3),
    ("All 3 URLs scraped without critical errors",
     len(documents) == 3),
    ("Corpus >= 50 chunks",
     len(corpus) >= 50),
    ("Embeddings stored in ChromaDB (count matches corpus)",
     collection.count() == len(corpus)),
    ("Embedding dimensionality reported",
     bool(vector_dim)),
    (">= 3 semantic search queries executed",
     len(search_evidence) >= 3),
    ("Chunk-size / overlap / model experiment matrix completed",
     len(experiments) >= 6),
    ("Comparison charts generated",
     all((EVIDENCE_DIR / f).exists() for f in
         ["chart_1_mrr_by_config.png", "chart_2_quality_vs_cost.png", "chart_3_precision_heatmap.png"])),
    ("Semantic vs. keyword baseline compared",
     (EVIDENCE_DIR / "07_semantic_vs_keyword.json").exists()),
    ("Retriever disagreement diagnosed",
     (EVIDENCE_DIR / "10_retriever_disagreement.json").exists()),
    ("Citation integrity validated",
     (EVIDENCE_DIR / "11_citation_integrity.json").exists()),
    ("Reproducible config present (.env + requirements.txt)",
     (NOTEBOOK_DIR / ".env").exists() and (NOTEBOOK_DIR / "requirements.txt").exists()),
]

print("=" * 96)
print("SUBMISSION CHECKLIST")
print("=" * 96)
for label, ok in checks:
    print(f"  [{'x' if ok else ' '}] {label}")
passed = sum(1 for _, ok in checks if ok)
print("-" * 96)
print(f"  {passed}/{len(checks)} automated checks passed")

manifest = {
    "run_id": RUN_ID,
    "generated_at": datetime.now(timezone.utc).isoformat(),
    "urls": [{"url": d.url, "domain": d.domain, "category": d.category,
              "title": d.title, "chars": d.n_chars} for d in documents],
    "collection": {"name": COLLECTION_NAME, "count": collection.count(),
                   "vector_dim": vector_dim, "space": SPACE_TYPE,
                   "embedding_model": EMBEDDING_MODEL},
    "chunking": {"strategy": "sentence_aware", "size": CHUNK_SIZE,
                 "overlap": CHUNK_OVERLAP, "total_chunks": len(corpus)},
    "best_experiment": {k: (v.item() if hasattr(v, "item") else v) for k, v in best.to_dict().items()},
    "checks": {label: bool(ok) for label, ok in checks},
    "evidence_files": sorted(p.name for p in EVIDENCE_DIR.iterdir() if p.is_file()),
}
save_evidence("09_manifest.json", manifest)
print("\nEvidence bundle:")
for f in manifest["evidence_files"]:
    print("   -", f)

================================================================================================
SUBMISSION CHECKLIST
================================================================================================
  [x] All 3 required containers running
  [x] ChromaDB heartbeat verified
  [x] 3 URLs from 3 distinct domains
  [x] robots.txt evaluated for every source
  [x] All 3 URLs scraped without critical errors
  [x] Corpus >= 50 chunks
  [x] Embeddings stored in ChromaDB (count matches corpus)
  [x] Embedding dimensionality reported
  [x] >= 3 semantic search queries executed
  [x] Chunk-size / overlap / model experiment matrix completed
  [x] Comparison charts generated
  [x] Semantic vs. keyword baseline compared
  [x] Retriever disagreement diagnosed
  [x] Citation integrity validated
  [x] Reproducible config present (.env + requirements.txt)
------------------------------------------------------------------------------------------------
  15/15 automated checks passed
23:32:34 | INFO    | evidence written -> 09_manifest.json

Evidence bundle:
   - .keep
   - 00_environment.json
   - 01_robots_compliance.json
   - 02_scrape_summary.json
   - 03_corpus_stats.json
   - 04_chroma_status.json
   - 05_experiment_results.csv
   - 05_experiment_results.json
   - 06_semantic_searches.json
   - 06_semantic_searches.md
   - 07_semantic_vs_keyword.json
   - 08_rag_generation.json
   - 09_manifest.json
   - 10_retriever_disagreement.json
   - 11_citation_integrity.json
   - chart_1_mrr_by_config.png
   - chart_2_quality_vs_cost.png
   - chart_3_precision_heatmap.png
   - container_evidence.txt

Layer	Component	Runtime	Endpoint
Orchestration	Jupyter Lab (venv)	host (macOS, Apple Silicon)	`http://localhost:8888`
Vector store	`chromadb/chroma:latest`	Docker container `chromadb-server`	`http://localhost:8000`
Embedding	`sentence-transformers` (MiniLM / MPNet)	host process (MPS accelerated)	in-process
Generation	`ollama/ollama`	Docker container `ollama`	`http://localhost:11434`
Chat UI	`ghcr.io/open-webui/open-webui:main`	Docker container `open-webui`	`http://localhost:3000`

Rubric section	Pts	Where it is addressed
A. Execution & interpretation (end-to-end)	15	Parts 0 → 4: container verification, 3 URLs scraped, chunk counts, embedding dimensionality, ≥3 semantic queries
B. Experiments & analysis	20	Part 3B (chunk-size × overlap × embedding-model matrix, 3 charts), Part 4B (semantic vs. lexical baseline), Observations blocks
C. Code quality & documentation	7	`.env` + `requirements.txt`, typed helper functions, structured logging, retry/rate limiting, robots.txt compliance
D. Reflection	3	Part 5 write-up + `assignment_report.docx`

#	Source	Category	Register	Why it was chosen
1	`en.wikipedia.org` — Retrieval-augmented generation	Educational / encyclopedic	Neutral, citation-dense, formal	Deep, stable, well-structured prose; the canonical definitional baseline
2	`aws.amazon.com` — What is RAG?	Cloud-vendor documentation	Instructional, architecture-oriented	Implementation and operational framing (data pipelines, cost, managed services)
3	`blogs.nvidia.com` — What is RAG?	Industry blog	Narrative, analogy-driven, promotional	Informal phrasing and metaphor — the hardest register for lexical matching

Run	Reported index time	Rate	What was different
1	11.53 s	4.7 chunks/s	SentenceTransformer loaded cold inside the timed region
2	3.84 s	14.1 chunks/s	model files warm in the OS page cache
3 (final)	19.48 s	2.8 chunks/s	cold load again — 13.1 s of the 19.5 s was `SentenceTransformer(...)`

Metric	Definition	Why it matters
`hit@5`	share of queries with ≥1 relevant chunk in the top 5	does the context window contain the answer at all
`precision@5`	mean share of the top 5 that are relevant	how much of the LLM's context budget is wasted
`MRR`	mean reciprocal rank of the first relevant chunk	is the answer at the top, where the model attends most
`mean_top1_sim`	mean cosine similarity of the best hit	retrieval confidence / calibration
`domain_coverage@5`	distinct source domains in the top 5	evidence diversity vs. single-source echo
`chunks`, `index_s`	corpus size and build time	the cost side of the trade-off

Case Study 8 — Web Scraping to Vector Database¶

System architecture¶

How this notebook maps to the rubric (45 pts)¶

Part 0 — Setup & environment verification¶

Observations — Part 0¶

Part 1 — Source selection, robots.txt compliance, scraping & cleaning¶

URL selection rationale¶

Ethical scraping controls implemented¶

Observations — Part 1¶

Part 2 — Chunking¶

Observations — Part 2¶

Part 3 — Embeddings & ChromaDB storage¶

Observations — Part 3¶

Part 3B — Experiments: chunk size × overlap × embedding model¶

Why this experiment, and how it is scored¶

Observations — Part 3B¶

Part 4 — Semantic search¶

Part 4.3 — Why the two retrievers tie, and what that actually means¶

Observations — Part 4.2 / 4.3¶

Observations — Part 4¶

Part 4C — Retrieval-augmented generation with Ollama (bonus: integration challenge)¶

Part 4D — Citation integrity of the generated answers¶

Observations — Part 4D¶

Observations — Part 4C¶

Part 5 — Deliverables, verification & reflection¶

Reflection¶

Source	Verdict	Evidence
en.wikipedia.org	ALLOW	parsed OK (HTTP 200, 712 lines), no crawl-delay declared
aws.amazon.com	ALLOW	parsed OK (HTTP 200, 314 lines)
blogs.nvidia.com	ALLOW	parsed OK (HTTP 200, 180 lines)

	best configuration	MRR	precision@5	chunks	index
all-MiniLM-L6-v2 (384d)	1000 / 300	0.906	0.775	60	0.20 s
all-mpnet-base-v2 (768d)	1500 / 300	0.9375	0.925	36	0.90 s

Query set	semantic MRR	lexical MRR	delta
SET A — literal (n=8)	0.817	0.844	−0.027
SET B — paraphrase (n=4)	1.000	1.000	0.000

Query set	mean Jaccard@5	top-1 agreement
SET A — literal	0.303	3 / 8
SET B — paraphrase	0.083	0 / 4
Overall	0.23	5 / 12

Question	top-1 similarity	Outcome
What problem does RAG solve?	0.6325	answered, cited
Main components of a RAG pipeline?	0.3462	answered as a 7-step pipeline, cited
What is the population of Saltillo?	0.0652	"The indexed sources do not cover this."