from __future__ import annotations

import json
import re
from datetime import datetime, timezone
from typing import Any

from .provider import call_chat

MEMORY_TYPES = [
    "Event", "Conversation", "Promise", "Conflict", "Relationship Moment",
    "Discovery", "Secret Learned", "Preference", "Habit", "Emotional Moment",
    "Achievement", "Injury/Physical Event", "Other",
]
IMPORTANCE_LEVELS = ["Minor", "Normal", "Important", "Critical"]
EMOTIONAL_WEIGHTS = ["positive", "negative", "mixed", "neutral", "unresolved"]
RELEVANCE_LEVELS = ["active", "resolved", "superseded", "archived"]
WORLD_SECTIONS = [
    "history_backstory", "background_facts", "characters", "relationship_history",
    "core_canon", "locations", "secrets_knowledge", "groups_factions",
    "important_objects", "world_rules", "current_state", "physical_continuity",
    "story_timeline", "custom_entries",
]

MEMORY_SYSTEM = """You are the continuity extractor inside a persistent roleplay engine.
Analyze ONE canonical roleplay turn and return ONLY valid JSON. Never write prose outside JSON.

The JSON object MUST have these keys:
{
  "memories": [],
  "world_bible_updates": [],
  "state_updates": {},
  "conflicts": []
}

MEMORIES
Create a memory only when something would realistically matter later. Do NOT save mundane gestures, ordinary walking/sitting/smiling, filler dialogue, or every line of conversation.
Each memory object may contain:
- stable_key: short stable identity for the same evolving fact/state, e.g. "relationship:xaden:user:trust" or "promise:xaden:return-home". Use a new unique event key for one-time events like a first kiss.
- action: "create" or "update". Use update only when the same memory/state already exists and evolves.
- title
- what_happened: concise factual event or remembered fact
- who_remembers: array of characters/personas who actually know/experienced it. Do NOT make memories globally omniscient.
- who_involved: array
- when: story date/time, relative time, chapter/session, or empty string
- where: location or empty string
- memory_type: one of Event, Conversation, Promise, Conflict, Relationship Moment, Discovery, Secret Learned, Preference, Habit, Emotional Moment, Achievement, Injury/Physical Event, Other
- importance: one of Minor, Normal, Important, Critical
- emotional_weight: positive, negative, mixed, neutral, unresolved, or empty string
- current_relevance: active, resolved, superseded, archived
- confidence: 0.0-1.0

WORLD BIBLE
The World Bible represents structured long-term truth/current state; Memories represent remembered events/facts.
Add new established facts automatically when the RP explicitly establishes them. Do not require the user to type them manually.
Each world_bible_updates object:
- section: one of history_backstory, background_facts, characters, relationship_history, core_canon, locations, secrets_knowledge, groups_factions, important_objects, world_rules, current_state, physical_continuity, story_timeline, custom_entries
- stable_key: stable identity for dedup/update
- action: "create" or "update"
- item: a compact JSON object using natural fields for the section (name, fact, event, details, known_by, participants, date, location, relationship/status, etc.)
- confidence: 0.0-1.0
- conflicts_with_user_canon: true/false

Rules:
- Never invent missing facts.
- Tiny facts are valid ("raised in Texas", "son of Fen Riorson").
- Past/pre-RP events go to history_backstory; events happening during this RP go to story_timeline.
- New current state may also update current_state or physical_continuity.
- A secret belongs in secrets_knowledge with who knows it.
- Existing user-authored canon outranks model inference. If the new turn conflicts with user-authored canon, set conflicts_with_user_canon=true and explain it in conflicts; do NOT propose overwriting the user fact.
- Distinguish historical truth from current status. A breakup does not erase that the couple previously dated.
- Only extract facts explicitly established by this turn and supplied state. Preserve uncertainty.

STATE UPDATES
Optional conservative patches for relationship/scene/knowledge/physical. Only include keys explicitly changed by this turn. Never modify hard canon here.
"""


def _clean_list(value: Any) -> list[str]:
    if not isinstance(value, list):
        return []
    out: list[str] = []
    for x in value:
        s = re.sub(r"\s+", " ", str(x or "")).strip()
        if s and s.casefold() not in {y.casefold() for y in out}:
            out.append(s[:160])
    return out[:24]


def _slug(text: str) -> str:
    s = re.sub(r"[^a-z0-9]+", "-", str(text or "").lower()).strip("-")
    return s[:120] or "item"


def _now() -> str:
    return datetime.now(timezone.utc).isoformat()


def normalize_memory(m: dict[str, Any], *, fallback_key_seed: str = "") -> dict[str, Any]:
    what = re.sub(r"\s+", " ", str(m.get("what_happened") or m.get("content") or "")).strip()[:1800]
    title = re.sub(r"\s+", " ", str(m.get("title") or "Memory")).strip()[:160] or "Memory"
    mtype = str(m.get("memory_type") or "Other")
    if mtype not in MEMORY_TYPES:
        aliases = {
            "episodic": "Event", "relationship": "Relationship Moment", "knowledge": "Secret Learned",
            "canon": "Discovery", "physical": "Injury/Physical Event",
        }
        mtype = aliases.get(mtype.lower(), "Other")
    importance = str(m.get("importance") or "Normal")
    if importance not in IMPORTANCE_LEVELS:
        aliases = {"short-term": "Minor", "long-term": "Normal", "relationship": "Important", "hard canon": "Critical"}
        importance = aliases.get(importance.lower(), "Normal")
    emotion = str(m.get("emotional_weight") or "").lower().strip()
    if emotion not in EMOTIONAL_WEIGHTS:
        emotion = ""
    relevance = str(m.get("current_relevance") or "active").lower().strip()
    if relevance not in RELEVANCE_LEVELS:
        relevance = "active"
    stable = str(m.get("stable_key") or "").strip().lower()
    if not stable:
        stable = f"{_slug(mtype)}:{_slug(title or what or fallback_key_seed)}"
    try:
        confidence = max(0.0, min(1.0, float(m.get("confidence", 0.85))))
    except Exception:
        confidence = 0.85
    return {
        "stable_key": stable[:180],
        "action": "update" if str(m.get("action") or "create").lower() == "update" else "create",
        "title": title,
        "what_happened": what,
        "content": what,
        "who_remembers": _clean_list(m.get("who_remembers")),
        "who_involved": _clean_list(m.get("who_involved")),
        "story_when": str(m.get("when") or m.get("story_when") or "").strip()[:180],
        "story_where": str(m.get("where") or m.get("story_where") or "").strip()[:240],
        "memory_type": mtype,
        "importance": importance,
        "emotional_weight": emotion,
        "current_relevance": relevance,
        "confidence": confidence,
    }


def heuristic_extract(user_text: str, assistant_text: str, state: dict[str, Any] | None = None) -> dict[str, Any]:
    """Very conservative offline fallback. The real extractor should use an LLM."""
    joined = re.sub(r"\s+", " ", f"{user_text}\n{assistant_text}").strip()
    memories: list[dict[str, Any]] = []
    # Save only unmistakable promises/revelations in mock/offline mode.
    promise = re.search(r"\b(?:i|we)\s+promise\b[^.!?]{0,220}", joined, flags=re.I)
    if promise:
        text = promise.group(0).strip()
        memories.append(normalize_memory({
            "stable_key": f"promise:{_slug(text[:80])}", "title": "Promise made", "what_happened": text,
            "memory_type": "Promise", "importance": "Important", "emotional_weight": "unresolved",
            "current_relevance": "active", "who_remembers": [], "who_involved": [], "confidence": .72,
        }))
    return {"memories": memories, "world_bible_updates": [], "state_updates": {}, "conflicts": []}


def llm_extract(
    provider: str,
    model: str | None,
    user_text: str,
    assistant_text: str,
    *,
    state: dict[str, Any] | None = None,
    conversation: dict[str, Any] | None = None,
    recent_memories: list[dict[str, Any]] | None = None,
) -> dict[str, Any]:
    if provider == "mock":
        return heuristic_extract(user_text, assistant_text, state)
    context = {
        "conversation": {
            "persona_name": (conversation or {}).get("persona_name"),
            "scenario_name": (conversation or {}).get("scenario_name"),
        },
        "hard_canon": (state or {}).get("hard_canon", {}),
        "relationship": (state or {}).get("relationship", {}),
        "knowledge": (state or {}).get("knowledge", {}),
        "scene": (state or {}).get("scene", {}),
        "physical": (state or {}).get("physical", {}),
        "world_bible": (state or {}).get("world_bible", {}),
        "existing_memories": [
            {
                "stable_key": x.get("stable_key"), "title": x.get("title"), "what_happened": x.get("what_happened") or x.get("content"),
                "who_remembers": x.get("who_remembers"), "current_relevance": x.get("current_relevance"),
            }
            for x in (recent_memories or [])[:40]
        ],
    }
    prompt = (
        "CURRENT STRUCTURED STATE:\n" + json.dumps(context, ensure_ascii=False)[:15000] +
        "\n\nUSER TURN:\n" + user_text[:8000] +
        "\n\nCANONICAL AI TURN:\n" + assistant_text[:12000]
    )
    messages = [{"role": "system", "content": MEMORY_SYSTEM}, {"role": "user", "content": prompt}]
    result = call_chat(provider, model, messages, temperature=0.05, max_tokens=2200)
    raw = str(result.get("content") or "").strip()
    raw = re.sub(r"^```(?:json)?\s*|\s*```$", "", raw, flags=re.I | re.S)
    try:
        data = json.loads(raw)
        if isinstance(data, dict):
            out = {
                "memories": [],
                "world_bible_updates": data.get("world_bible_updates") if isinstance(data.get("world_bible_updates"), list) else [],
                "state_updates": data.get("state_updates") if isinstance(data.get("state_updates"), dict) else {},
                "conflicts": data.get("conflicts") if isinstance(data.get("conflicts"), list) else [],
            }
            for m in (data.get("memories") or [])[:12]:
                if isinstance(m, dict):
                    n = normalize_memory(m, fallback_key_seed=assistant_text[:120])
                    if n["what_happened"]:
                        out["memories"].append(n)
            return out
    except Exception:
        pass
    return heuristic_extract(user_text, assistant_text, state)


def _json_load(value: Any, fallback: Any) -> Any:
    if isinstance(value, (dict, list)):
        return value
    try:
        return json.loads(value or "")
    except Exception:
        return fallback


def _descriptor(item: dict[str, Any]) -> str:
    for k in ("stable_key", "name", "title", "fact", "event", "rule", "relationship", "status", "value", "details", "description"):
        v = item.get(k)
        if isinstance(v, str) and v.strip():
            return _slug(v)
    return _slug(json.dumps(item, ensure_ascii=False, sort_keys=True)[:240])


def _is_auto(item: Any) -> bool:
    return isinstance(item, dict) and str(item.get("source") or "").lower().startswith("auto-rp")


def _merge_auto_array(existing: list[Any], incoming: dict[str, Any], stable_key: str, provenance: dict[str, Any]) -> tuple[list[Any], str | None]:
    item = dict(incoming or {})
    item["stable_key"] = stable_key
    item["source"] = "auto-rp"
    item["provenance"] = provenance
    item["updated_at"] = _now()
    target_desc = _descriptor(item)
    for i, old in enumerate(existing):
        if not isinstance(old, dict):
            continue
        same = str(old.get("stable_key") or "").lower() == stable_key.lower() or _descriptor(old) == target_desc
        if not same:
            continue
        if not _is_auto(old):
            return existing, f"Protected user/imported entry matched {stable_key}; automatic update was not applied."
        history = list(old.get("revision_history") or [])
        snapshot = {k: v for k, v in old.items() if k not in {"revision_history", "updated_at"}}
        history.append({"replaced_at": _now(), "snapshot": snapshot})
        item["revision_history"] = history[-20:]
        existing[i] = {**old, **item}
        return existing, None
    existing.append(item)
    return existing, None



def _source_message_id(item: Any) -> Any:
    if not isinstance(item, dict):
        return None
    if item.get("source_message_id") is not None:
        return item.get("source_message_id")
    provenance = item.get("provenance") if isinstance(item.get("provenance"), dict) else {}
    if provenance.get("source_message_id") is not None:
        return provenance.get("source_message_id")
    context = item.get("original_context") if isinstance(item.get("original_context"), dict) else {}
    if context.get("source_message_id") is not None:
        return context.get("source_message_id")
    return None


def _id_in(value: Any, message_ids: set[str]) -> bool:
    return value is not None and str(value) in message_ids


def _rollback_auto_array(items: Any, message_ids: set[str]) -> tuple[list[Any], int]:
    """Remove/restore auto-rp entries sourced only from abandoned reroll messages."""
    out: list[Any] = []
    changed = 0
    for raw in list(items or []) if isinstance(items, list) else []:
        if not isinstance(raw, dict) or not _is_auto(raw) or not _id_in(_source_message_id(raw), message_ids):
            out.append(raw)
            continue
        history = list(raw.get("revision_history") or []) if isinstance(raw.get("revision_history"), list) else []
        restored = None
        while history:
            rev = history.pop()
            snap = rev.get("snapshot") if isinstance(rev, dict) and isinstance(rev.get("snapshot"), dict) else None
            if snap and not _id_in(_source_message_id(snap), message_ids):
                restored = dict(snap)
                restored["revision_history"] = history[-20:]
                break
        if restored is not None:
            out.append(restored)
        changed += 1
    return out, changed


def _rollback_auto_current(doc: dict[str, Any], message_ids: set[str]) -> tuple[dict[str, Any], int]:
    current = doc.get("auto_current") if isinstance(doc.get("auto_current"), dict) else None
    if not current or not _id_in(_source_message_id(current), message_ids):
        return doc, 0
    history = list(doc.get("auto_current_history") or []) if isinstance(doc.get("auto_current_history"), list) else []
    restored = None
    while history:
        rev = history.pop()
        snap = rev.get("snapshot") if isinstance(rev, dict) and isinstance(rev.get("snapshot"), dict) else None
        if snap and not _id_in(_source_message_id(snap), message_ids):
            restored = dict(snap)
            break
    if restored is None:
        doc.pop("auto_current", None)
    else:
        doc["auto_current"] = restored
    if history:
        doc["auto_current_history"] = history[-20:]
    else:
        doc.pop("auto_current_history", None)
    return doc, 1


def rollback_auto_continuity_for_messages(
    db_module: Any,
    conversation_id: int | str,
    source_message_ids: list[int | str] | tuple[int | str, ...] | set[int | str],
) -> dict[str, int]:
    """Undo only auto-derived continuity from abandoned canonical reroll variants.

    User-created/imported memories and user-authored World Bible entries are never
    touched. Where an auto entry evolved from an earlier valid state, its revision
    snapshot is restored instead of deleting the whole fact.
    """
    ids = {str(x) for x in source_message_ids if x is not None}
    if not ids:
        return {"memories": 0, "world_entries": 0, "state_entries": 0, "conflicts": 0}

    memories_changed = 0
    for row in db_module.rows(
        "SELECT * FROM memories WHERE conversation_id=? AND source_kind='Auto-created from RP'",
        (conversation_id,),
    ):
        if not _id_in(row.get("source_message_id"), ids):
            continue
        history = _json_load(row.get("revision_history"), [])
        restored = None
        while isinstance(history, list) and history:
            rev = history.pop()
            if not isinstance(rev, dict):
                continue
            snap = rev.get("snapshot") if isinstance(rev.get("snapshot"), dict) else {
                k: v for k, v in rev.items() if k != "replaced_at"
            }
            if snap and not _id_in(_source_message_id(snap), ids):
                restored = snap
                break
        if restored:
            who_remembers = restored.get("who_remembers") if isinstance(restored.get("who_remembers"), list) else _json_load(restored.get("who_remembers"), [])
            who_involved = restored.get("who_involved") if isinstance(restored.get("who_involved"), list) else _json_load(restored.get("who_involved"), [])
            original_context = restored.get("original_context") if isinstance(restored.get("original_context"), dict) else _json_load(restored.get("original_context"), {})
            relevance = str(restored.get("current_relevance") or "active").lower()
            db_module.execute(
                """UPDATE memories SET memory_type=?,title=?,content=?,what_happened=?,importance=?,who_remembers=?,who_involved=?,story_when=?,story_where=?,emotional_weight=?,current_relevance=?,source_kind=?,original_context=?,revision_history=?,confidence=?,source_message_id=?,active=?,updated_at=CURRENT_TIMESTAMP WHERE id=?""",
                (
                    restored.get("memory_type") or row.get("memory_type") or "Other",
                    restored.get("title") or row.get("title") or "Memory",
                    restored.get("content") or restored.get("what_happened") or "",
                    restored.get("what_happened") or restored.get("content") or "",
                    restored.get("importance") or "Normal",
                    json.dumps(who_remembers, ensure_ascii=False), json.dumps(who_involved, ensure_ascii=False),
                    restored.get("story_when") or "", restored.get("story_where") or "",
                    restored.get("emotional_weight") or "", relevance,
                    restored.get("source_kind") or "Auto-created from RP",
                    json.dumps(original_context, ensure_ascii=False), json.dumps(history[-20:], ensure_ascii=False),
                    restored.get("confidence") if restored.get("confidence") is not None else 1.0,
                    restored.get("source_message_id"), 0 if relevance == "archived" else 1, row["id"],
                ),
            )
        else:
            db_module.execute("DELETE FROM memories WHERE id=? AND conversation_id=?", (row["id"], conversation_id))
        memories_changed += 1

    state = db_module.get_state(conversation_id)
    world = dict(state.get("world_bible") or {})
    hard = dict(state.get("hard_canon") or {})
    knowledge = dict(state.get("knowledge") or {})
    scene = dict(state.get("scene") or {})
    physical = dict(state.get("physical") or {})
    relationship = dict(state.get("relationship") or {})
    world_changed = state_changed = conflict_changed = 0

    for section in WORLD_SECTIONS:
        if isinstance(world.get(section), list):
            world[section], n = _rollback_auto_array(world[section], ids)
            world_changed += n
    if isinstance(world.get("auto_review_conflicts"), list):
        before = len(world["auto_review_conflicts"])
        world["auto_review_conflicts"] = [x for x in world["auto_review_conflicts"] if not _id_in(_source_message_id(x), ids)]
        conflict_changed += before - len(world["auto_review_conflicts"])

    for doc, key in ((hard, "auto_facts"), (knowledge, "facts"), (scene, "auto_facts"), (physical, "continuity_notes")):
        if isinstance(doc.get(key), list):
            doc[key], n = _rollback_auto_array(doc[key], ids)
            state_changed += n

    for doc in (relationship, scene, knowledge, physical):
        doc, n = _rollback_auto_current(doc, ids)
        state_changed += n

    if world_changed or conflict_changed:
        db_module.update_state(conversation_id, "world_bible", world)
    if state_changed:
        db_module.update_state(conversation_id, "hard_canon", hard)
        db_module.update_state(conversation_id, "knowledge", knowledge)
        db_module.update_state(conversation_id, "scene", scene)
        db_module.update_state(conversation_id, "physical", physical)
        db_module.update_state(conversation_id, "relationship", relationship)

    return {
        "memories": memories_changed,
        "world_entries": world_changed,
        "state_entries": state_changed,
        "conflicts": conflict_changed,
    }

def apply_extracted_local(
    db_module: Any,
    conversation_id: int | str,
    extracted: dict[str, Any],
    *,
    source_message_id: int | str | None,
    user_text: str,
    assistant_text: str,
) -> dict[str, Any]:
    """Persist structured memories + automatic World Bible/state growth for local/offline mode."""
    created = updated = bible_added = bible_updated = 0
    conflicts: list[Any] = list(extracted.get("conflicts") or [])
    provenance = {
        "source": "rp_turn", "source_message_id": source_message_id,
        "captured_at": _now(), "user_excerpt": user_text[:320], "assistant_excerpt": assistant_text[:520],
    }

    for raw in (extracted.get("memories") or [])[:12]:
        if not isinstance(raw, dict):
            continue
        m = normalize_memory(raw, fallback_key_seed=assistant_text[:100])
        if not m["what_happened"] or m["confidence"] < 0.55:
            continue
        row = db_module.row(
            "SELECT * FROM memories WHERE conversation_id=? AND stable_key=? ORDER BY id DESC LIMIT 1",
            (conversation_id, m["stable_key"]),
        )
        context = {"source": "Auto-created from RP", "source_message_id": source_message_id, "turn": provenance}
        if row:
            # Manual/imported memories are authoritative. Automatic RP extraction
            # may reference them, but must never silently rewrite them.
            if str(row.get("source_kind") or "").strip().lower() != "auto-created from rp":
                conflicts.append({
                    "type": "memory",
                    "stable_key": m["stable_key"],
                    "reason": "Existing user/imported memory is protected from automatic overwrite.",
                })
                continue
            history = _json_load(row.get("revision_history"), [])
            # Keep a complete prior snapshot so a later reroll-canon switch can
            # restore the previous valid memory instead of deleting history.
            history.append({
                "replaced_at": _now(),
                "snapshot": {
                    "memory_type": row.get("memory_type"), "title": row.get("title"),
                    "content": row.get("content"), "what_happened": row.get("what_happened") or row.get("content"),
                    "importance": row.get("importance"), "who_remembers": _json_load(row.get("who_remembers"), []),
                    "who_involved": _json_load(row.get("who_involved"), []), "story_when": row.get("story_when"),
                    "story_where": row.get("story_where"), "emotional_weight": row.get("emotional_weight"),
                    "current_relevance": row.get("current_relevance"), "source_kind": row.get("source_kind"),
                    "original_context": _json_load(row.get("original_context"), {}), "stable_key": row.get("stable_key"),
                    "confidence": row.get("confidence"), "source_message_id": row.get("source_message_id"),
                    "active": row.get("active"),
                },
            })
            db_module.execute(
                """UPDATE memories SET memory_type=?,title=?,content=?,what_happened=?,importance=?,who_remembers=?,who_involved=?,story_when=?,story_where=?,emotional_weight=?,current_relevance=?,source_kind='Auto-created from RP',original_context=?,revision_history=?,confidence=?,source_message_id=?,active=?,updated_at=CURRENT_TIMESTAMP WHERE id=?""",
                (
                    m["memory_type"], m["title"], m["what_happened"], m["what_happened"], m["importance"],
                    json.dumps(m["who_remembers"], ensure_ascii=False), json.dumps(m["who_involved"], ensure_ascii=False),
                    m["story_when"], m["story_where"], m["emotional_weight"], m["current_relevance"],
                    json.dumps(context, ensure_ascii=False), json.dumps(history[-20:], ensure_ascii=False), m["confidence"], source_message_id,
                    0 if m["current_relevance"] == "archived" else 1, row["id"],
                ),
            )
            updated += 1
        else:
            db_module.execute(
                """INSERT INTO memories(conversation_id,memory_type,title,content,importance,source_message_id,confidence,active,what_happened,who_remembers,who_involved,story_when,story_where,emotional_weight,current_relevance,source_kind,original_context,stable_key,revision_history) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
                (
                    conversation_id, m["memory_type"], m["title"], m["what_happened"], m["importance"], source_message_id,
                    m["confidence"], 0 if m["current_relevance"] == "archived" else 1, m["what_happened"],
                    json.dumps(m["who_remembers"], ensure_ascii=False), json.dumps(m["who_involved"], ensure_ascii=False),
                    m["story_when"], m["story_where"], m["emotional_weight"], m["current_relevance"], "Auto-created from RP",
                    json.dumps(context, ensure_ascii=False), m["stable_key"], "[]",
                ),
            )
            created += 1

    state = db_module.get_state(conversation_id)
    world = dict(state.get("world_bible") or {})
    for key in ("history_backstory", "background_facts", "characters", "relationship_history", "locations", "groups_factions", "important_objects", "world_rules", "story_timeline", "custom_entries"):
        if not isinstance(world.get(key), list):
            world[key] = []
    world.setdefault("auto_review_conflicts", [])
    hard = dict(state.get("hard_canon") or {})
    knowledge = dict(state.get("knowledge") or {})
    scene = dict(state.get("scene") or {})
    physical = dict(state.get("physical") or {})
    hard.setdefault("auto_facts", [])
    knowledge.setdefault("facts", [])
    scene.setdefault("auto_facts", [])
    physical.setdefault("continuity_notes", [])

    changed_docs: set[str] = set()
    for upd in (extracted.get("world_bible_updates") or [])[:24]:
        if not isinstance(upd, dict):
            continue
        section = str(upd.get("section") or "custom_entries")
        if section not in WORLD_SECTIONS:
            section = "custom_entries"
        try:
            conf = max(0.0, min(1.0, float(upd.get("confidence", .8))))
        except Exception:
            conf = .8
        if conf < .58:
            continue
        if bool(upd.get("conflicts_with_user_canon")):
            conflicts.append({"section": section, "item": upd.get("item"), "reason": "Extractor flagged a conflict with user-authored canon."})
            continue
        item = upd.get("item") if isinstance(upd.get("item"), dict) else {"details": str(upd.get("item") or "")}
        if not any(str(v or "").strip() for v in item.values() if not isinstance(v, (dict, list))):
            continue
        stable = str(upd.get("stable_key") or f"{section}:{_descriptor(item)}").strip().lower()[:180]
        p = {**provenance, "confidence": conf}
        if section == "core_canon":
            arr, conflict = _merge_auto_array(list(hard.get("auto_facts") or []), item, stable, p)
            hard["auto_facts"] = arr; changed_docs.add("hard_canon")
        elif section == "secrets_knowledge":
            arr, conflict = _merge_auto_array(list(knowledge.get("facts") or []), item, stable, p)
            knowledge["facts"] = arr; changed_docs.add("knowledge")
        elif section == "current_state":
            arr, conflict = _merge_auto_array(list(scene.get("auto_facts") or []), item, stable, p)
            scene["auto_facts"] = arr; changed_docs.add("scene")
        elif section == "physical_continuity":
            arr, conflict = _merge_auto_array(list(physical.get("continuity_notes") or []), item, stable, p)
            physical["continuity_notes"] = arr; changed_docs.add("physical")
        else:
            arr = list(world.get(section) or [])
            before = len(arr)
            arr, conflict = _merge_auto_array(arr, item, stable, p)
            world[section] = arr; changed_docs.add("world_bible")
            if conflict is None:
                if len(arr) > before: bible_added += 1
                else: bible_updated += 1
        if conflict:
            conflicts.append({"section": section, "stable_key": stable, "reason": conflict, "item": item})

    # Conservative special-state patches. These are stored under auto_* namespaces so user-authored root fields are never silently replaced.
    # Keep source lineage/history so changing the canonical reroll can undo only
    # the abandoned branch while preserving earlier valid state.
    def apply_auto_current(doc: dict[str, Any], patch: dict[str, Any]) -> dict[str, Any]:
        previous = doc.get("auto_current") if isinstance(doc.get("auto_current"), dict) else None
        hist = list(doc.get("auto_current_history") or []) if isinstance(doc.get("auto_current_history"), list) else []
        if previous:
            hist.append({"replaced_at": _now(), "snapshot": dict(previous)})
            doc["auto_current_history"] = hist[-20:]
        doc["auto_current"] = {
            **dict(previous or {}), **patch,
            "source": "auto-rp", "source_message_id": source_message_id, "updated_at": _now(),
        }
        return doc

    state_updates = extracted.get("state_updates") if isinstance(extracted.get("state_updates"), dict) else {}
    if isinstance(state_updates.get("relationship"), dict) and state_updates["relationship"]:
        rel = apply_auto_current(dict(state.get("relationship") or {}), state_updates["relationship"])
        db_module.update_state(conversation_id, "relationship", rel)
    if isinstance(state_updates.get("scene"), dict) and state_updates["scene"]:
        scene = apply_auto_current(scene, state_updates["scene"])
        changed_docs.add("scene")
    if isinstance(state_updates.get("knowledge"), dict) and state_updates["knowledge"]:
        knowledge = apply_auto_current(knowledge, state_updates["knowledge"])
        changed_docs.add("knowledge")
    if isinstance(state_updates.get("physical"), dict) and state_updates["physical"]:
        physical = apply_auto_current(physical, state_updates["physical"])
        changed_docs.add("physical")

    if conflicts:
        review = list(world.get("auto_review_conflicts") or [])
        for c in conflicts[-20:]:
            review.append({"detected_at": _now(), "source_message_id": source_message_id, **(c if isinstance(c, dict) else {"reason": str(c)})})
        world["auto_review_conflicts"] = review[-60:]
        changed_docs.add("world_bible")
    if "world_bible" in changed_docs:
        world["schema_version"] = max(3, int(world.get("schema_version") or 0))
        db_module.update_state(conversation_id, "world_bible", world)
    if "hard_canon" in changed_docs:
        db_module.update_state(conversation_id, "hard_canon", hard)
    if "knowledge" in changed_docs:
        db_module.update_state(conversation_id, "knowledge", knowledge)
    if "scene" in changed_docs:
        db_module.update_state(conversation_id, "scene", scene)
    if "physical" in changed_docs:
        db_module.update_state(conversation_id, "physical", physical)

    return {
        "memories_created": created, "memories_updated": updated,
        "world_bible_added": bible_added, "world_bible_updated": bible_updated,
        "conflicts": len(conflicts),
    }
