import hashlib
import logging
import os
import re
import time
import uuid
from typing import Optional

import requests
from qdrant_client import QdrantClient
from qdrant_client.models import Distance, FieldCondition, Filter, MatchValue, PointStruct, VectorParams

logger = logging.getLogger(__name__)


class QdrantBase:
    _connection_failed_logged = False

    def __init__(self, collection_name, vector_size=768):
        if QdrantBase._connection_failed_logged:
            self.client = None
            return

        self.collection_name = collection_name
        self.client = None
        self._vector_size = vector_size
        self._consecutive_errors = 0
        self._circuit_open = False

        try:
            qdrant_url = os.environ.get("QDRANT_URL", "http://localhost:6344")
            if qdrant_url == ":memory:":
                self.client = QdrantClient(location=":memory:")
            else:
                self.client = QdrantClient(url=qdrant_url, timeout=10.0)

            if self.client:
                if self.client.collection_exists(collection_name):
                    # Verify dimension match
                    try:
                        info = self.client.get_collection(collection_name)
                        existing_size = info.config.params.vectors.size
                        if existing_size != vector_size:
                            logger.warning(
                                f"Qdrant dimension mismatch for '{collection_name}': "
                                f"collection has {existing_size}, expected {vector_size}. "
                                f"Recreating collection..."
                            )
                            self.client.delete_collection(collection_name)
                            self.client.create_collection(
                                collection_name=collection_name,
                                vectors_config=VectorParams(size=vector_size, distance=Distance.COSINE),
                            )
                            logger.info(
                                f"Recreated Qdrant collection '{collection_name}' with dimension {vector_size}."
                            )
                    except Exception as dim_err:
                        logger.debug(f"Qdrant dimension check failed (non-fatal): {dim_err}")
                else:
                    self.client.create_collection(
                        collection_name=collection_name,
                        vectors_config=VectorParams(size=vector_size, distance=Distance.COSINE),
                    )
                    logger.info(f"Created Qdrant collection '{collection_name}' with dimension {vector_size}.")
        except Exception as e:
            if not QdrantBase._connection_failed_logged:
                if "ConnectError" in str(type(e)):
                    logger.error(f"⚠️ [Qdrant] Connection refused at {qdrant_url}. Is Qdrant running?")
                else:
                    logger.error(f"Failed to initialize Qdrant memory (likely not running): {e}")
                QdrantBase._connection_failed_logged = True
            # No debug logs here to prevent noise in --debug mode
            self.client = None

    @property
    def is_connected(self) -> bool:
        return self.client is not None and not self._circuit_open

    def wipe_collection(self):
        """Safely wipes and recreates the collection to prevent 404 errors."""
        if not self.is_connected:
            return
        try:
            self.client.delete_collection(self.collection_name)
            self.client.create_collection(
                collection_name=self.collection_name,
                vectors_config=VectorParams(size=self._vector_size, distance=Distance.COSINE),
            )
            logger.info(f"🗑️ Wiped and recreated Qdrant collection: {self.collection_name}")
        except Exception as e:
            logger.error(f"Failed to wipe collection {self.collection_name}: {e}")

    def _handle_error(self, e: Exception, context: str):
        self._consecutive_errors += 1
        logger.error(f"❌ [Qdrant] {context}: {e}")
        if self._consecutive_errors >= 3:
            if not self._circuit_open:
                logger.critical(
                    f"🚨 [Qdrant] Circuit breaker triggered! Disabling memory engine for {self.collection_name} due to persistent failures."
                )
            self._circuit_open = True

    def _handle_success(self):
        if self._circuit_open:
            logger.info(f"🟢 [Qdrant] Connection recovered for {self.collection_name}. Circuit closed.")
        self._consecutive_errors = 0
        self._circuit_open = False

    def _get_embedding(self, text: str) -> Optional[list]:
        from GramAddict.core.config import Config

        if not hasattr(self, "_cached_args"):
            cfg = Config()
            self._cached_args = cfg.args if hasattr(cfg, "args") else None
        args = self._cached_args

        # Pull specific embedding config or fallback to defaults
        model = getattr(args, "ai_embedding_model")
        url = getattr(args, "ai_embedding_url")

        try:
            # Generate embeddings
            is_cloud = "openrouter.ai" in url.lower() or "openai.com" in url.lower()

            headers = {}
            if is_cloud:
                key = os.environ.get("OPENROUTER_API_KEY")
                if key:
                    headers["Authorization"] = f"Bearer {key}"
                # OpenAI/OpenRouter use 'input' instead of 'prompt'
                payload = {"model": model, "input": str(text)[:2000]}
            else:
                # Local Ollama
                payload = {"model": model, "prompt": str(text)[:2000]}

            # Log to prevent user from thinking the bot is hung during model swap in VRAM
            if not getattr(self, "_has_logged_embedding", False):
                logger.debug(
                    "🧠 [Ollama] Generating embeddings... (First local hit may take 5-10s to load model into VRAM)"
                )
                self._has_logged_embedding = True

            resp = requests.post(
                url,
                json=payload,
                headers=headers,
                timeout=30,
            )
            if resp.status_code != 200:
                logger.debug(f"Embedding API Error {resp.status_code}: {resp.text}")
                resp.raise_for_status()

            data = resp.json()
            # Handle both Ollama and OpenAI-style embedding responses
            if "embedding" in data:
                return data["embedding"]
            elif "data" in data and len(data["data"]) > 0:
                return data["data"][0]["embedding"]
            return None
        except Exception as e:
            logger.error(f"Failed to generate embedding via {url}: {e}")
            raise

    def generate_uuid(self, seed_string: str) -> str:
        """
        Generates a valid UUID string required by Qdrant from any raw seed string.
        """
        import hashlib
        import uuid

        return str(uuid.UUID(hex=hashlib.sha256(seed_string.encode("utf-8")).hexdigest()[:32]))

    def upsert_point(self, seed_string: str, payload: dict, vector: list = None, log_success: str = None) -> bool:
        """
        Safely upserts a point into Qdrant, handling valid UUID generation.
        """
        if not self.is_connected:
            return False

        point_id = self.generate_uuid(seed_string)

        # If no vector provided, use a zero-vector if expected, though Qdrant requires vectors unless using empty vectors config.
        # Most of our DBs provide vectors, or we just pass a list of 0.0s.
        safe_vector = vector if vector is not None else [0.0] * self._vector_size

        try:
            self.client.upsert(
                collection_name=self.collection_name,
                points=[PointStruct(id=point_id, vector=safe_vector, payload=payload)],
                wait=True,
            )

            # ABSOLUTE LOGGING: User requirement for full observability
            msg = log_success or f"📥 [Qdrant] Learned new data in {self.collection_name}"
            logger.info(f"{msg} (UUID: {point_id[:8]}...)", extra={"color": "\x1b[35m"})
            self._handle_success()
            return True
        except Exception as e:
            self._handle_error(e, "Failed to upsert point")
            return False

    def delete_point(self, seed_string: str) -> bool:
        """
        Deletes a point from Qdrant autonomously based on its seed string.
        Used by the Unlearn mechanism to purge false-positive memories.
        """
        if not self.is_connected:
            return False

        point_id = self.generate_uuid(seed_string)
        try:
            res = self.client.retrieve(collection_name=self.collection_name, ids=[point_id])
            if res:
                self.client.delete(collection_name=self.collection_name, points_selector=[point_id])
                logger.info(
                    f"🗑️ [Qdrant] Purged poisoned memory vector from {self.collection_name} (UUID: {point_id[:8]}...)",
                    extra={"color": "\x1b[31m"},
                )
            return True
        except Exception as e:
            self._handle_error(e, f"Failed to delete point {point_id}")
            return False

    def search_points(self, vector: list, limit: int = 5, score_threshold: float = 0.5) -> list:
        """
        Performs vector search with absolute logging.
        """
        if not self.is_connected:
            return []

        try:
            results = self.client.search(
                collection_name=self.collection_name, query_vector=vector, limit=limit, score_threshold=score_threshold
            )
            logger.debug(f"🔍 [Qdrant] Search in {self.collection_name} returned {len(results)} matches.")
            self._handle_success()
            return results
        except Exception as e:
            self._handle_error(e, "Search failed")
            return []

    def get_collection_size(self) -> int:
        """Returns the number of points in the collection."""
        if not self.is_connected:
            return 0
        try:
            count_result = self.client.count(collection_name=self.collection_name)
            return count_result.count
        except Exception:
            return 0


class HeuristicMemoryDB(QdrantBase):
    """
    Stores successfully generated heuristics from the VLMCompilerEngine.
    This allows the ZeroLatencyEngine to pull pre-compiled Regex/XPath rules securely.
    """

    def __init__(self):
        super().__init__(collection_name="gramaddict_heuristics_v7")

    def cache_heuristic(self, intent_description: str, rule: dict):
        if not self.is_connected or not rule:
            return
        try:
            # We hash the intent description to create a deterministic PointID
            str(uuid.uuid5(uuid.NAMESPACE_DNS, intent_description))
            vector = self._get_embedding(intent_description)
            if not vector:
                return

            payload = {
                "intent": intent_description,
                "rule_type": rule.get("rule_type", "regex"),
                "target_attribute": rule.get("target_attribute", "text"),
                "pattern": rule.get("pattern", ""),
                "timestamp": time.time(),
                "confidence": 0.9,  # Initial confidence
            }

            self.upsert_point(intent_description, payload, vector=vector)

        except Exception as e:
            logger.error(f"Failed to cache generated heuristic: {e}")

    def fetch_heuristic(self, intent_description: str) -> Optional[dict]:
        if not self.is_connected:
            return None
        try:
            vector = self._get_embedding(intent_description)
            if not vector:
                return None

            results = self.client.query_points(
                collection_name=self.collection_name,
                query=vector,
                limit=1,
                score_threshold=0.98,  # Very strict match on intent description
            )
            if results and getattr(results, "points", None):
                payload = results.points[0].payload
                return {
                    "rule_type": payload.get("rule_type"),
                    "target_attribute": payload.get("target_attribute"),
                    "pattern": payload.get("pattern"),
                }
            logger.debug(f"fetch_heuristic: No points found matching {intent_description} above threshold.")
            return None
        except Exception as e:
            logger.debug(f"Failed to fetch heuristic for {intent_description}: {e}")
            return None


class UIMemoryDB(QdrantBase):
    """
    Hardened Qdrant-based UI memory for storing and retrieving learned element locations.

    Key design decisions:
    - Deterministic IDs: Hash of (intent + structural_signature) → same situation overwrites old entry
    - Confidence scoring: Entries gain/lose confidence based on success/failure feedback
    - No negative caching: NOT_FOUND results are NEVER stored (prevents memory poisoning)
    - Post-validation: Retrieved entries are checked for staleness
    """

    # Minimum confidence score to return a cached result
    MIN_CONFIDENCE = 0.3
    # Default confidence for new entries
    DEFAULT_CONFIDENCE = 0.7
    # Similarity threshold for vector lookup (Increased to reduce cross-intent confusion)
    SIMILARITY_THRESHOLD = 0.95

    def __init__(self):
        super().__init__(collection_name="gramaddict_ui_cache")

    def _create_structural_signature(self, xml_context: str) -> str:
        """
        Removes language-specific strings and non-structural attributes from the XML
        to create a pure, compact DOM topology string for the embedding model.
        """
        # 1. Remove large, high-frequency but non-structural attributes
        # We keep class and resource-id as core structural identifiers.
        # We remove bounds as they change with resolution/scaling and bloat the string.
        attributes_to_remove = [
            "text",
            "content-desc",
            "bounds",
            "checkable",
            "clickable",
            "focusable",
            "focused",
            "scrollable",
            "long-clickable",
            "password",
            "selected",
            "index",
            "package",
            "instance",
        ]

        sig = xml_context
        for attr in attributes_to_remove:
            sig = re.sub(rf'{attr}="[^"]*"', "", sig)

        # 2. Cleanup whitespace
        sig = re.sub(r"\s+", " ", sig).strip()

        # 3. Strict truncation for nomic-embed-text context window
        return sig[:2000]

    def _deterministic_id(self, intent: str) -> str:
        """
        Generate a deterministic point ID from intent.
        Same intent = same ID → deduplication via overwrite across all screens.
        """
        return hashlib.sha256(intent.encode("utf-8")).hexdigest()[:32]

    def retrieve_memory(
        self, intent: str, xml_context: str, similarity_threshold: float = None, exact_only: bool = False
    ) -> Optional[dict]:
        """
        Queries Qdrant for a known resolution representing the given intent.
        Returns the cached intent result (e.g. is_ad boolean, or bounding box) if found.
        """
        if not self.is_connected:
            return None

        # Lowered threshold slightly just in case nomic embedding drifts, but it should be 1.0 for exact intent
        if similarity_threshold is None:
            similarity_threshold = 0.85

        def _evaluate_payload(payload: dict, score: float = 1.0, point_id: str = None) -> Optional[dict]:
            solution = payload.get("solution")
            confidence = payload.get("confidence", self.DEFAULT_CONFIDENCE)

            # ── TTL-based confidence decay ──
            stored_at = payload.get("stored_at", 0)
            if stored_at > 0:
                import time as _time

                age_hours = (_time.time() - stored_at) / 3600
                time_decay = min(age_hours * 0.05, 0.4)  # Max 40% decay over 8 hours
                effective_confidence = confidence - time_decay
            else:
                effective_confidence = confidence

            # Filter 1: Never return NOT_FOUND/error entries from cache
            if isinstance(solution, dict) and solution.get("type") == "error":
                logger.debug(f"Purging poisoned NOT_FOUND entry for '{intent}' from Qdrant Memory.")
                if point_id:
                    self._delete_point(point_id)
                return None

            # Filter 2: Skip low-confidence entries
            if effective_confidence < self.MIN_CONFIDENCE:
                logger.debug(
                    f"Skipping expired/low-confidence ({effective_confidence:.2f}, raw: {confidence:.2f}) entry for '{intent}'."
                )
                if effective_confidence < 0.1 and point_id:
                    self._delete_point(point_id)
                return None

            return {"solution": solution, "effective_confidence": effective_confidence}

        # 0. Zero-cost deterministic ID exact match (bypasses LLM embedding API)
        try:
            point_id = self._deterministic_id(intent)
            exact_points = self.client.retrieve(
                collection_name=self.collection_name,
                ids=[point_id],
                with_payload=True,
                with_vectors=False,
            )
            if exact_points:
                eval_result = _evaluate_payload(exact_points[0].payload, score=1.0, point_id=point_id)
                if eval_result:
                    logger.info(
                        f"🧠 [Memory] Applying learned pattern for '{intent}' (EXACT MATCH, Confidence: {eval_result['effective_confidence']:.2f})",
                        extra={"color": "\x1b[36m"},  # Cyan color
                    )
                    return eval_result["solution"]
                # If exact match failed evaluation (e.g. decayed), we shouldn't fall back to vector search because it's the exact intent!
                return None
        except Exception as e:
            logger.debug(f"Qdrant exact match retrieval error: {e}")

        if exact_only:
            return None

        # 1. Fallback to vector similarity (requires embedding API call)
        # Embed ONLY the intent. We learn elements globally to drastically reduce LLM calls.
        text_to_embed = f"Intent: {intent}"
        vector = self._get_embedding(text_to_embed)

        if not vector:
            return None

        try:
            results = self.client.query_points(
                collection_name=self.collection_name,
                query=vector,
                limit=1,
            ).points

            if results and results[0].score >= similarity_threshold:
                eval_result = _evaluate_payload(results[0].payload, score=results[0].score, point_id=results[0].id)
                if eval_result:
                    logger.info(
                        f"🧠 [Memory] Applying learned pattern for '{intent}' (VECTOR MATCH, Score: {results[0].score:.3f}, Confidence: {eval_result['effective_confidence']:.2f})",
                        extra={"color": "\x1b[36m"},  # Cyan color
                    )
                    return eval_result["solution"]
                return None
            else:
                score = results[0].score if results else 0.0
                logger.debug(f"No high-confidence match found in Qdrant (Best score: {score:.3f})")
                return None
        except Exception as e:
            logger.debug(f"Qdrant retrieval error: {e}")
            return None

    def store_memory(self, intent: str, xml_context: str, solution: dict):
        """
        Saves a successful resolution for an intent into the Qdrant DB.
        """
        if not self.is_connected:
            return

        # GUARD: Never store negative/error results
        if isinstance(solution, dict):
            if solution.get("type") == "error" or solution.get("value") == "NOT_FOUND":
                logger.debug(f"Refusing to store NOT_FOUND result for '{intent}'. This prevents memory poisoning.")
                return

        text_to_embed = f"Intent: {intent}"
        vector = self._get_embedding(text_to_embed)

        if not vector:
            return

        try:
            # Deterministic ID: same intent = overwrite, not duplicate
            point_id = self._deterministic_id(intent)

            self.client.upsert(
                collection_name=self.collection_name,
                points=[
                    PointStruct(
                        id=point_id,
                        vector=vector,
                        payload={
                            "intent": intent,
                            "solution": solution,
                            "confidence": self.DEFAULT_CONFIDENCE,
                            "stored_at": time.time(),
                        },
                    )
                ],
                wait=True,
            )
            logger.info(
                f"📥 [Memory] Learned new pattern for '{intent}' and saved to Qdrant (ID: {point_id[:8]}...)",
                extra={"color": "\x1b[35m"},  # Magenta color
            )
        except Exception as e:
            logger.debug(f"Qdrant storage error: {e}")

    def boost_confidence(self, *args, **kwargs):
        """Called when an element from memory was successfully used."""
        intent = kwargs.get("intent") or (args[0] if args else None)
        amount = kwargs.get("amount", 0.15)
        if not intent:
            return
        try:
            amount = float(amount)
        except (ValueError, TypeError):
            amount = 0.15
        self._adjust_confidence(intent, amount)

    def decay_confidence(self, *args, **kwargs):
        """Called when an element from memory failed validation (not found on screen)."""
        intent = kwargs.get("intent") or (args[0] if args else None)
        amount = kwargs.get("amount", 0.50)  # Harder punishment: 2 failures = near-dead
        if not intent:
            return
        try:
            amount = float(amount)
        except (ValueError, TypeError):
            amount = 0.50
        self._adjust_confidence(intent, -amount)

    def _adjust_confidence(self, intent: str, delta: float):
        """Adjusts the confidence score of a cached entry using only its intent ID."""
        if not self.is_connected:
            return

        point_id = self._deterministic_id(intent)

        try:
            # Fetch current point
            points = self.client.retrieve(
                collection_name=self.collection_name,
                ids=[point_id],
                with_payload=True,
                with_vectors=False,
            )

            if not points:
                return

            current = points[0].payload
            new_confidence = max(0.0, min(1.0, current.get("confidence", self.DEFAULT_CONFIDENCE) + delta))

            # If confidence dropped below threshold, delete the entry entirely
            if new_confidence < 0.1:
                logger.info(f"Confidence for '{intent}' dropped to {new_confidence:.2f}. Removing stale entry.")
                self._delete_point(point_id)
                return

            # Update payload with new confidence
            self.client.set_payload(
                collection_name=self.collection_name,
                payload={"confidence": new_confidence},
                points=[point_id],
            )
            color = "\x1b[32m" if delta > 0 else "\x1b[31m"  # Green for positive, Red for negative
            symbol = "📈 [Memory] Positive Reinforcement:" if delta > 0 else "📉 [Memory] Negative Reinforcement:"
            logger.info(
                f"{symbol} Confidence for '{intent}' adjusted to {new_confidence:.2f} (delta: {delta:+.2f})",
                extra={"color": color},
            )
        except Exception as e:
            logger.debug(f"Confidence adjustment error: {e}")

    def _delete_point(self, point_id: str):
        """Deletes a single point from Qdrant by ID."""
        try:
            from qdrant_client.models import PointIdsList

            self.client.delete(
                collection_name=self.collection_name,
                points_selector=PointIdsList(points=[point_id]),
            )
        except Exception as e:
            logger.debug(f"Failed to delete point {point_id}: {e}")

    def purge_stale_entries(self):
        """Startup hygiene: remove low-confidence and expired entries."""
        if not self.is_connected:
            return
        try:
            points, _ = self.client.scroll(
                collection_name=self.collection_name,
                limit=500,
                with_payload=True,
            )
            purged = 0
            import time as _time

            now = _time.time()
            for pt in points:
                payload = pt.payload or {}
                confidence = payload.get("confidence", self.DEFAULT_CONFIDENCE)
                stored_at = payload.get("stored_at", 0)
                age_hours = (now - stored_at) / 3600 if stored_at > 0 else 0

                # Purge if: confidence dropped below 0.5, or older than 24 hours
                if confidence < 0.5 or age_hours > 24:
                    self._delete_point(pt.id)
                    purged += 1
            if purged:
                logger.info(f"🧹 Startup purge: Removed {purged} stale/expired Qdrant entries.")
        except Exception as e:
            logger.debug(f"Startup purge error (non-fatal): {e}")


class CommentMemoryDB(QdrantBase):
    def __init__(self):
        super().__init__(collection_name="gramaddict_learned_comments")

    def store_comment(self, text: str, vibe: str, author: str = "unknown"):
        """Saves a high-quality learned comment."""
        if not self.is_connected:
            return

        vector = self._get_embedding(text)
        if not vector:
            return

        self.upsert_point(
            seed_string=str(time.time()),
            vector=vector,
            payload={"text": text, "vibe": vibe, "author": author},
            log_success=f"🧠 [CommentMemory] Learned and stored comment: '{text[:30]}...' (Vibe: {vibe})",
        )

    def get_relevant_comments(self, context_text: str, limit: int = 3) -> list[str]:
        """Retrieves similar comments from memory to use as examples."""
        if not self.is_connected:
            return []

        vector = self._get_embedding(context_text)
        if not vector:
            return []

        try:
            results = self.client.query_points(
                collection_name=self.collection_name,
                query=vector,
                limit=limit,
            ).points
            return [hit.payload.get("text") for hit in results if hit.score > 0.7]
        except Exception as e:
            logger.debug(f"Comment retrieval error: {e}")
            return []


class ContentMemoryDB(QdrantBase):
    def __init__(self):
        super().__init__(collection_name="gramaddict_content_memory")

    def store_evaluation(self, description: str, classification: str, reason: str):
        """Saves an AI-evaluated description to memory."""
        if not self.is_connected or not description:
            return

        vector = self._get_embedding(description)
        if not vector:
            return

        self.upsert_point(
            seed_string=description,
            vector=vector,
            payload={
                "description": description,
                "classification": classification,
                "reason": reason,
                "stored_at": time.time(),
            },
            log_success=f"🧠 [ContentMemory] Stored evaluation in memory: {classification} ({reason})",
        )

    def get_cached_evaluation(self, description: str, similarity_threshold: float = 0.95) -> Optional[dict]:
        """Queries for a nearly identical past post to act as a cache."""
        if not self.is_connected or not description:
            return None

        vector = self._get_embedding(description)
        if not vector:
            return None

        try:
            results = self.client.query_points(
                collection_name=self.collection_name,
                query=vector,
                limit=1,
            ).points

            if results and results[0].score >= similarity_threshold:
                payload = results[0].payload
                logger.info(
                    f"Qdrant Cache Hit! Similar post found (score: {results[0].score:.3f}). Classification: {payload.get('classification')}"
                )
                return payload
            return None
        except Exception as e:
            logger.debug(f"Content memory retrieval error: {e}")
            return None

    def get_similar_examples(self, description: str, limit: int = 3) -> list[dict]:
        """Retrieves similar past post evaluations to use in RAG few-shot prompting."""
        if not self.is_connected or not description:
            return []

        vector = self._get_embedding(description)
        if not vector:
            return []

        try:
            results = self.client.query_points(
                collection_name=self.collection_name,
                query=vector,
                limit=limit,
            ).points

            examples = []
            for hit in results:
                payload = hit.payload
                examples.append(
                    {
                        "description": payload.get("description", "")[:200],
                        "classification": payload.get("classification"),
                        "reason": payload.get("reason"),
                    }
                )
            return examples
        except Exception as e:
            logger.debug(f"Content RAG retrieval error: {e}")
            return []


class ScreenMemoryDB(QdrantBase):
    """
    Learns and caches structural screen classifications mapping (XML Signature -> ScreenType).
    Replaces hardcoded string checks in GOAP.
    """

    def __init__(self):
        super().__init__(collection_name="gramaddict_screen_types_v1")

    def store_screen(self, xml_signature: str, screen_type: str):
        if not self.is_connected or not xml_signature:
            return

        vector = self._get_embedding(xml_signature)
        if not vector:
            return

        self.upsert_point(
            seed_string=xml_signature,
            vector=vector,
            payload={
                "signature": xml_signature[:500],
                "screen_type": screen_type,
                "stored_at": time.time(),
            },
            log_success=f"🧠 [ScreenMemory] Learned new layout mapping: {screen_type}",
        )

    def get_screen_type(self, xml_signature: str, similarity_threshold: float = 0.90) -> Optional[str]:
        if not self.is_connected or not xml_signature:
            return None

        vector = self._get_embedding(xml_signature)
        if not vector:
            return None

        try:
            results = self.client.query_points(
                collection_name=self.collection_name,
                query=vector,
                limit=1,
            ).points

            if results and results[0].score >= similarity_threshold:
                payload = results[0].payload
                logger.info(
                    f"🧠 [ScreenMemory] Cache Hit! Screen recognized as: {payload.get('screen_type')} (Score: {results[0].score:.2f})"
                )
                return payload.get("screen_type")
            return None
        except Exception as e:
            logger.debug(f"Screen memory error: {e}")
            return None

    def purge_screen(self, xml_signature: str):
        """Unlearns a screen classification to recover from hallucinations."""
        if not self.is_connected or not xml_signature:
            return
        self.delete_point(xml_signature)


class NavigationMemoryDB(QdrantBase):
    """
    Topological Navigation Persistence.
    Stores learned paths and transitions discovered during autonomous exploration.
    This eliminates "Context Stagnation" by sharing learned navigation across the fleet.
    """

    def __init__(self):
        # We use a smaller vector size for navigation tags or just zero-vectors for KV storage
        super().__init__(collection_name="gramaddict_nav_graph_v8", vector_size=128)

    def store_transition(self, from_state: str, action: str, to_state: str):
        key = f"{from_state}_{action}"
        payload = {"from": from_state, "action": action, "to": to_state, "timestamp": time.time()}
        self.upsert_point(
            key, payload, log_success=f"🧠 [NavBrain] Learned transition: {from_state} --({action})--> {to_state}"
        )

    def unlearn_transition(self, from_state: str, action: str):
        """Unlearns a topological transition if it is proven to be a dead-end."""
        key = f"{from_state}_{action}"
        self.delete_point(key)

    def get_all_transitions(self) -> dict:
        """Retrieves the full topological map from Qdrant."""
        if not self.is_connected:
            return {}
        try:
            resp = self.client.scroll(collection_name=self.collection_name, limit=100)
            points = resp[0]
            nodes = {}
            for p in points:
                data = p.payload
                f, a, t = data["from"], data["action"], data["to"]
                if f not in nodes:
                    nodes[f] = {"transitions": {}}
                nodes[f]["transitions"][a] = t
            return nodes
        except Exception as e:
            logger.error(f"Failed to fetch nav graph from Qdrant: {e}")
            return {}


class PersonaMemoryDB(QdrantBase):
    def __init__(self):
        super().__init__(collection_name="gramaddict_persona_memory")

    def store_persona_insight(self, category: str, insight: str):
        """Saves a learned insight about the logged-in user's brand persona."""
        if not self.is_connected or not insight:
            return

        vector = self._get_embedding(insight)
        if not vector:
            return

        self.upsert_point(
            seed_string=category + insight,
            vector=vector,
            payload={
                "category": category,
                "insight": insight,
                "stored_at": time.time(),
            },
            log_success=f"🧠 [PersonaMemory] Learned Brand Persona ({category}): {insight[:50]}...",
        )

    def get_persona_context(self, category: str = None, limit: int = 5) -> str:
        """Retrieves and formats learned persona guidelines."""
        if not self.is_connected:
            return ""

        try:
            query_filter = None
            if category:
                query_filter = Filter(must=[FieldCondition(key="category", match=MatchValue(value=category))])

            # Use scroll to just fetch the most recent or all relevant insights since we don't have a semantic query to match against.
            # We just want the general guidelines for the prompt.
            points, _ = self.client.scroll(
                collection_name=self.collection_name,
                scroll_filter=query_filter,
                limit=limit,
                with_payload=True,
            )

            if not points:
                return ""

            contexts = []
            for pt in points:
                payload = pt.payload
                cat = payload.get("category", "General")
                ins = payload.get("insight", "")
                contexts.append(f"[{cat}] {ins}")

            return "\n".join(contexts)
        except Exception as e:
            logger.debug(f"Persona retrieval error: {e}")
            return ""


class BannedPathsDB:
    """
    Negative feedback memory: stores which specific elements FAILED
    for which goals. Simple dict-based storage (no vectors needed).

    Key insight: This is NOT a vector similarity problem. The relationship
    is exact: "goal X + element Y = failure". So we use a simple hash map
    stored in Qdrant as payload-only points.
    """

    # Max age for banned entries (7 days) — Instagram updates can change layouts
    MAX_AGE_SECONDS = 7 * 24 * 3600

    def __init__(self):
        if QdrantBase._connection_failed_logged:
            self._client = None
            return

        self._banned = {}  # {goal_hash: set(element_ids)}
        self._client = None
        self._collection = "gramaddict_banned_paths"

        try:
            qdrant_url = os.environ.get("QDRANT_URL", "http://localhost:6344")
            self._client = QdrantClient(url=qdrant_url, timeout=5.0)

            # Create collection with a dummy 1-dim vector (Qdrant requires vectors)
            if self._client and not self._client.collection_exists(self._collection):
                self._client.create_collection(
                    collection_name=self._collection,
                    vectors_config=VectorParams(size=4, distance=Distance.COSINE),
                )

            # Load all banned entries into memory at startup
            self._load_all()
        except Exception as e:
            logger.debug(f"BannedPaths init (non-fatal): {e}")

    def _goal_hash(self, goal: str) -> str:
        return hashlib.sha256(goal.lower().strip().encode("utf-8")).hexdigest()[:16]

    def _point_id(self, goal: str, element_id: str) -> str:
        combined = f"{goal.lower().strip()}||{element_id}"
        import uuid

        return str(uuid.UUID(hex=hashlib.sha256(combined.encode("utf-8")).hexdigest()[:32]))

    def _load_all(self):
        """Load all banned paths from Qdrant into memory at startup."""
        if not self._client:
            return
        try:
            points, _ = self._client.scroll(
                collection_name=self._collection,
                limit=500,
                with_payload=True,
            )
            now = time.time()
            expired = []
            for pt in points:
                payload = pt.payload or {}
                # Expire old entries
                if now - payload.get("banned_at", 0) > self.MAX_AGE_SECONDS:
                    expired.append(pt.id)
                    continue
                gh = payload.get("goal_hash", "")
                eid = payload.get("element_id", "")
                if gh and eid:
                    if gh not in self._banned:
                        self._banned[gh] = set()
                    self._banned[gh].add(eid)

            # Clean expired
            if expired:
                from qdrant_client.models import PointIdsList

                self._client.delete(
                    collection_name=self._collection,
                    points_selector=PointIdsList(points=expired),
                )
                logger.debug(f"BannedPaths: Expired {len(expired)} old entries.")

            total = sum(len(v) for v in self._banned.values())
            if total > 0:
                logger.info(f"BannedPaths: Loaded {total} banned element-goal pairs.")
        except Exception as e:
            logger.debug(f"BannedPaths load error (non-fatal): {e}")

    def ban(self, goal: str, element_id: str, reason: str = ""):
        """Ban a specific element for a specific goal."""
        if not element_id:
            return

        gh = self._goal_hash(goal)
        if gh not in self._banned:
            self._banned[gh] = set()
        self._banned[gh].add(element_id)

        logger.warning(f"BannedPaths: Banned '{element_id}' for goal '{goal[:60]}'. Reason: {reason}")

        # Persist to Qdrant
        if self._client:
            try:
                pid = self._point_id(goal, element_id)
                self._client.upsert(
                    collection_name=self._collection,
                    points=[
                        PointStruct(
                            id=pid,
                            vector=[0.0, 0.0, 0.0, 0.0],  # dummy vector
                            payload={
                                "goal": goal[:200],
                                "goal_hash": gh,
                                "element_id": element_id,
                                "reason": reason[:200],
                                "banned_at": time.time(),
                            },
                        )
                    ],
                )
            except Exception as e:
                logger.debug(f"BannedPaths persist error: {e}")

    def is_banned(self, goal: str, element_id: str) -> bool:
        """Check if a specific element is banned for a specific goal."""
        if not element_id:
            return False
        gh = self._goal_hash(goal)
        return element_id in self._banned.get(gh, set())

    def get_banned_ids(self, goal: str) -> set:
        """Get all banned element IDs for a specific goal."""
        gh = self._goal_hash(goal)
        return self._banned.get(gh, set()).copy()


class DMMemoryDB(QdrantBase):
    def __init__(self):
        super().__init__(collection_name="gramaddict_dm_memory")

    def log_sent_dm(self, target_username: str, message: str, biography: str, recent_descriptions: list):
        if not self.is_connected:
            return

        vector = self._get_embedding(message)
        if not vector:
            return

        self.upsert_point(
            seed_string=f"{target_username}_{message}_{time.time()}",
            vector=vector,
            payload={
                "target_username": target_username,
                "message": message,
                "biography": biography,
                "recent_descriptions": recent_descriptions,
                "status": "pending",
                "score": 0.0,
                "timestamp": time.time(),
            },
            log_success="🧠 [Growth Brain] Logged sent DM to memory for future response tracking.",
        )

    def get_pending_dms(self, limit=10):
        if not self.is_connected:
            return []
        try:
            points = self.client.query_points(
                collection_name=self.collection_name,
                query=self._get_embedding(""),
                query_filter=Filter(must=[FieldCondition(key="status", match=MatchValue(value="pending"))]),
                limit=limit,
            ).points

            return [p.payload for p in points]
        except Exception as e:
            logger.debug(f"[DMMemory] get_pending_dms failed: {e}")
            return []

    def update_dm_score(self, point_id: str, new_score: float, status: str = "evaluated"):
        if not self.is_connected:
            return
        try:
            self.client.set_payload(
                collection_name=self.collection_name, payload={"status": status, "score": new_score}, points=[point_id]
            )
            logger.info(f"🧠 [Growth Brain] Updated DM Resonance Score: {new_score:.1f}")
        except Exception as e:
            logger.debug(f"Failed to update DM score: {e}")

    def get_best_performing_dms(self, limit=3):
        if not self.is_connected:
            return []
        try:
            results = self.client.scroll(
                collection_name=self.collection_name,
                scroll_filter=Filter(must=[FieldCondition(key="status", match=MatchValue(value="evaluated"))]),
                limit=100,
                with_payload=True,
            )[0]

            dms = [r.payload["message"] for r in results if r.payload.get("score", 0.0) >= 0.8]
            dms = list(set(dms))
            return dms[:limit]
        except Exception as e:
            logger.debug(f"Failed to fetch best DMs: {e}")
            return []


class ParasocialCRMDB(QdrantBase):
    collection_name = "ig_parasocial_crm"

    def __init__(self):
        super().__init__(self.collection_name)

    def get_relationship_stage(self, username: str) -> dict:
        """
        Returns the current CRM stage and last interaction timestamp.
        Stages: 0=Awareness, 1=Curiosity, 2=Rapport, 3=Conversion

        Uses exact-match filter (not embedding) — usernames are deterministic keys,
        not semantic content. This is ~100x faster than embedding search.
        """
        if not self.is_connected:
            return {"stage": 0, "last_interaction": 0.0, "interactions": []}

        try:
            points, _ = self.client.scroll(
                collection_name=self.collection_name,
                scroll_filter=Filter(must=[FieldCondition(key="username", match=MatchValue(value=username))]),
                limit=1,
                with_payload=True,
            )

            if points:
                return points[0].payload
        except Exception as e:
            logger.debug(f"[ParasocialCRM] query failed: {e}")

        return {"stage": 0, "last_interaction": 0.0, "interactions": []}

    def log_interaction(self, username: str, intent_type: str, new_stage: int = None):
        """
        Logs an interaction (e.g. 'story_view', 'deep_like', 'comment_reply').
        Upgrades stage automatically if new_stage is provided.
        """
        if not self.is_connected:
            return

        current = self.get_relationship_stage(username)
        stage = new_stage if new_stage is not None else current["stage"]

        interactions = current.get("interactions", [])
        import time

        now = time.time()

        if interactions:
            last_interaction = interactions[-1]
            if last_interaction.get("type") == intent_type and (now - last_interaction.get("timestamp", 0)) < 300:
                logger.debug(f"🧠 [ParasocialCRM] Skipping redundant '{intent_type}' write for @{username}.")
                return

        interactions.append({"type": intent_type, "timestamp": now})

        payload = {
            "username": username,
            "stage": stage,
            "last_interaction": now,
            "interactions": interactions[-10:],  # keep last 10
        }

        vector = self._get_embedding(f"User: {username}")
        if vector:
            self.upsert_point(
                seed_string=f"User_{username}",
                vector=vector,
                payload=payload,
                log_success=f"🧠 [ParasocialCRM] Updated @{username} into Qdrant. Stage {stage} ({intent_type})",
            )

    def enrich_lead(self, username: str, data: dict):
        """
        Enriches a lead with scraped data.
        """
        if not self.is_connected:
            return

        current = self.get_relationship_stage(username)
        current.update(data)

        vector = self._get_embedding(f"User: {username}")
        if vector:
            self.upsert_point(
                seed_string=f"User_{username}",
                vector=vector,
                payload=current,
                log_success=f"🧠 [ParasocialCRM] Enriched @{username} data.",
            )

    def log_generated_comment(self, username: str, comment_text: str):
        """Phase 10: RAG memory point for specific users."""
        if not self.is_connected:
            return

        current = self.get_relationship_stage(username)
        interactions = current.get("interactions", [])

        interactions.append({"type": "comment_sent", "text": comment_text, "timestamp": time.time()})

        payload = {
            "username": username,
            "stage": current["stage"],
            "last_interaction": time.time(),
            "interactions": interactions[-20:],  # Keep last 20 interactions
            "bio": current.get("bio", ""),
        }

        vector = self._get_embedding(f"User: {username}")
        if vector:
            self.upsert_point(
                seed_string=f"User_{username}",
                vector=vector,
                payload=payload,
                log_success=f"🧠 [ParasocialCRM] Logged generated comment for @{username} into Qdrant.",
            )

    def log_profile_context(self, username: str, bio: str):
        """Phase 10: Store parsed user bio for hyper-personalization."""
        if not self.is_connected or not username:
            return

        current = self.get_relationship_stage(username)
        # Avoid unnecessary DB writes if the bio hasn't fundamentally changed
        if current.get("bio") == bio:
            return

        payload = {
            "username": username,
            "stage": current["stage"],
            "last_interaction": current["last_interaction"],
            "interactions": current.get("interactions", []),
            "bio": bio,
        }

        vector = self._get_embedding(f"User: {username}")
        if vector:
            self.upsert_point(
                seed_string=f"User_{username}",
                vector=vector,
                payload=payload,
                log_success=f"🧠 [ParasocialCRM] Learned profile context for @{username} (Bio: {bio[:30]}...)",
            )

    def get_conversation_context(self, username: str) -> str:
        """Phase 10: Constructs semantic context from prior comments with this user."""
        current = self.get_relationship_stage(username)
        interactions = current.get("interactions", [])

        context_parts = []
        if current.get("bio"):
            context_parts.append(f"User Bio: {current['bio']}")

        comments = [i["text"] for i in interactions if i.get("type") == "comment_sent" and "text" in i]
        if comments:
            context_parts.append("Previous Comments: " + " | ".join(comments))

        return "\n".join(context_parts)


def wipe_all_ai_caches():
    """
    Wipes ALL global (non-user-specific) Qdrant AI caches.
    Used by blank_start to ensure a completely fresh learning state.
    Does NOT wipe user-specific collections like PathMemory or ParasocialCRM.
    """
    global_dbs = [
        HeuristicMemoryDB,
        UIMemoryDB,
        CommentMemoryDB,
        ContentMemoryDB,
        ScreenMemoryDB,
        NavigationMemoryDB,
        PersonaMemoryDB,
        DMMemoryDB,
    ]
    wiped_count = 0
    for db_cls in global_dbs:
        try:
            db = db_cls()
            if db.is_connected:
                db.wipe_collection()
                wiped_count += 1
        except Exception as e:
            logger.warning(f"⚠️ Failed to wipe {db_cls.__name__}: {e}")

    logger.info(f"🗑️ [Blank Start] Wiped {wiped_count}/{len(global_dbs)} global AI caches.")
