mmintel/instagram-bot
1
0
Fork 0
Code Issues Pull Requests 4 Actions Packages Projects Releases Wiki Activity

feat: complete modular plugin refactor with 100% E2E coverage for interactions

Browse Source
This commit is contained in:
Marc Mintel
2026-04-25 20:58:07 +02:00
parent 77e8251aa7
commit 144d6401b5
232 changed files with 66259 additions and 5410 deletions
Download Patch File Download Diff File Expand all files Collapse all files

114
benchmarks/ai_memory_diagnostics.py
View File

@@ -1,25 +1,32 @@
import json
import os
import sys
import json
import time
# Root path alignment
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.append(ROOT_DIR)
def colored(text, color, attrs=None):
colors = {
"red": "\033[91m", "green": "\033[92m", "yellow": "\033[93m",
"blue": "\033[94m", "magenta": "\033[95m", "cyan": "\033[96m",
"white": "\033[97m"
"red": "\033[91m",
"green": "\033[92m",
"yellow": "\033[93m",
"blue": "\033[94m",
"magenta": "\033[95m",
"cyan": "\033[96m",
"white": "\033[97m",
}
reset = "\033[0m"
bold = "\033[1m" if attrs and "bold" in attrs else ""
return f"{bold}{colors.get(color, '')}{text}{reset}"
from GramAddict.core.config import Config
from GramAddict.core.qdrant_memory import ParasocialCRMDB, CommentMemoryDB
from GramAddict.core.qdrant_memory import CommentMemoryDB, ParasocialCRMDB
from GramAddict.core.resonance_engine import ResonanceEngine
import xml.etree.ElementTree as ET
class MockArgs:
def __init__(self):
@@ -31,10 +38,12 @@ class MockArgs:
self.ai_vision_navigation = False
self.ai_vision_context = False
class MockConfig:
def __init__(self):
self.args = MockArgs()
class AIMemoryDiagnosticRunner:
def __init__(self):
self.configs = MockConfig()
@@ -42,7 +51,7 @@ class AIMemoryDiagnosticRunner:
self.crm_db = ParasocialCRMDB()
self.comment_db = CommentMemoryDB()
self.resonance_oracle = ResonanceEngine("benchmark_agent", crm=self.crm_db)
def setup(self):
print(colored("🧹 Initializing benchmark data...", "cyan"))
# We handle unique targets so we don't wipe the DB
@@ -56,19 +65,24 @@ class AIMemoryDiagnosticRunner:
fixture_path = os.path.join(ROOT_DIR, "tests", "fixtures", "comments_mock.xml")
with open(fixture_path, "r", encoding="utf-8") as f:
xml_data = f.read()
print(colored(f" -> Extracing comments using RAG Condenser ({self.configs.args.ai_condenser_model})...", "yellow"))
print(
colored(
f" -> Extracing comments using RAG Condenser ({self.configs.args.ai_condenser_model})...", "yellow"
)
)
start = time.time()
# Intercept the database write to bypass Qdrant indexing limits and solely test RAG filter logic
intercepted_comments = []
def mock_log(self, text: str, vibe: str, author: str = "unknown"):
intercepted_comments.append(text)
try:
from unittest.mock import patch
with patch.object(CommentMemoryDB, 'store_comment', new=mock_log):
with patch.object(CommentMemoryDB, "store_comment", new=mock_log):
# Override the author logic
test_author = f"benchmark_source_{int(time.time())}"
self.resonance_oracle.extract_and_learn_comments(xml_data, self.configs, author=test_author)
@@ -76,26 +90,41 @@ class AIMemoryDiagnosticRunner:
except Exception as e:
print(f"❌ EXCEPTION: {e}")
return {"passed": False, "reason": str(e)}
try:
learned_texts = [c.lower() for c in intercepted_comments]
dur = time.time() - start
print(colored(f" -> Intercepted: {learned_texts}", "yellow"))
toxic_count = sum(1 for t in learned_texts if "onlyfans" in t or "bitcoin" in t or "dm" in t or "$" in t)
good_count = sum(1 for t in learned_texts if "majestic" in t or "lighting" in t)
if toxic_count > 0:
print(colored(" ❌ [Sub-Test] LLM Condenser hallucinated or failed to block toxic queries (OnlyFans/Crypto).", "red"))
print(
colored(
" ❌ [Sub-Test] LLM Condenser hallucinated or failed to block toxic queries (OnlyFans/Crypto).",
"red",
)
)
return {"passed": False, "reason": "Toxic comments leaked"}
if good_count == 0:
print(colored(" ❌ [Sub-Test] LLM Condenser stripped everything or crashed. No good comments persisted.", "red"))
print(
colored(
" ❌ [Sub-Test] LLM Condenser stripped everything or crashed. No good comments persisted.",
"red",
)
)
return {"passed": False, "reason": "Good comments dropped"}
print(colored(f" ✅ [Sub-Test] RAG Filter passed! 0 toxic comments, {good_count} valid comments mapped. Latency {dur:.2f}s", "green"))
print(
colored(
f" ✅ [Sub-Test] RAG Filter passed! 0 toxic comments, {good_count} valid comments mapped. Latency {dur:.2f}s",
"green",
)
)
return {"passed": True, "reason": "Toxic filtered, good preserved."}
except Exception as e:
return {"passed": False, "reason": f"DB Error: {e}"}
@@ -105,18 +134,18 @@ class AIMemoryDiagnosticRunner:
"""
target = "benchmark_target"
context_string = "234 Posts | 1.2M Followers | 🏔️ Alpine Photographer | Link in bio"
try:
self.crm_db.log_profile_context(target, context_string)
time.sleep(0.5) # indexing buffer
time.sleep(0.5) # indexing buffer
history = self.crm_db.get_conversation_context(target)
if context_string in history or "1.2M Followers" in history:
print(colored(" ✅ [Sub-Test] Profile context cleanly injected into RAG CRM payload.", "green"))
return {"passed": True, "reason": "Context string found."}
else:
return {"passed": False, "reason": "Profile context missing from CRM retrieval."}
except Exception as e:
return {"passed": False, "reason": str(e)}
@@ -136,61 +165,60 @@ class AIMemoryDiagnosticRunner:
self.crm_db.log_generated_comment(target, "Wow great photo!")
self.crm_db.log_interaction(target, "tap_comment_button", new_stage=3)
time.sleep(0.5)
stage_info = self.crm_db.get_relationship_stage(target)
stage = stage_info.get("stage", 0)
if stage >= 3:
print(colored(f" ✅ [Sub-Test] CRM safely advanced state memory to Stage {stage}.", "green"))
return {"passed": True, "reason": "Evolution logic passed."}
else:
print(colored(f" ❌ [Sub-Test] CRM stalled at Stage {stage}!", "red"))
return {"passed": False, "reason": "Failed to evolve stage"}
except Exception as e:
return {"passed": False, "reason": str(e)}
def execute_all(self):
self.setup()
results = {
"timestamp": time.time(),
"model": self.configs.args.ai_condenser_model,
"scenarios": {}
}
results = {"timestamp": time.time(), "model": self.configs.args.ai_condenser_model, "scenarios": {}}
def run_and_log(name, func):
print(colored(f"\n--- SCENARIO: {name} ---", "magenta"))
start_time = time.time()
data = {"passed": False, "reason": "Unknown error", "latency_ms": 0}
try:
res = func()
if isinstance(res, dict): data.update(res)
elif res is True: data["passed"] = True
if isinstance(res, dict):
data.update(res)
elif res is True:
data["passed"] = True
except Exception as e:
print(colored(f"❌ EXCEPTION: {e}", "red"))
data["reason"] = str(e)
dur = time.time() - start_time
data["latency_ms"] = int(dur * 1000)
results["scenarios"][name] = data
if data["passed"]:
print(colored(f"🏁 {name} completed successfully in {dur:.2f}s", "green"))
else:
print(colored(f"🚨 {name} FAILED! (Elapsed: {dur:.2f}s)", "red", attrs=["bold"]))
print(colored(f" Reason: {data['reason']}", "yellow"))
run_and_log("RAG Comment Blacklist Extraction", self.test_rag_comment_extraction)
run_and_log("CRM Profile Context Injection", self.test_crm_profile_context)
run_and_log("CRM Sequential Evolution", self.test_crm_interaction_evolution)
self.setup() # Teardown
self.setup() # Teardown
out_path = os.path.join(ROOT_DIR, "benchmarks", "data", "ai_memory_results.json")
with open(out_path, "w") as f:
json.dump(results, f, indent=4)
print(colored(f"\n📄 Saved AI Memory Benchmark results to: {out_path}", "cyan", attrs=["bold"]))
if __name__ == "__main__":
runner = AIMemoryDiagnosticRunner()
runner.execute_all()

88
benchmarks/live_brain_diagnostics.py
View File

@@ -1,8 +1,9 @@
import json
import logging
import os
import sys
import time
import logging
import json
from colorama import Fore, Style, init
# Init Colorama for cross-platform color support
@@ -12,8 +13,8 @@ init(autoreset=True)
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, ROOT_DIR)
from GramAddict.core.telepathic_engine import TelepathicEngine
from GramAddict.core.qdrant_memory import UIMemoryDB
from GramAddict.core.telepathic_engine import TelepathicEngine
# Mute noisy loggers
logging.getLogger("requests").setLevel(logging.WARNING)
@@ -21,6 +22,7 @@ logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
logger = logging.getLogger(__name__)
def colored(text, color, attrs=None):
c = getattr(Fore, color.upper(), "")
attr_str = ""
@@ -28,6 +30,7 @@ def colored(text, color, attrs=None):
attr_str = Style.BRIGHT
return f"{attr_str}{c}{text}"
class MockArgs:
def __init__(self):
self.ai_telepathic_model = "qwen3.5:latest"
@@ -37,46 +40,55 @@ class MockArgs:
self.ai_vision_navigation = True
self.ai_vision_context = True
import base64
class MockDevice:
def __init__(self):
self.args = MockArgs()
self.app_id = "com.instagram.android"
def screenshot(self):
# Return a simple 1x1 black pixel PNG to test the True Vision payload mapping
# without crashing on invalid image data.
return base64.b64decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAADsMAAA7DAcdvqGQAAAAXSURBVBhXY3jP4PgfAAWEAziO3O8MAAAAASUVORK5CYII=")
return base64.b64decode(
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAADsMAAA7DAcdvqGQAAAAXSURBVBhXY3jP4PgfAAWEAziO3O8MAAAAASUVORK5CYII="
)
from GramAddict.core.config import Config
Config().args = MockArgs()
class BrainDiagnosticRunner:
"""
Professional diagnostic suite for Live integration testing of the
Singularity LLM Cognitive Stack and Vector DB (Qdrant) persistence.
Tested against heavy real-world XML dumps from Instagram.
"""
def __init__(self):
self.device = MockDevice()
self.engine = TelepathicEngine.get_instance()
self.mem_db = UIMemoryDB()
# Test Namespaces
self.intents = {
"modal": "diagnostics_dismiss_obstacle",
"ad": "diagnostics_find_sponsored",
"hallucination": "diagnostics_tap_like_button",
"unfollow": "diagnostics_tap_following_button"
"unfollow": "diagnostics_tap_following_button",
}
# Load heavy real-world XML files
self.fixtures_dir = os.path.join(ROOT_DIR, "tests", "fixtures")
self.xmls = {
"modal": self._load_fixture("blocked_ui.xml"),
"ad": self._load_fixture("peugeot_ad.xml"),
"hallucination": self._load_fixture("vlm_hallucination.xml"),
"unfollow": self._load_fixture("unfollow_list_dump.xml")
"unfollow": self._load_fixture("unfollow_list_dump.xml"),
}
def _load_fixture(self, filename) -> str:
@@ -91,7 +103,7 @@ class BrainDiagnosticRunner:
if not self.mem_db.is_connected:
logger.error("❌ Qdrant is offline! Diagnostics cannot proceed.")
sys.exit(1)
print(colored("🧹 Initializing diagnostic namespace (clearing old cache)...", "yellow"))
for intent in self.intents.values():
pt_id = self.mem_db._deterministic_id(intent)
@@ -111,20 +123,20 @@ class BrainDiagnosticRunner:
xml = self.xmls["modal"]
intent = self.intents["modal"]
node = self.engine.find_best_node(xml, intent, min_confidence=0.8, device=self.device)
if not node:
print(colored(" ❌ LLM failed to find the dismiss button entirely.", "red"))
return {"passed": False, "reason": "No node found"}
semantic = str(node.get("semantic", "")).lower()
if "try again later" in semantic or "action block" in semantic:
print(colored(" ❌ LLM selected the title text instead of the dismiss button.", "red"))
return {"passed": False, "reason": "Selected title instead of button"}
if "dismiss" in semantic or "ok" in semantic:
print(colored(f" ✅ VLM correctly reasoned the popup OK/Dismiss button: {semantic}", "green"))
return {"passed": True, "reason": f"Found correct button: {semantic}"}
return {"passed": False, "reason": f"Selected unrelated element: {semantic}"}
def test_ad_deception(self) -> dict:
@@ -134,20 +146,21 @@ class BrainDiagnosticRunner:
xml = self.xmls["ad"]
intent = self.intents["ad"]
node = self.engine.find_best_node(xml, intent, min_confidence=0.8, device=self.device)
if not node:
print(colored(" ❌ LLM failed to identify the sponsored indicator.", "red"))
return {"passed": False, "reason": "Missed sponsored text"}
semantic = str(node.get("semantic", "")).lower()
if "sponsored" in semantic:
print(colored(" ✅ VLM correctly identified the tiny 'Sponsored' label amidst a huge post.", "green"))
# --- Test Fast Path Recall Sub-Scenario ---
# Save it
self.engine.confirm_click(intent)
self.mem_db.store_memory(intent, xml, node)
import time
time.sleep(0.5)
# Try to grab it again
start = time.time()
@@ -159,7 +172,7 @@ class BrainDiagnosticRunner:
else:
print(colored(" ❌ [Sub-Test] Memory recall failed.", "red"))
return {"passed": False, "reason": "Found ad, but memory persistence failed."}
return {"passed": False, "reason": f"Picked wrong node: {semantic}"}
def test_vlm_hallucination(self) -> dict:
@@ -169,63 +182,66 @@ class BrainDiagnosticRunner:
xml = self.xmls["hallucination"]
intent = self.intents["hallucination"]
node = self.engine.find_best_node(xml, intent, min_confidence=0.8, device=self.device)
if not node:
print(colored(" ❌ LLM failed to find any like button.", "red"))
return {"passed": False, "reason": "No node found"}
semantic = str(node.get("semantic", "")).lower()
is_caption = ("double tap" in semantic or "like" in semantic) and "row feed button" not in semantic
if is_caption:
print(colored(" ❌ LLM fell for the semantic hallucination gap and selected the text caption!", "red"))
return {"passed": False, "reason": "Fell for caption text trap"}
if "row feed button like" in semantic or "heart" in semantic:
print(colored(" ✅ VLM successfully ignored the deceptive caption and found the structural like button.", "green"))
print(
colored(
" ✅ VLM successfully ignored the deceptive caption and found the structural like button.", "green"
)
)
return {"passed": True, "reason": "Ignored text trap, clicked structural button"}
return {"passed": False, "reason": f"Picked unrelated node: {semantic}"}
def execute_all(self):
self.setup()
results = {
"timestamp": time.time(),
"model": self.device.args.ai_telepathic_model,
"scenarios": {}
}
results = {"timestamp": time.time(), "model": self.device.args.ai_telepathic_model, "scenarios": {}}
def run_and_log(name, func):
print(colored(f"\n--- SCENARIO: {name} ---", "magenta"))
start_time = time.time()
data = {"passed": False, "reason": "Unknown error", "latency_ms": 0}
try:
res = func()
if isinstance(res, dict): data.update(res)
elif res is True: data["passed"] = True
if isinstance(res, dict):
data.update(res)
elif res is True:
data["passed"] = True
except Exception as e:
print(colored(f"❌ EXCEPTION: {e}", "red"))
data["reason"] = str(e)
dur = time.time() - start_time
data["latency_ms"] = int(dur * 1000)
results["scenarios"][name] = data
if data["passed"]:
print(colored(f"🏁 {name} completed successfully in {dur:.2f}s", "green"))
else:
print(colored(f"🚨 {name} FAILED! (Elapsed: {dur:.2f}s)", "red", attrs=["bold"]))
run_and_log("The Modal Trap (Blocked UI)", self.test_modal_trap)
run_and_log("The Ad Deception (Sponsored)", self.test_ad_deception)
run_and_log("The VLM Hallucination Gap (Text Trap)", self.test_vlm_hallucination)
self.teardown()
out_path = os.path.join(ROOT_DIR, "benchmarks", "data", "live_learning_results.json")
with open(out_path, "w") as f:
json.dump(results, f, indent=4)
print(colored(f"\n📄 Saved intensive learning results to: {out_path}", "cyan", attrs=["bold"]))
if __name__ == "__main__":
runner = BrainDiagnosticRunner()
runner.execute_all()

103
benchmarks/run_competitive_benchmark.py
View File

@@ -1,9 +1,9 @@
import os
import sys
import json
import time
import argparse
import json
import os
import subprocess
import sys
import time
from datetime import datetime
# Add root project path so we can import internal modules safely
@@ -14,6 +14,7 @@ from GramAddict.core.llm_provider import query_telepathic_llm
BENCHMARKS_FILE = os.path.join(os.path.dirname(__file__), "data/llm_benchmarks.json")
SCENARIOS_FILE = os.path.join(os.path.dirname(__file__), "data/benchmark_scenarios.json")
def load_json(path):
if os.path.exists(path):
try:
@@ -23,22 +24,24 @@ def load_json(path):
return None
return None
def save_json(path, data):
with open(path, "w") as f:
json.dump(data, f, indent=4)
def normalize_scores(db):
if not db.get("models"):
return db
# 1. Find the highest raw score across all models
max_raw = 0
leader_model = None
for name, data in db["models"].items():
if data.get("is_unsuitable"):
continue
raw = data.get("raw_score", 0)
if raw > max_raw:
max_raw = raw
@@ -57,10 +60,11 @@ def normalize_scores(db):
for name, data in db["models"].items():
raw = data.get("raw_score", 0)
data["relative_performance_pct"] = round((raw / max_raw) * 100, 1)
data["is_leader"] = (name == leader_model)
data["is_leader"] = name == leader_model
return db
def get_installed_ollama_models():
"""
Finds truly local Ollama models by parsing 'ollama list'.
@@ -76,25 +80,26 @@ def get_installed_ollama_models():
if len(parts) >= 3:
name = parts[0]
size = parts[2]
# 1. Skip if size is '-' (remote/cloud model)
if size == "-":
continue
# 2. Skip ':cloud' tagged models explicitly
if ":cloud" in name:
continue
# 3. Filter out purely embedding models
if any(k in name.lower() for k in ["embed", "minilm", "rerank"]):
continue
models.append(name)
return models
except Exception as e:
print(f"⚠️ Could not list Ollama models: {e}")
return []
def benchmark_model(model_name: str, url: str, force: bool = False):
db = load_json(BENCHMARKS_FILE) or {"models": {}}
scenarios_data = load_json(SCENARIOS_FILE)
@@ -105,26 +110,25 @@ def benchmark_model(model_name: str, url: str, force: bool = False):
if not force and model_name in db.get("models", {}):
pct = db["models"][model_name].get("relative_performance_pct", "N/A")
if not db["models"][model_name].get("is_unsuitable"):
print(f"Typical execution skip for {model_name} (Rel: {pct}%). Use --force.")
return
print(f"Typical execution skip for {model_name} (Rel: {pct}%). Use --force.")
return
print(f"\n🚀 [Competitive Benchmarking] Model: {model_name}")
total_raw = 0
total_latency = 0
results_detail = {}
passed_all = True
blank_b64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII="
system_prompt = (
"You identify which UI element to tap based ONLY on a JSON array of parsed Android elements. "
"Output ONLY valid JSON: {\"index\": number, \"reason\": \"brief reason\"}"
'Output ONLY valid JSON: {"index": number, "reason": "brief reason"}'
)
scenarios = scenarios_data["scenarios"]
for scenario in scenarios:
print(f"--- Running: {scenario['name']} ---")
user_prompt = (
f"Which element should I tap to: {scenario['task']}\n\n"
f"Elements:\n{json.dumps(scenario['nodes'], indent=1)}\n\n"
@@ -147,14 +151,16 @@ def benchmark_model(model_name: str, url: str, force: bool = False):
raw_points = 0
try:
clean = resp_str.strip()
if clean.startswith("```json"): clean = clean[7:]
if clean.endswith("```"): clean = clean[:-3]
if clean.startswith("```json"):
clean = clean[7:]
if clean.endswith("```"):
clean = clean[:-3]
data = json.loads(clean)
# Points for structural adherence
if "index" in data and "reason" in data:
raw_points += 40
# Points for correctness
if data["index"] == scenario["target_index"]:
raw_points += 60
@@ -173,39 +179,44 @@ def benchmark_model(model_name: str, url: str, force: bool = False):
total_raw += raw_points
avg_latency = total_latency // len(scenarios) if scenarios else 0
print(f"\n📊 {model_name} Result: {'PASS' if passed_all else 'FAIL'} | Score: {total_raw} | Latency: {avg_latency}ms")
print(
f"\n📊 {model_name} Result: {'PASS' if passed_all else 'FAIL'} | Score: {total_raw} | Latency: {avg_latency}ms"
)
if model_name not in db["models"]:
db["models"][model_name] = {}
db["models"][model_name].update({
"raw_score": total_raw,
"telepathic_score": int((total_raw / (len(scenarios) * 100)) * 100) if scenarios else 0,
"latency_ms": avg_latency,
"last_tested": datetime.utcnow().isoformat() + "Z",
"details": results_detail,
"passed_all": passed_all,
"is_unsuitable": not passed_all
})
db["models"][model_name].update(
{
"raw_score": total_raw,
"telepathic_score": int((total_raw / (len(scenarios) * 100)) * 100) if scenarios else 0,
"latency_ms": avg_latency,
"last_tested": datetime.utcnow().isoformat() + "Z",
"details": results_detail,
"passed_all": passed_all,
"is_unsuitable": not passed_all,
}
)
# Recalculate relative scores across all models
db = normalize_scores(db)
save_json(BENCHMARKS_FILE, db)
if __name__ == "__main__":
from GramAddict.core.config import Config
parser = argparse.ArgumentParser(description="Competitive Benchmark for Singularity", add_help=False)
parser.add_argument("--config", type=str, help="Bot config file")
parser.add_argument("--model", type=str, help="Explicit model name")
parser.add_argument("--url", type=str, help="Explicit endpoint URL")
parser.add_argument("--force", action="store_true", help="Force re-testing")
parser.add_argument("--all-ollama", action="store_true", help="Automatically find and test all local Ollama models")
args, unknown = parser.parse_known_args()
models_to_test = []
if args.all_ollama:
ollama_models = get_installed_ollama_models()
for m in ollama_models:
@@ -215,8 +226,12 @@ if __name__ == "__main__":
elif args.config:
configs = Config(first_run=True, config=args.config)
configs.parse_args()
for attr, pref in [("ai_telepathic_model", "ai_telepathic_url"), ("ai_model", "ai_model_url"), ("ai_condenser_model", "ai_condenser_url")]:
for attr, pref in [
("ai_telepathic_model", "ai_telepathic_url"),
("ai_model", "ai_model_url"),
("ai_condenser_model", "ai_condenser_url"),
]:
m = getattr(configs.args, attr, None)
u = getattr(configs.args, pref, "http://localhost:11434/api/generate")
if m:
@@ -224,7 +239,7 @@ if __name__ == "__main__":
else:
print("❌ Syntax: --all-ollama OR --config test_config.yml OR --model x --url y")
sys.exit(1)
for m, u in set(models_to_test):
benchmark_model(m, u, args.force)
time.sleep(1)