3cx-telefonie-middleware/tools/filter_transcripts.py

import json, sqlite3, time
from pathlib import Path
from concurrent.futures import ThreadPoolExecutor, as_completed
import requests

BASE = Path("/opt/3cx-middleware/3cx-telefonie-middleware")
TRANSCRIPTS = BASE / "data/transcripts"
OUT = BASE / "data/transcript_relevance.jsonl"

OLLAMA = "http://192.168.1.101:11434/api/chat"
MODEL = "qwen3:4b"
WORKERS = 2

SYSTEM = """Du entscheidest ausschließlich, ob ein Telefontranskript für
einen Gartenpflanzen-/Onlineshop-Kundenservice relevant ist.

RELEVANT:
Ein echtes Kunden-, Lieferanten- oder Geschäftsgespräch mit verwertbarem
Inhalt.

IRRELEVANT:
Mailbox, reine Ansage, verpasster Anruf ohne Gespräch, Werbung, technische
Ansage oder praktisch kein Gespräch.

UNCERTAIN:
Nicht eindeutig entscheidbar.

Antworte ausschließlich als JSON:
{"relevance":"RELEVANT|IRRELEVANT|UNCERTAIN","confidence":0.0,"reason":"kurze Begründung"}

Keine Intent-Klassifizierung durchführen.
"""

def classify(item):
    path = item
    text = path.read_text(encoding="utf-8", errors="replace").strip()

    payload = {
        "model": MODEL,
        "messages": [
            {"role": "system", "content": SYSTEM},
            {"role": "user", "content": text[:12000]}
        ],
        "stream": False,
        "think": False,
        "format": "json",
        "options": {"temperature": 0}
    }

    start = time.time()
    r = requests.post(OLLAMA, json=payload, timeout=180)
    r.raise_for_status()

    result = json.loads(r.json()["message"]["content"])
    return {
        "file": path.name,
        "relevance": result.get("relevance", "UNCERTAIN"),
        "confidence": result.get("confidence", 0),
        "reason": result.get("reason", ""),
        "seconds": round(time.time() - start, 1)
    }

files = sorted(TRANSCRIPTS.glob("*.txt"))

# offensichtliche Kurz-/Leer-/reine Ansagetexte nicht erneut an Qwen schicken
candidates = []
for p in files:
    text = p.read_text(encoding="utf-8", errors="replace").strip()
    words = len(text.split())

    if not text or words < 12:
        continue

    candidates.append(p)

print(f"Transkripte gesamt: {len(files)}")
print(f"Qwen-Kandidaten:     {len(candidates)}")

OUT.parent.mkdir(parents=True, exist_ok=True)

with OUT.open("w", encoding="utf-8") as f:
    with ThreadPoolExecutor(max_workers=WORKERS) as pool:
        futures = {pool.submit(classify, p): p for p in candidates}

        done = 0
        for future in as_completed(futures):
            p = futures[future]
            done += 1

            try:
                result = future.result()
                f.write(json.dumps(result, ensure_ascii=False) + "\n")
                f.flush()

                print(
                    f"[{done}/{len(candidates)}] "
                    f"{p.name} {result['seconds']}s → "
                    f"{result['relevance']} "
                    f"{result['confidence']}",
                    flush=True
                )

            except Exception as e:
                print(f"ERROR {p.name}: {e}", flush=True)

print(f"\nErgebnis: {OUT}")