#!/usr/bin/env python3
# Author: Tom Sapletta · https://tom.sapletta.com
# Part of the ifURI solution.

"""Re-check current month files and move confident non-invoices aside.

This script works on the *current* flattened filesystem state exposed by the
node. It does not trust the old audit CSV alone:

1. Lists current month files through ``fs://host/dir/query/list``.
2. OCRs current PDF/image files through ``ocr://host/document/query/batch``.
3. Reads spreadsheet/zip metadata through ``fs://host/file/query/blob``.
4. Produces a CSV/JSON classification report.
5. Optionally moves confident non-invoices to ``no_invoice``.
"""

from __future__ import annotations

import argparse
import base64
import csv
import io
import json
import re
import urllib.error
import urllib.request
import zipfile
from collections import Counter
from pathlib import Path, PurePosixPath
from typing import Any
from xml.etree import ElementTree

from flatten_tree_files import discover_months, list_dir, walk_files
from move_no_invoice import DEFAULT_NODE_URL

DEFAULT_ROOT = "/home/tom/Downloads/2026/5"
OCR_EXTENSIONS = "pdf,png,jpg,jpeg"
TEXT_EXTENSIONS = {"csv", "json", "md", "txt", "xml", "log"}
SPREADSHEET_EXTENSIONS = {"xlsx"}
ARCHIVE_EXTENSIONS = {"zip"}

INVOICE_WEIGHTS = {
    "faktura": 5,
    "invoice": 5,
    "receipt": 4,
    "rachunek": 4,
    "paragon": 4,
    "tax invoice": 5,
    "numer faktury": 5,
    "invoice number": 5,
    "data wystawienia": 3,
    "invoice date": 3,
    "sprzedawca": 2,
    "nabywca": 2,
    "seller": 2,
    "buyer": 2,
    "bill to": 2,
    "vat": 1,
    "nip": 1,
    "amount due": 2,
    "razem do zaplaty": 2,
    "razem do zapłaty": 2,
    "suma": 1,
    "total": 1,
}

NON_INVOICE_WEIGHTS = {
    "regulamin": 6,
    "cennik": 6,
    "formularz": 6,
    "odstapienia": 6,
    "odstąpienia": 6,
    "odstapieniu": 6,
    "odstąpieniu": 6,
    "oswiadczenie_o_odstapieniu": 6,
    "oświadczenie o odstąpieniu": 6,
    "warunki": 5,
    "terms": 6,
    "conditions": 6,
    "privacy": 5,
    "polityka": 5,
    "instrukcja": 5,
    "manual": 5,
    "portfolio": 6,
    "newsletter": 6,
    "domeny": 6,
    "vessel specification": 6,
    "historia operacji": 6,
    "historia_operacji": 6,
    "historia transakcji": 6,
    "elektroniczne zestawienie operacji": 6,
    "wykaz polaczen": 5,
    "wykaz połączeń": 5,
    "widerrufsbelehrung": 6,
    "muster-widerrufsformular": 6,
    "chron swoje dane": 5,
    "cybertarcza": 5,
    "bezpieczny swiat dziecka": 6,
    "bezpieczny świat dziecka": 6,
    "grunder- und unternehmertag": 6,
    "gründer- und unternehmertag": 6,
    "potwierdzenie wykonania przelewu": 6,
    "potwierdzenie przelewu": 6,
    "declaration_tax": 6,
    "zaliczka na podatek": 6,
    "podatek dochodowy": 5,
    "outlookemoji": 6,
    "banner": 5,
    "logo": 6,
    "icon": 5,
    "messageicon": 6,
    "phoneicon": 6,
    "static-footer": 6,
    "static-social-media": 6,
    "static-graphics": 5,
    "kolo": 5,
    "posts termine": 5,
    "vereinbarung zur auftragsverarbeitung": 6,
    "auftragsverarbeitung": 6,
    "dsgvo": 6,
    "zus prototypowanie": 6,
    "vat prototypowanie": 6,
    "vat za": 6,
    "image001": 6,
    "image002": 6,
    "image003": 6,
    "il_blue_przekazalismy": 6,
    "il_blue_przekazaliśmy": 6,
}

NON_INVOICE_LABELS = {
    "email-assets-review",
    "unknown-review",
}


class NodeClient:
    def __init__(self, base_url: str, timeout: int = 180):
        self.base_url = base_url.rstrip("/")
        self.timeout = timeout

    def run(self, uri: str, payload: dict[str, Any], timeout: int | None = None) -> dict[str, Any]:
        body = json.dumps({"uri": uri, "payload": payload}).encode("utf-8")
        request = urllib.request.Request(
            self.base_url + "/run",
            data=body,
            headers={"Content-Type": "application/json"},
            method="POST",
        )
        try:
            with urllib.request.urlopen(request, timeout=timeout or self.timeout) as response:
                return json.loads(response.read().decode("utf-8") or "{}")
        except urllib.error.HTTPError as exc:
            raw = exc.read().decode("utf-8", "replace") if exc.fp else ""
            try:
                return json.loads(raw or "{}")
            except json.JSONDecodeError:
                return {"ok": False, "status": exc.code, "error": raw}


def route_value(envelope: dict[str, Any]) -> dict[str, Any]:
    result = envelope.get("result") or {}
    if isinstance(result, dict) and isinstance(result.get("value"), dict):
        return result["value"]
    return result if isinstance(result, dict) else {}


def split_words(raw: str) -> list[str]:
    return [item.strip() for item in raw.replace(";", ",").split(",") if item.strip()]


def rel_from_node_path(path: str, root: str) -> str:
    root = root.rstrip("/")
    if path.startswith(root + "/"):
        return path[len(root) + 1:]
    return path.lstrip("/")


def list_current_files(client: NodeClient, args: argparse.Namespace) -> list[dict[str, Any]]:
    months = discover_months(client, args.list_uri, args.root_path, args.months)
    files: list[dict[str, Any]] = []
    exclude_parts = set(split_words(args.exclude_parts))
    for month in months:
        for rel in walk_files(client, args.list_uri, month, exclude_parts):
            if len(PurePosixPath(rel).parts) != 2:
                continue
            ext = PurePosixPath(rel).suffix.lower().lstrip(".")
            files.append({
                "path": rel,
                "name": PurePosixPath(rel).name,
                "month": month,
                "ext": ext,
            })
    return sorted(files, key=lambda row: row["path"])


def manifest_rows(path: Path) -> list[dict[str, str]]:
    if not path.exists():
        return []
    if path.suffix == ".json":
        data = json.loads(path.read_text(encoding="utf-8"))
        return [row for row in data.get("results", []) if isinstance(row, dict)]
    with path.open(encoding="utf-8", newline="") as handle:
        return list(csv.DictReader(handle))


def load_manifest_metadata(out_dir: Path) -> dict[str, dict[str, str]]:
    metadata: dict[str, dict[str, str]] = {}
    for stem in ["flat_invoice_manifest", "flat_tree_manifest"]:
        for row in manifest_rows(out_dir / f"{stem}.csv"):
            target = str(row.get("target") or "")
            if not target:
                continue
            metadata[target] = {
                "source": stem,
                "source_path": str(row.get("path") or ""),
                "date": str(row.get("date") or ""),
                "label": str(row.get("label") or row.get("category") or ""),
                "category": str(row.get("category") or ""),
                "reason": str(row.get("reason") or ""),
            }
    for row in manifest_rows(out_dir / "flat_date_repair_manifest.csv"):
        source = str(row.get("source") or "")
        target = str(row.get("target") or "")
        if not source or not target:
            continue
        base = dict(metadata.get(source, {}))
        base["source"] = "flat_date_repair_manifest"
        base["date_repair_from"] = source
        metadata[target] = base
    return metadata


def run_ocr_batch(client: NodeClient, args: argparse.Namespace) -> dict[str, dict[str, Any]]:
    envelope = client.run(args.ocr_uri, {
        "root": args.absolute_root,
        "extensions": OCR_EXTENSIONS,
        "max_files": args.max_files,
        "max_chars_per_file": args.max_chars_per_file,
        "backend": args.backend,
        "recursive": True,
    }, timeout=args.ocr_timeout)
    data = route_value(envelope)
    if not data.get("ok"):
        raise RuntimeError(f"OCR batch failed: {data.get('error') or envelope}")
    out: dict[str, dict[str, Any]] = {}
    for item in data.get("results") or []:
        rel = rel_from_node_path(str(item.get("path", "")), args.absolute_root)
        if rel.startswith("no_invoice/") or len(PurePosixPath(rel).parts) != 2:
            continue
        out[rel] = item
    return out


def blob_bytes(client: NodeClient, path: str, max_bytes: int) -> bytes | None:
    data = route_value(client.run("fs://host/file/query/blob", {"path": path, "max_bytes": max_bytes}))
    if not data.get("ok") or not data.get("bytes_b64"):
        return None
    return base64.b64decode(str(data["bytes_b64"]))


def xml_text(raw: bytes) -> str:
    try:
        root = ElementTree.fromstring(raw)
    except ElementTree.ParseError:
        return ""
    texts = []
    for elem in root.iter():
        if elem.text and elem.text.strip():
            texts.append(elem.text.strip())
    return " ".join(texts)


def xlsx_text(raw: bytes, max_chars: int = 8000) -> str:
    try:
        with zipfile.ZipFile(io.BytesIO(raw)) as archive:
            parts = []
            for name in archive.namelist():
                if name == "xl/sharedStrings.xml" or re.fullmatch(r"xl/worksheets/sheet\d+\.xml", name):
                    parts.append(xml_text(archive.read(name)))
                    if sum(len(part) for part in parts) >= max_chars:
                        break
            return " ".join(parts)[:max_chars]
    except (zipfile.BadZipFile, KeyError, OSError):
        return ""


def zip_listing_text(raw: bytes, max_names: int = 120) -> str:
    try:
        with zipfile.ZipFile(io.BytesIO(raw)) as archive:
            names = archive.namelist()[:max_names]
            return " ".join(names)
    except zipfile.BadZipFile:
        return ""


def supplemental_text(client: NodeClient, file_row: dict[str, Any], max_bytes: int) -> tuple[str, str]:
    ext = str(file_row["ext"])
    path = str(file_row["path"])
    if ext in SPREADSHEET_EXTENSIONS:
        raw = blob_bytes(client, path, max_bytes)
        return (xlsx_text(raw) if raw else "", "xlsx")
    if ext in ARCHIVE_EXTENSIONS:
        raw = blob_bytes(client, path, max_bytes)
        return (zip_listing_text(raw) if raw else "", "zip-list")
    if ext in TEXT_EXTENSIONS:
        data = route_value(client.run("fs://host/file/query/read", {"path": path, "max_bytes": min(max_bytes, 65536)}))
        return (str(data.get("content") or "") if data.get("ok") else "", "text")
    return "", ""


def normalize_text(value: str) -> str:
    return re.sub(r"\s+", " ", value.lower()).strip()


def score_terms(text: str, weights: dict[str, int]) -> tuple[int, list[str]]:
    score = 0
    hits = []
    for term, weight in weights.items():
        if term in text:
            score += weight
            hits.append(term)
    return score, hits


def label_from_flat_name(path: str) -> str:
    name = PurePosixPath(path).name
    match = re.match(r"^20\d{2}[.-]\d{2}[.-]\d{2}-(.+?)-", name)
    return match.group(1) if match else ""


def classify_file(path: str, ext: str, text: str, meta: dict[str, str]) -> dict[str, Any]:
    label = meta.get("label") or label_from_flat_name(path)
    haystack = normalize_text(" ".join([path, label, meta.get("category", ""), meta.get("reason", ""), text]))
    invoice_score, invoice_hits = score_terms(haystack, INVOICE_WEIGHTS)
    non_score, non_hits = score_terms(haystack, NON_INVOICE_WEIGHTS)
    reasons = []
    if invoice_hits:
        reasons.append("invoice:" + "|".join(invoice_hits[:6]))
    if non_hits:
        reasons.append("non_invoice:" + "|".join(non_hits[:6]))
    if label in NON_INVOICE_LABELS and invoice_score < 8:
        non_score += 4
        reasons.append(f"label:{label}")
    if label in NON_INVOICE_LABELS and ext in {"png", "jpg", "jpeg", "gif", "webp"} and invoice_score < 6:
        non_score += 4
        reasons.append(f"image_asset_label:{label}")
    if re.search(r"image\d+", haystack) and invoice_score < 6:
        non_score += 5
        reasons.append("inline_image_name")
    if ext in ARCHIVE_EXTENSIONS and invoice_score < 8:
        non_score += 5
        reasons.append(f"archive:{ext}")
    elif ext in ARCHIVE_EXTENSIONS and (label in NON_INVOICE_LABELS or "historia_operacji" in haystack or "domeny" in haystack):
        non_score += 8
        reasons.append(f"mixed_archive:{ext}")
    if ext in SPREADSHEET_EXTENSIONS and invoice_score < 8:
        non_score += 3
        reasons.append(f"spreadsheet:{ext}")

    if invoice_score >= 6 and invoice_score >= non_score:
        return {
            "decision": "invoice",
            "confidence": min(0.99, 0.62 + invoice_score / 30),
            "invoice_score": invoice_score,
            "non_invoice_score": non_score,
            "reason": "; ".join(reasons),
        }
    if non_score >= 6 and (invoice_score < 6 or non_score >= invoice_score + 3):
        return {
            "decision": "no_invoice",
            "confidence": min(0.98, 0.62 + non_score / 24),
            "invoice_score": invoice_score,
            "non_invoice_score": non_score,
            "reason": "; ".join(reasons),
        }
    return {
        "decision": "review",
        "confidence": 0.45,
        "invoice_score": invoice_score,
        "non_invoice_score": non_score,
        "reason": "; ".join(reasons) or "weak_or_missing_evidence",
    }


def write_csv(path: Path, rows: list[dict[str, Any]]) -> None:
    fields = [
        "path",
        "decision",
        "confidence",
        "invoice_score",
        "non_invoice_score",
        "ext",
        "label",
        "source",
        "text_source",
        "chars",
        "reason",
        "target",
        "ok",
        "moved",
        "error",
        "preview",
    ]
    path.parent.mkdir(parents=True, exist_ok=True)
    with path.open("w", encoding="utf-8", newline="") as handle:
        writer = csv.DictWriter(handle, fieldnames=fields)
        writer.writeheader()
        for row in rows:
            writer.writerow({field: row.get(field, "") for field in fields})


def move_no_invoice_rows(client: NodeClient, rows: list[dict[str, Any]], execute: bool) -> list[dict[str, Any]]:
    moved_rows = []
    for row in rows:
        if row["decision"] != "no_invoice":
            continue
        envelope = client.run("fs://host/file/command/move_to_dir", {
            "path": row["path"],
            "target_dir": "no_invoice",
            "preserve_relative": True,
            "dry_run": not execute,
            "overwrite": False,
        })
        data = route_value(envelope)
        moved_rows.append({
            **row,
            "target": data.get("target", ""),
            "ok": bool(data.get("ok")),
            "moved": bool(data.get("moved")),
            "error": data.get("error", envelope.get("error", "")),
        })
    return moved_rows


def verify_invoice_files(args: argparse.Namespace, client: NodeClient | None = None) -> dict[str, Any]:
    client = client or NodeClient(args.node_url, timeout=args.timeout)
    out_dir = Path(args.output_dir)
    metadata = load_manifest_metadata(out_dir)
    files = list_current_files(client, args)
    ocr = {} if args.skip_ocr else run_ocr_batch(client, args)
    rows: list[dict[str, Any]] = []
    for file_row in files:
        path = str(file_row["path"])
        ext = str(file_row["ext"])
        meta = metadata.get(path, {})
        text = ""
        text_source = ""
        if path in ocr:
            text = str(ocr[path].get("text") or "")
            text_source = str(ocr[path].get("backend") or "ocr")
        if not text:
            text, text_source = supplemental_text(client, file_row, args.max_blob_bytes)
        decision = classify_file(path, ext, text, meta)
        rows.append({
            "path": path,
            "ext": ext,
            "label": meta.get("label") or label_from_flat_name(path),
            "source": meta.get("source", ""),
            "text_source": text_source,
            "chars": len(text),
            "preview": " ".join(text.split())[:240],
            **decision,
        })

    move_rows = move_no_invoice_rows(client, rows, execute=args.execute)
    by_path = {row["path"]: row for row in move_rows}
    for index, row in enumerate(rows):
        if row["path"] in by_path:
            rows[index] = by_path[row["path"]]

    stem = "invoice_recheck_manifest" if args.execute else "invoice_recheck_plan"
    files_json = out_dir / f"{stem}.json"
    files_csv = out_dir / f"{stem}.csv"
    move_csv = out_dir / f"{stem}_moves.csv"
    out_dir.mkdir(parents=True, exist_ok=True)
    counts = Counter(row["decision"] for row in rows)
    payload = {
        "ok": all(not row.get("error") for row in rows),
        "dryRun": not args.execute,
        "files": len(rows),
        "counts": dict(counts),
        "moveCandidates": sum(1 for row in rows if row["decision"] == "no_invoice"),
        "moved": sum(1 for row in rows if row.get("moved")),
        "rows": rows,
    }
    files_json.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
    write_csv(files_csv, rows)
    write_csv(move_csv, [row for row in rows if row["decision"] == "no_invoice"])
    payload["reports"] = {"json": str(files_json), "csv": str(files_csv), "moves_csv": str(move_csv)}
    return payload


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description="Re-check current month files and move confident non-invoices.")
    parser.add_argument("--node-url", default=DEFAULT_NODE_URL)
    parser.add_argument("--absolute-root", default=DEFAULT_ROOT)
    parser.add_argument("--root-path", default=".")
    parser.add_argument("--months", default="")
    parser.add_argument("--exclude-parts", default="no_invoice,.deleted")
    parser.add_argument("--output-dir", default=".state")
    parser.add_argument("--list-uri", default="fs://host/dir/query/list")
    parser.add_argument("--ocr-uri", default="ocr://host/document/query/batch")
    parser.add_argument("--backend", default="auto")
    parser.add_argument("--max-files", type=int, default=10000)
    parser.add_argument("--max-chars-per-file", type=int, default=4000)
    parser.add_argument("--max-blob-bytes", type=int, default=10 * 1024 * 1024)
    parser.add_argument("--timeout", type=int, default=180)
    parser.add_argument("--ocr-timeout", type=int, default=600)
    parser.add_argument("--skip-ocr", action="store_true")
    parser.add_argument("--execute", action="store_true", help="move confident non-invoices to no_invoice")
    return parser


def main(argv: list[str] | None = None) -> int:
    args = build_parser().parse_args(argv)
    result = verify_invoice_files(args)
    print(json.dumps({
        "ok": result["ok"],
        "dryRun": result["dryRun"],
        "files": result["files"],
        "counts": result["counts"],
        "moveCandidates": result["moveCandidates"],
        "moved": result["moved"],
        "reports": result["reports"],
    }, ensure_ascii=False, indent=2))
    return 0 if result["ok"] else 1


if __name__ == "__main__":
    raise SystemExit(main())