|
"""Run the CHANGED parser on stored NSDL exports and compare with manually created Frappe docs (read-only)."""
|
|
import json
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import urllib.parse
|
|
import urllib.request
|
|
from collections import Counter
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
|
|
import pandas as pd
|
|
|
|
S = Path(sys.argv[1])
|
|
sys.path.insert(0, "app/phase_a_fetch")
|
|
sys.path.insert(0, ".")
|
|
import nsdl_parser # noqa: E402
|
|
|
|
secret = json.loads(
|
|
subprocess.check_output(
|
|
["aws", "secretsmanager", "get-secret-value", "--secret-id", "nsdl-boe-automation-secrets-TEST",
|
|
"--region", "ap-south-1", "--query", "SecretString", "--output", "text"],
|
|
text=True,
|
|
)
|
|
)
|
|
|
|
# Latest stored export per request id
|
|
frames = []
|
|
latest = {}
|
|
for i, f in enumerate(sorted((S / "raw").glob("*/dta_export_v1.xlsx"))):
|
|
df = nsdl_parser.read_nsdl_export(str(f))
|
|
df["__file"] = i
|
|
frames.append(df)
|
|
for rid in df["REQUEST ID"].astype(str).str.strip().unique():
|
|
latest[rid] = i
|
|
all_df = pd.concat(frames, ignore_index=True)
|
|
all_df["__rid"] = all_df["REQUEST ID"].astype(str).str.strip()
|
|
all_df = all_df[all_df.apply(lambda r: latest[r["__rid"]] == r["__file"], axis=1)]
|
|
print("requests in stored exports:", all_df["__rid"].nunique())
|
|
|
|
|
|
def norm(s):
|
|
return re.sub(r"\s+", " ", (s or "").strip()).upper()
|
|
|
|
|
|
overall = Counter()
|
|
for client in ("wud", "eap"):
|
|
c = secret["CLIENTS"][client]
|
|
base = c["FRAPPE_URL"].rstrip("/")
|
|
H = {"Authorization": f"token {c['api_key']}:{c['api_secret']}"}
|
|
|
|
def get(path, base=base, H=H):
|
|
return json.load(urllib.request.urlopen(urllib.request.Request(base + path, headers=H), timeout=60))
|
|
|
|
q = urllib.parse.urlencode({"fields": json.dumps(["name", "owner", "creation", "request_id"]),
|
|
"limit_page_length": "5000", "order_by": "creation asc"})
|
|
lst = get("/api/resource/FTWZ%20DTA%20Sale?" + q)["data"]
|
|
manual = [r for r in lst if r["owner"] != "<automation-user>" and r["creation"] >= "2026-09-05" and r.get("request_id") in latest]
|
|
with ThreadPoolExecutor(8) as pool:
|
|
docs = list(pool.map(lambda n, get=get: get("/api/resource/FTWZ%20DTA%20Sale/" + urllib.parse.quote(n))["data"], [r["name"] for r in manual]))
|
|
sel = all_df[all_df["__rid"].isin({d["request_id"] for d in docs})].drop(columns=["__file", "__rid"])
|
|
res = nsdl_parser.parse_dta_records(sel, len(sel))
|
|
parsed = {r["request_id"]: r for r in res.records}
|
|
tally = Counter()
|
|
bad = []
|
|
for d in docs:
|
|
p = parsed.get(d["request_id"])
|
|
if p is None:
|
|
tally["request not produced by parser"] += 1
|
|
continue
|
|
for fld in ("buyer_address", "client_address"):
|
|
ok = (p.get(fld) or "") == (d.get(fld) or "")
|
|
tally[f"{fld}: parser == manual"] += ok
|
|
tally[f"{fld}: DIFFERENT"] += (not ok)
|
|
if not ok and len(bad) < 4:
|
|
bad.append((d["name"], fld, p.get(fld), d.get(fld)))
|
|
for item in d.get("items", []):
|
|
cand = [i for i in p["items"] if norm(i["invoice_no"]) == norm(item.get("invoice_no"))
|
|
and norm(i["item_description"])[:40] == norm(item.get("item_description"))[:40]]
|
|
if len(cand) != 1:
|
|
tally["item row not matched 1:1 (skipped)"] += 1
|
|
continue
|
|
for fld in ("freight_in_inr", "bcd_to_be_considered_for_calculating_igst"):
|
|
pv, mv = str(cand[0][fld]), str(item.get(fld))
|
|
if pv == mv:
|
|
tally[f"{fld}: parser == manual (exact)"] += 1
|
|
else:
|
|
try:
|
|
same_number = abs(float(pv) - float(mv)) < 1e-9
|
|
except ValueError:
|
|
same_number = False
|
|
tally[f"{fld}: " + ("same number, different formatting" if same_number else "DIFFERENT")] += 1
|
|
if not same_number and len(bad) < 8:
|
|
bad.append((d["name"], fld, pv, mv))
|
|
print(f"\n#### {client}: {len(docs)} manual docs, parser produced {len(parsed)} records")
|
|
for k, v in sorted(tally.items()):
|
|
if v:
|
|
print(f" {k:70s} {v}")
|
|
for b in bad:
|
|
print(" !!", b)
|
|
overall.update(tally)
|