|
<!doctype html><html><head><meta charset=utf8><meta name=viewport content="width=device-width,initial-scale=1"><style>:root{color-scheme:light}body{margin:0;padding:0;font:14px -apple-system,BlinkMacSystemFont,sans-serif;background:#faf9f5;color:#141413}img{max-width:100%}[hidden]:not([hidden=until-found i]){display:none!important}</style></head><body>
|
|
<title>NSDL SEZ BOE Automation</title>
|
|
<style>
|
|
@import url('https://fonts.googleapis.com/css2?family=Archivo:wght@500;700;800;900&family=Source+Sans+3:wght@400;500;600;700&family=IBM+Plex+Mono:wght@400;500;600&display=swap');
|
|
|
|
:root{
|
|
--bg:#eef1ef; --surface:#ffffff; --surface-2:#e3e8e4; --surface-3:#dbe1dc;
|
|
--ink:#161b1a; --ink-soft:#4c5654; --ink-faint:#7c8783;
|
|
--line:#cbd3cd; --line-soft:#dde3de;
|
|
--accent:#1f4e5c; --accent-2:#0f6e82; --accent-warm:#9a5f1b;
|
|
--st-staged:#5b6b78; --st-pushed:#9a5f1b; --st-attached:#1f7a53; --st-failed:#ae3a34;
|
|
--code-bg:#e3e8e4; --shadow: 0 1px 2px rgba(20,30,25,.06), 0 6px 20px -8px rgba(20,30,25,.12);
|
|
}
|
|
@media (prefers-color-scheme: dark){
|
|
:root:not([data-theme="light"]){
|
|
--bg:#121613; --surface:#191f1b; --surface-2:#212823; --surface-3:#28312a;
|
|
--ink:#e7ece8; --ink-soft:#a6b2ac; --ink-faint:#75827b;
|
|
--line:#2c332e; --line-soft:#242b26;
|
|
--accent:#5cb7cb; --accent-2:#6fd0e3; --accent-warm:#d99a52;
|
|
--st-staged:#8b98a2; --st-pushed:#d99a52; --st-attached:#4fbf8c; --st-failed:#e2716b;
|
|
--code-bg:#1d2420; --shadow: 0 1px 2px rgba(0,0,0,.3), 0 6px 24px -10px rgba(0,0,0,.5);
|
|
}
|
|
}
|
|
:root[data-theme="dark"]{
|
|
--bg:#121613; --surface:#191f1b; --surface-2:#212823; --surface-3:#28312a;
|
|
--ink:#e7ece8; --ink-soft:#a6b2ac; --ink-faint:#75827b;
|
|
--line:#2c332e; --line-soft:#242b26;
|
|
--accent:#5cb7cb; --accent-2:#6fd0e3; --accent-warm:#d99a52;
|
|
--st-staged:#8b98a2; --st-pushed:#d99a52; --st-attached:#4fbf8c; --st-failed:#e2716b;
|
|
--code-bg:#1d2420; --shadow: 0 1px 2px rgba(0,0,0,.3), 0 6px 24px -10px rgba(0,0,0,.5);
|
|
}
|
|
|
|
*{box-sizing:border-box;}
|
|
body{
|
|
background:var(--bg); color:var(--ink);
|
|
font-family:'Source Sans 3', ui-sans-serif, system-ui, sans-serif;
|
|
font-size:16px; line-height:1.6;
|
|
margin:0; padding:0 20px;
|
|
}
|
|
h1,h2,h3,h4{
|
|
font-family:'Archivo', ui-sans-serif, system-ui, sans-serif;
|
|
text-wrap:balance; color:var(--ink); margin:0;
|
|
}
|
|
code, pre, .mono, .k, .env-name{
|
|
font-family:'IBM Plex Mono', ui-monospace, 'SF Mono', monospace;
|
|
}
|
|
a{ color:var(--accent-2); text-decoration-thickness:1px; }
|
|
p{ margin:0 0 .9em 0; max-width:68ch; }
|
|
.wrap{ max-width:1180px; margin:0 auto; }
|
|
|
|
/* ---------- top bar ---------- */
|
|
.topbar{
|
|
position:sticky; top:0; z-index:40;
|
|
background:color-mix(in srgb, var(--bg) 88%, transparent);
|
|
backdrop-filter:blur(8px);
|
|
border-bottom:1px solid var(--line);
|
|
padding-block:12px;
|
|
}
|
|
.topbar-inner{ display:flex; align-items:center; justify-content:space-between; gap:16px; max-width:1180px; margin:0 auto; }
|
|
.brand{ display:flex; align-items:baseline; gap:10px; }
|
|
.brand .mark{ font-family:'Archivo'; font-weight:900; font-size:1.05rem; letter-spacing:.01em; }
|
|
.brand .sub{ font-size:.78rem; color:var(--ink-faint); font-family:'IBM Plex Mono'; }
|
|
.topbar nav{ display:flex; gap:4px; }
|
|
.navlink{
|
|
font-size:.82rem; color:var(--ink-soft); padding:6px 10px; border-radius:6px;
|
|
text-decoration:none; white-space:nowrap;
|
|
}
|
|
.navlink:hover{ background:var(--surface-2); color:var(--ink); }
|
|
#menuToggle{ display:none; }
|
|
@media (max-width: 980px){
|
|
.topbar nav{ display:none; }
|
|
#menuToggle{ display:inline-flex; }
|
|
}
|
|
|
|
/* ---------- hero ---------- */
|
|
.hero{ padding-block:44px 28px; }
|
|
.eyebrow{
|
|
display:inline-flex; align-items:center; gap:8px;
|
|
font-family:'IBM Plex Mono'; font-size:.74rem; letter-spacing:.08em; text-transform:uppercase;
|
|
color:var(--accent-2); background:var(--surface-2); border:1px solid var(--line);
|
|
padding:5px 10px; border-radius:100px; margin-bottom:18px;
|
|
}
|
|
.eyebrow .dot{ width:6px; height:6px; border-radius:50%; background:var(--st-attached); }
|
|
h1.title{ font-size:clamp(1.9rem, 4vw, 3rem); font-weight:900; line-height:1.05; max-width:16ch; }
|
|
.lede{ font-size:1.08rem; color:var(--ink-soft); max-width:62ch; margin-top:16px; }
|
|
|
|
.stat-strip{
|
|
display:grid; grid-template-columns:repeat(auto-fit, minmax(140px,1fr));
|
|
gap:1px; background:var(--line); border:1px solid var(--line); border-radius:10px;
|
|
overflow:hidden; margin-top:28px;
|
|
}
|
|
.stat{ background:var(--surface); padding:14px 16px; }
|
|
.stat .n{ font-family:'Archivo'; font-weight:800; font-size:1.5rem; }
|
|
.stat .l{ font-size:.76rem; color:var(--ink-faint); font-family:'IBM Plex Mono'; margin-top:2px; }
|
|
|
|
/* ---------- layout: rail + content ---------- */
|
|
.layout{ display:grid; grid-template-columns:220px 1fr; gap:36px; align-items:start; padding-bottom:80px; }
|
|
@media (max-width: 980px){ .layout{ grid-template-columns:1fr; } }
|
|
|
|
.rail{ position:sticky; top:66px; max-height:calc(100vh - 90px); overflow:auto; padding-top:8px; }
|
|
@media (max-width: 980px){ .rail{ display:none; } }
|
|
.rail-title{ font-family:'IBM Plex Mono'; font-size:.7rem; letter-spacing:.08em; text-transform:uppercase; color:var(--ink-faint); margin-bottom:10px; }
|
|
.rail a{
|
|
display:block; font-size:.86rem; color:var(--ink-soft); text-decoration:none;
|
|
padding:5px 0 5px 12px; border-left:2px solid var(--line-soft); margin-bottom:1px;
|
|
}
|
|
.rail a:hover{ color:var(--ink); border-left-color:var(--accent-2); }
|
|
.rail a.sub{ padding-left:24px; font-size:.8rem; }
|
|
|
|
main{ min-width:0; }
|
|
section{ padding-top:52px; scroll-margin-top:80px; }
|
|
section:first-of-type{ padding-top:8px; }
|
|
.kicker{
|
|
font-family:'IBM Plex Mono'; font-size:.74rem; letter-spacing:.08em; text-transform:uppercase;
|
|
color:var(--accent-2); margin-bottom:8px;
|
|
}
|
|
h2.h{ font-size:1.55rem; font-weight:800; margin-bottom:6px; }
|
|
h3.h{ font-size:1.15rem; font-weight:700; margin:28px 0 10px 0; }
|
|
h4.h{ font-family:'Archivo'; font-size:.95rem; font-weight:700; margin:18px 0 8px; color:var(--ink); }
|
|
.section-intro{ color:var(--ink-soft); margin-bottom:22px; }
|
|
|
|
/* ---------- cards / panels ---------- */
|
|
.panel{
|
|
background:var(--surface); border:1px solid var(--line); border-radius:12px;
|
|
padding:20px 22px; box-shadow:var(--shadow);
|
|
}
|
|
.grid2{ display:grid; grid-template-columns:1fr 1fr; gap:16px; }
|
|
.grid3{ display:grid; grid-template-columns:repeat(3,1fr); gap:16px; }
|
|
@media (max-width:760px){ .grid2, .grid3{ grid-template-columns:1fr; } }
|
|
|
|
.card{
|
|
background:var(--surface); border:1px solid var(--line); border-radius:10px;
|
|
padding:16px 18px;
|
|
}
|
|
.card h4{ font-family:'Archivo'; font-size:.92rem; font-weight:800; margin:0 0 6px 0; }
|
|
.card p{ font-size:.92rem; color:var(--ink-soft); margin:0; }
|
|
.card .foot{ margin-top:10px; font-size:.78rem; color:var(--ink-faint); font-family:'IBM Plex Mono'; }
|
|
|
|
/* ---------- code / config blocks ---------- */
|
|
pre.code{
|
|
background:var(--code-bg); border:1px solid var(--line); border-radius:8px;
|
|
padding:14px 16px; overflow-x:auto; font-size:.82rem; line-height:1.55;
|
|
color:var(--ink);
|
|
}
|
|
code.inline{
|
|
background:var(--code-bg); border:1px solid var(--line-soft); border-radius:4px;
|
|
padding:.1em .4em; font-size:.87em;
|
|
}
|
|
|
|
/* ---------- tables ---------- */
|
|
.tbl-wrap{ overflow-x:auto; border:1px solid var(--line); border-radius:10px; }
|
|
table{ border-collapse:collapse; width:100%; font-size:.86rem; background:var(--surface); }
|
|
thead th{
|
|
text-align:left; font-family:'IBM Plex Mono'; font-size:.72rem; letter-spacing:.05em; text-transform:uppercase;
|
|
color:var(--ink-faint); background:var(--surface-2); padding:10px 14px; border-bottom:1px solid var(--line);
|
|
white-space:nowrap;
|
|
}
|
|
tbody td{ padding:10px 14px; border-bottom:1px solid var(--line-soft); vertical-align:top; }
|
|
tbody tr:last-child td{ border-bottom:none; }
|
|
tbody td .k{ font-size:.82rem; color:var(--ink); }
|
|
td.num{ font-variant-numeric:tabular-nums; }
|
|
|
|
/* ---------- status pills ---------- */
|
|
.pill{
|
|
display:inline-flex; align-items:center; gap:6px;
|
|
font-family:'IBM Plex Mono'; font-size:.72rem; font-weight:500; letter-spacing:.02em;
|
|
padding:3px 9px; border-radius:100px; border:1px solid transparent; white-space:nowrap;
|
|
}
|
|
.pill::before{ content:''; width:6px; height:6px; border-radius:50%; background:currentColor; }
|
|
.pill.staged{ color:var(--st-staged); background:color-mix(in srgb, var(--st-staged) 14%, transparent); border-color:color-mix(in srgb, var(--st-staged) 30%, transparent);}
|
|
.pill.pushed{ color:var(--st-pushed); background:color-mix(in srgb, var(--st-pushed) 14%, transparent); border-color:color-mix(in srgb, var(--st-pushed) 30%, transparent);}
|
|
.pill.attached{ color:var(--st-attached); background:color-mix(in srgb, var(--st-attached) 14%, transparent); border-color:color-mix(in srgb, var(--st-attached) 30%, transparent);}
|
|
.pill.failed{ color:var(--st-failed); background:color-mix(in srgb, var(--st-failed) 14%, transparent); border-color:color-mix(in srgb, var(--st-failed) 30%, transparent);}
|
|
|
|
.lifecycle-row{ display:flex; align-items:center; flex-wrap:wrap; gap:10px; margin:18px 0; }
|
|
.lifecycle-row .arrow{ color:var(--ink-faint); font-size:1.1rem; }
|
|
|
|
/* ---------- diagrams ---------- */
|
|
figure.diagram{ margin:18px 0 8px 0; }
|
|
figure.diagram .mermaid{
|
|
background:var(--surface); border:1px solid var(--line); border-radius:12px;
|
|
padding:18px; overflow-x:auto; box-shadow:var(--shadow);
|
|
}
|
|
figure.diagram .mermaid svg{ max-width:100%; height:auto; display:block; margin:0 auto; }
|
|
@media (max-width:640px){
|
|
figure.diagram .mermaid svg{ max-width:none; height:auto; } /* allow horizontal scroll instead of squashing dense diagrams unreadable */
|
|
}
|
|
figcaption{ font-size:.82rem; color:var(--ink-faint); margin-top:8px; }
|
|
|
|
/* ---------- callouts ---------- */
|
|
.callout{
|
|
border-left:3px solid var(--accent-warm); background:var(--surface-2);
|
|
border-radius:0 8px 8px 0; padding:12px 16px; font-size:.9rem; color:var(--ink-soft);
|
|
margin:16px 0;
|
|
}
|
|
.callout strong{ color:var(--ink); }
|
|
.callout.danger{ border-left-color:var(--st-failed); }
|
|
.callout.done{ border-left-color:var(--st-attached); }
|
|
|
|
/* ---------- checklist ---------- */
|
|
.checklist{ list-style:none; margin:0; padding:0; display:flex; flex-direction:column; gap:10px; }
|
|
.checklist li{ display:flex; gap:12px; align-items:flex-start; }
|
|
.checklist .num{
|
|
font-family:'IBM Plex Mono'; font-size:.72rem; color:var(--surface); background:var(--accent);
|
|
width:20px; height:20px; border-radius:50%; display:flex; align-items:center; justify-content:center;
|
|
flex:none; margin-top:2px;
|
|
}
|
|
.checklist .body b{ display:block; font-size:.94rem; }
|
|
.checklist .body span{ font-size:.86rem; color:var(--ink-soft); }
|
|
.checklist .tag{
|
|
font-family:'IBM Plex Mono'; font-size:.68rem; text-transform:uppercase; letter-spacing:.05em;
|
|
color:var(--accent-2); margin-left:8px;
|
|
}
|
|
|
|
hr.rule{ border:none; border-top:1px solid var(--line); margin:14px 0; }
|
|
|
|
footer{ border-top:1px solid var(--line); padding:28px 0 60px; color:var(--ink-faint); font-size:.82rem; }
|
|
</style>
|
|
|
|
<div class="topbar">
|
|
<div class="topbar-inner wrap">
|
|
<div class="brand">
|
|
<span class="mark">NSDL SEZ BOE</span>
|
|
<span class="sub">architecture & runbook</span>
|
|
</div>
|
|
<nav>
|
|
<a class="navlink" href="#architecture">Architecture</a>
|
|
<a class="navlink" href="#phase-a">Phase A</a>
|
|
<a class="navlink" href="#phase-b">Phase B</a>
|
|
<a class="navlink" href="#frappe-side">Frappe</a>
|
|
<a class="navlink" href="#configuration">Config</a>
|
|
<a class="navlink" href="#hardening">Hardening</a>
|
|
<a class="navlink" href="#setup-checklist">Setup</a>
|
|
</nav>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="wrap">
|
|
<section class="hero">
|
|
<div class="eyebrow"><span class="dot"></span> TEST stack · ap-south-1 · nsdl-sez-boe-automation-test</div>
|
|
<h1 class="title">NSDL SEZ BOE/DTA Automation</h1>
|
|
<p class="lede">
|
|
A two-Lambda AWS pipeline that logs into NSDL's SEZ Online portal every day, downloads
|
|
Bill of Entry and DTA Sale trade data, quality-checks and stages it per client,
|
|
then pushes it into each client's Canofy/Frappe site as <code class="inline">FTWZ BOE Import</code>
|
|
/ <code class="inline">FTWZ DTA Sale</code> records with the original NSDL PDF attached.
|
|
</p>
|
|
<div class="stat-strip">
|
|
<div class="stat"><div class="n">2</div><div class="l">LAMBDA FUNCTIONS</div></div>
|
|
<div class="stat"><div class="n">3</div><div class="l">DYNAMODB TABLES</div></div>
|
|
<div class="stat"><div class="n">1</div><div class="l">EXTERNAL PORTAL (NSDL)</div></div>
|
|
<div class="stat"><div class="n">6</div><div class="l">QC CHECKS PER PIPELINE</div></div>
|
|
<div class="stat"><div class="n">900s</div><div class="l">LAMBDA TIMEOUT (MAX)</div></div>
|
|
<div class="stat"><div class="n">N</div><div class="l">CLIENT SITES (MULTI-TENANT)</div></div>
|
|
</div>
|
|
</section>
|
|
|
|
<div class="layout">
|
|
<aside class="rail">
|
|
<div class="rail-title">Contents</div>
|
|
<a href="#overview">Overview</a>
|
|
<a href="#architecture">Architecture at a glance</a>
|
|
<a href="#lifecycle">Record lifecycle</a>
|
|
<a href="#phase-a">Phase A — Fetch</a>
|
|
<a class="sub" href="#phase-a-trigger">Trigger & date window</a>
|
|
<a class="sub" href="#phase-a-login">Login & export</a>
|
|
<a class="sub" href="#phase-a-qc">Parse, partition, QC</a>
|
|
<a href="#phase-b">Phase B — Deliver</a>
|
|
<a class="sub" href="#phase-b-trigger">Trigger</a>
|
|
<a class="sub" href="#phase-b-push">Push to Frappe</a>
|
|
<a class="sub" href="#phase-b-pdf">PDF fetch & attach</a>
|
|
<a href="#watchdog">Watchdog</a>
|
|
<a href="#frappe-side">Frappe-side integration</a>
|
|
<a href="#configuration">Configuration reference</a>
|
|
<a href="#hardening">Hardening notes</a>
|
|
<a href="#setup-checklist">Setup checklist</a>
|
|
<a href="#open-items">Open items</a>
|
|
</aside>
|
|
|
|
<main>
|
|
<section id="overview">
|
|
<div class="kicker">00 — Overview</div>
|
|
<h2 class="h">What this system does, in one pass</h2>
|
|
<p class="section-intro">
|
|
Every SEZ/FTWZ unit's import (Bill of Entry) and domestic-tariff-area sale (DTA Sale)
|
|
transactions are recorded on NSDL's government SEZ Online portal, one entry at a time,
|
|
with no bulk export API — only a browser UI. This automation replaces the manual
|
|
"log in, export, re-key into Frappe, print, re-attach the PDF" routine with two
|
|
scheduled Lambda functions either side of a DynamoDB staging table.
|
|
</p>
|
|
<div class="grid3">
|
|
<div class="card">
|
|
<h4>Phase A — Fetch</h4>
|
|
<p>Once a day, logs into NSDL, exports BE + DTA for a rolling 7-day window, quality-checks
|
|
every request, and stages new records per client in DynamoDB.</p>
|
|
<div class="foot">PhaseAFunction · EventBridge daily</div>
|
|
</div>
|
|
<div class="card">
|
|
<h4>Phase B — Deliver</h4>
|
|
<p>Triggered per-client by that client's own Frappe site. Pushes staged records into Canofy,
|
|
then logs back into NSDL to fetch and attach the official PDF for each BE record.</p>
|
|
<div class="foot">PhaseBFunction · API Gateway</div>
|
|
</div>
|
|
<div class="card">
|
|
<h4>Watchdog</h4>
|
|
<p>Runs hourly. Flags any client whose records have sat in <code class="inline">STAGED</code>
|
|
for more than 24 hours — either a dead Frappe scheduler or a chronically failing push.</p>
|
|
<div class="foot">WatchdogFunction · rate(1 hour)</div>
|
|
</div>
|
|
</div>
|
|
<div class="callout">
|
|
<strong>Multi-tenant by design.</strong> One AWS deployment serves every client. Phase A stages
|
|
records for <em>all</em> configured clients from a single NSDL login; Phase B is called
|
|
separately by each client's own Frappe site, identifying itself via an
|
|
<code class="inline">X-Frappe-Client</code> header so Phase B knows whose records to push
|
|
and which Frappe credentials to use.
|
|
</div>
|
|
</section>
|
|
|
|
<section id="architecture">
|
|
<div class="kicker">01 — Architecture</div>
|
|
<h2 class="h">Architecture at a glance</h2>
|
|
<p class="section-intro">Every resource in the deployed CloudFormation stack, and how data moves between them.</p>
|
|
<figure class="diagram">
|
|
<pre class="mermaid">
|
|
flowchart LR
|
|
NSDL[["NSDL SEZ Online Portal<br/>(external, no API)"]]
|
|
|
|
subgraph AWS["AWS · nsdl-sez-boe-automation-test"]
|
|
direction TB
|
|
EB["EventBridge<br/>cron, daily"] --> PA["PhaseAFunction<br/>Playwright + EasyOCR"]
|
|
PA --> S3["S3 RawFilesBucket<br/>raw BE/DTA exports"]
|
|
PA --> RCT["RunControlTable<br/>per-day idempotency"]
|
|
PA --> SRT[("StagedRecordsTable<br/>per-record status")]
|
|
PA --> RGT["RunGateTable<br/>per-client QC summary"]
|
|
|
|
APIGW["API Gateway<br/>/sez/deliver-pending"] --> PB["PhaseBFunction<br/>Playwright + EasyOCR"]
|
|
PB --> SRT
|
|
PB -.async failure.-> DLQ["SQS DLQ"] --> ALM1["CloudWatch Alarm"]
|
|
|
|
WD["WatchdogFunction<br/>hourly"] --> SRT
|
|
WD --> SNS(["SNS AlertsTopic"])
|
|
ALM1 --> SNS
|
|
ALM2["CloudWatch Alarm<br/>PhaseA Errors"] --> SNS
|
|
PA -.errors.-> ALM2
|
|
|
|
SEC[("Secrets Manager<br/>NSDL creds + per-client Frappe creds")]
|
|
PA --> SEC
|
|
PB --> SEC
|
|
end
|
|
|
|
PA <--> NSDL
|
|
PB <--> NSDL
|
|
PB <--> FRAPPE["Client Frappe / Canofy site<br/>FTWZ BOE Import · FTWZ DTA Sale"]
|
|
FRAPPE -- "daily scheduled job" --> APIGW
|
|
SNS --> HUMAN["Email / Slack<br/>(human subscribes post-deploy)"]
|
|
|
|
style SRT fill:#9a5f1b22,stroke:#9a5f1b
|
|
style NSDL fill:#1f4e5c22,stroke:#1f4e5c
|
|
style FRAPPE fill:#1f4e5c22,stroke:#1f4e5c
|
|
</pre>
|
|
<figcaption>Fig. 1 — Every resource in the CloudFormation stack (<code class="inline">template.yaml</code>) and the two external systems it bridges: NSDL and each client's Frappe site.</figcaption>
|
|
</figure>
|
|
<div class="grid2">
|
|
<div class="card">
|
|
<h4>Why two identical-looking Lambdas</h4>
|
|
<p>Both PhaseA and PhaseB ship the same heavy container image (Playwright + Chromium + EasyOCR + pandas/opencv,
|
|
~4GB) because both need a real NSDL browser session — Phase A to export data, Phase B to fetch each
|
|
record's official PDF. Neither is a lightweight HTTP function.</p>
|
|
</div>
|
|
<div class="card">
|
|
<h4>Why DynamoDB sits in the middle</h4>
|
|
<p>Phase A never talks to Frappe, and Phase B never talks to NSDL's export screens. <code class="inline">StagedRecordsTable</code>
|
|
is the sole handoff — this is what lets Phase A run once for every client while each client's Phase B call
|
|
stays independent and retry-safe.</p>
|
|
</div>
|
|
</div>
|
|
</section>
|
|
|
|
<section id="lifecycle">
|
|
<div class="kicker">02 — Data model</div>
|
|
<h2 class="h">A record's life, in one row of StagedRecordsTable</h2>
|
|
<p class="section-intro">
|
|
Every NSDL request (identified by its <code class="inline">request_id</code>) becomes exactly one row,
|
|
hashed on <code class="inline">request_id</code> with a <code class="inline">client_name-status-index</code>
|
|
GSI for Phase B's per-client queries.
|
|
</p>
|
|
<div class="lifecycle-row">
|
|
<span class="pill staged">STAGED</span><span class="arrow">→</span>
|
|
<span class="pill pushed">PUSHED</span><span class="arrow">→</span>
|
|
<span class="pill attached">ATTACHED</span>
|
|
<span class="arrow" style="margin-left:24px">/</span>
|
|
<span class="pill failed">QC_MISMATCH</span>
|
|
<span class="arrow">·</span>
|
|
<span class="pill failed">error_type set, status unchanged</span>
|
|
</div>
|
|
<figure class="diagram">
|
|
<pre class="mermaid">
|
|
stateDiagram-v2
|
|
[*] --> STAGED: Phase A stages a new,<br/>QC-passed request_id
|
|
STAGED --> QC_MISMATCH: check_6 IEC consistency<br/>failed for this group
|
|
STAGED --> PUSHED: Phase B — Frappe record<br/>created or found (get_record)
|
|
PUSHED --> ATTACHED: Phase B — NSDL PDF<br/>fetched and attached
|
|
STAGED --> STAGED: push failed —<br/>error_type/retry_count set,<br/>status unchanged, retried next run
|
|
PUSHED --> PUSHED: PDF fetch/attach failed —<br/>same retry-in-place pattern
|
|
QC_MISMATCH --> [*]
|
|
ATTACHED --> [*]
|
|
</pre>
|
|
<figcaption>Fig. 2 — A failed push or PDF attach never moves the record to a dead-end status: it stays exactly where it was,
|
|
with <code class="inline">error_type</code> / <code class="inline">error_message</code> / <code class="inline">retry_count</code>
|
|
set, so the next invocation retries it automatically. The Watchdog is what catches a record stuck like this for too long.</figcaption>
|
|
</figure>
|
|
<div class="tbl-wrap">
|
|
<table>
|
|
<thead><tr><th>Field</th><th>Meaning</th></tr></thead>
|
|
<tbody>
|
|
<tr><td class="k">request_id</td><td>NSDL's own request ID — the table's hash key, and the natural idempotency key across every stage.</td></tr>
|
|
<tr><td class="k">client_name</td><td>Which onboarded client this belongs to — the GSI hash key for Phase B's per-client queries.</td></tr>
|
|
<tr><td class="k">pipeline</td><td><code class="inline">BE</code> or <code class="inline">DTA</code> — decides the Frappe doctype and (formerly) the PDF step; both now share one path.</td></tr>
|
|
<tr><td class="k">status</td><td>STAGED · PUSHED · ATTACHED · QC_MISMATCH (see Fig. 2).</td></tr>
|
|
<tr><td class="k">payload</td><td>The full JSON-safe Frappe document body, built once at staging time and never re-derived.</td></tr>
|
|
<tr><td class="k">frappe_docname</td><td>Set on PUSHED — the created/found Frappe document's real name, needed for the PDF-attach call.</td></tr>
|
|
<tr><td class="k">error_type / error_message / retry_count</td><td>Set by a failed attempt; status is deliberately left unchanged (see Fig. 2 caption).</td></tr>
|
|
<tr><td class="k">ttl</td><td>Only ever set on ATTACHED (<code class="inline">STAGED_TTL_DAYS</code>, default 60) — non-terminal rows must never expire mid-retry.</td></tr>
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
</section>
|
|
|
|
<section id="phase-a">
|
|
<div class="kicker">03 — Phase A</div>
|
|
<h2 class="h">Phase A — Fetch</h2>
|
|
<p class="section-intro"><code class="inline">app/phase_a_fetch/</code> · one Lambda, one daily invocation, every client at once.</p>
|
|
|
|
<figure class="diagram">
|
|
<pre class="mermaid">
|
|
sequenceDiagram
|
|
participant EB as EventBridge
|
|
participant PA as PhaseAFunction
|
|
participant RCT as RunControlTable
|
|
participant NSDL as NSDL Portal
|
|
participant S3 as S3
|
|
participant Parse as Parser + QC
|
|
participant SRT as StagedRecordsTable
|
|
participant RGT as RunGateTable
|
|
|
|
EB->>PA: cron daily · {force_refresh:false}
|
|
PA->>RCT: get_run_control(run_date)
|
|
alt already PASSED today
|
|
RCT-->>PA: reuse existing S3 files
|
|
else not yet run
|
|
PA->>NSDL: login (Playwright + EasyOCR CAPTCHA)
|
|
PA->>NSDL: export BILL OF ENTRY + DTA SALE
|
|
PA->>S3: store raw exports
|
|
PA->>RCT: mark PASSED
|
|
end
|
|
PA->>Parse: read_nsdl_export, partition_by_client(IEC map)
|
|
Parse->>Parse: 6 QC checks per client per pipeline
|
|
Parse->>SRT: put_staged_record (new request_ids only)
|
|
Parse->>RGT: put_run_gate (per-client QC summary)
|
|
</pre>
|
|
<figcaption>Fig. 3 — One Phase A invocation, from EventBridge's static payload to two DynamoDB writes.</figcaption>
|
|
</figure>
|
|
|
|
<h3 class="h" id="phase-a-trigger">Trigger & the date window</h3>
|
|
<p>
|
|
EventBridge's <code class="inline">Schedule</code> target can only carry a static
|
|
<code class="inline">Input</code>, so it always sends <code class="inline">{"force_refresh": false}</code> —
|
|
no <code class="inline">run_date</code>. <code class="inline">lambda_handler.py</code> defaults it to
|
|
today's UTC date, then computes the actual NSDL query window:
|
|
</p>
|
|
<pre class="code">to_date = run_date - 1 day # NSDL's own data lags by one day
|
|
from_date = to_date - 7 days # rolling 7-day window
|
|
# entered into NSDL's date fields as DD/MM/YYYY</pre>
|
|
<div class="callout">
|
|
<strong>The one-day lag is load-bearing.</strong> NSDL's export for "today" is never fully populated
|
|
yet. Querying up through <code class="inline">run_date</code> itself silently returns incomplete data —
|
|
this was found and fixed against the real portal, not assumed.
|
|
</div>
|
|
|
|
<h3 class="h" id="phase-a-login">Login & export</h3>
|
|
<p>
|
|
<code class="inline">nsdl_download.py</code> drives a real headless Chromium session: fills username/password,
|
|
solves the CAPTCHA via <code class="inline">ocr_engine.py</code> (four OpenCV preprocessing variants fed to
|
|
EasyOCR, confidence-weighted character-by-character consensus across variants), retries login up to
|
|
<code class="inline">MAX_LOGIN_ATTEMPTS</code> times, then exports both modules
|
|
(<code class="inline">NSDL_MODULE_BE=17</code>, <code class="inline">NSDL_MODULE_DTA=26</code>) to
|
|
tab-delimited <code class="inline">.xls</code> files.
|
|
</p>
|
|
|
|
<h3 class="h" id="phase-a-qc">Parse, partition, six QC checks</h3>
|
|
<p>
|
|
<code class="inline">nsdl_parser.py</code> groups rows by <code class="inline">REQUEST ID</code>, resolves
|
|
each row's <code class="inline">CLIENT IEC</code> against the secret's reverse IEC→client map (an IEC
|
|
mapped to two different clients aborts the entire run — the one error class this pipeline never
|
|
resolves silently), and runs six checks per pipeline per client:
|
|
</p>
|
|
<div class="grid3">
|
|
<div class="card"><h4>1 · Header count</h4><p>Input rows vs. grouped output headers reconcile.</p></div>
|
|
<div class="card"><h4>2 · Invoice count</h4><p>Every invoice line survives grouping.</p></div>
|
|
<div class="card"><h4>3 · Item count</h4><p>Every item line survives grouping.</p></div>
|
|
<div class="card"><h4>4 · Shipment count</h4><p>Non-null shipment rows only (per BE/DTA drop rule).</p></div>
|
|
<div class="card"><h4>5 · Sum checks</h4><p>Quantity, BCD, assessable value totals match pre/post transform.</p></div>
|
|
<div class="card"><h4>6 · IEC consistency</h4><p>Every row in one request_id agrees on CLIENT IEC after normalization.</p></div>
|
|
</div>
|
|
<p>
|
|
A failed check 6 group is staged with status <code class="inline">QC_MISMATCH</code> and excluded
|
|
from the batch. Any other failing check either aborts staging for that client/pipeline
|
|
(<code class="inline">STEP2_QC_STRICT=true</code>, the default) or stages anyway with a logged warning.
|
|
</p>
|
|
</section>
|
|
|
|
<section id="phase-b">
|
|
<div class="kicker">04 — Phase B</div>
|
|
<h2 class="h">Phase B — Deliver</h2>
|
|
<p class="section-intro"><code class="inline">app/phase_b_deliver/</code> · one Lambda, invoked independently per client.</p>
|
|
|
|
<figure class="diagram">
|
|
<pre class="mermaid">
|
|
sequenceDiagram
|
|
participant FR as Client Frappe site
|
|
participant NSR as NSDL Sync Run doc
|
|
participant APIGW as API Gateway
|
|
participant PB as PhaseBFunction
|
|
participant SRT as StagedRecordsTable
|
|
participant CAN as Canofy (Frappe API)
|
|
participant NSDL as NSDL Portal
|
|
|
|
FR->>NSR: insert(status=Triggered)
|
|
FR->>APIGW: POST /sez/deliver-pending<br/>X-Frappe-Client, {sync_run_name}
|
|
APIGW->>PB: invoke — Event (async), 202 immediately
|
|
PB->>SRT: query_actionable_by_client (STAGED + PUSHED)
|
|
loop each STAGED record
|
|
PB->>CAN: get_record → found? reuse : create_record
|
|
PB->>SRT: mark_pushed(frappe_docname)
|
|
end
|
|
PB->>NSDL: login (once for the whole batch)
|
|
loop pending PDFs, capped at MAX_PDF_ATTACHES_PER_INVOCATION
|
|
PB->>NSDL: fetch print-PDF for request_id
|
|
PB->>CAN: attach_pdf
|
|
PB->>SRT: mark_attached(ttl)
|
|
Note over PB,NSDL: every 15th download: 120s<br/>keep-alive-scroll pause
|
|
end
|
|
PB->>NSR: update_audit_doc(status, counts)
|
|
</pre>
|
|
<figcaption>Fig. 4 — One Phase B invocation for one client. The async 202 means Frappe never waits on this; the real
|
|
outcome lands later via the NSDL Sync Run doc update.</figcaption>
|
|
</figure>
|
|
|
|
<h3 class="h" id="phase-b-trigger">Why the call is asynchronous</h3>
|
|
<p>
|
|
API Gateway's REST integration caps a <em>synchronous</em> Lambda proxy at 29 seconds. A real NSDL login plus
|
|
throttled PDF downloads routinely runs minutes. The integration sets
|
|
<code class="inline">X-Amz-Invocation-Type: Event</code>, so Frappe's scheduled job gets a bare
|
|
<code class="inline">202 {"accepted": true}</code> the instant AWS accepts the trigger — the actual delivery
|
|
result arrives later, written back onto the same <code class="inline">NSDL Sync Run</code> document.
|
|
</p>
|
|
|
|
<h3 class="h" id="phase-b-push">Idempotent push to Frappe</h3>
|
|
<p>
|
|
Before creating anything, Phase B calls <code class="inline">frappe.get_record(doctype, request_id)</code>.
|
|
Only if Frappe has nothing for that <code class="inline">request_id</code> does it call
|
|
<code class="inline">create_record</code>. This is the only check that touches Frappe's real state —
|
|
DynamoDB's own status is what decides whether Phase B even looks at a record at all.
|
|
</p>
|
|
<div class="callout done">
|
|
<strong>Proven under an actual duplicate invocation.</strong> A production Lambda timeout mid-run triggered
|
|
an automatic retry with the same 64-record batch. The retry pushed <strong>zero</strong> duplicate Frappe
|
|
records — every one was already <code class="inline">PUSHED</code> — and correctly resumed PDF-attach for
|
|
only the ones still outstanding.
|
|
</div>
|
|
|
|
<h3 class="h" id="phase-b-pdf">PDF fetch & attach — one path for BE and DTA</h3>
|
|
<p>
|
|
<code class="inline">pdf_fetch.py</code> drives <code class="inline">Search Request → request_id →
|
|
Print</code> on NSDL and renders the result to PDF via the browser's own print-to-PDF. BE and DTA requests
|
|
use the <em>identical</em> navigation — an earlier build assumed DTA had no printable document at all; that
|
|
assumption was wrong and has since been corrected (see <a href="#hardening">Hardening</a>).
|
|
</p>
|
|
<p>
|
|
Every <code class="inline">PDF_PAUSE_EVERY</code> (15) downloads, the batch pauses
|
|
<code class="inline">PDF_PAUSE_SECONDS</code> (120s) — not idle, but scrolling the page gently, because a
|
|
fully idle tab was observed tripping NSDL's own session timeout mid-run. A hard <code class="inline">
|
|
MAX_PDF_ATTACHES_PER_INVOCATION</code> (30) cap keeps one invocation inside Lambda's 900s ceiling on a
|
|
large backlog; anything past the cap stays <code class="inline">PUSHED</code> for the next invocation.
|
|
</p>
|
|
</section>
|
|
|
|
<section id="watchdog">
|
|
<div class="kicker">05 — Watchdog</div>
|
|
<h2 class="h">Watchdog</h2>
|
|
<p class="section-intro"><code class="inline">app/watchdog/</code> · a plain zip Lambda, no browser needed.</p>
|
|
<p>
|
|
Every hour, scans <code class="inline">StagedRecordsTable</code> for rows still <code class="inline">STAGED</code>
|
|
more than 24 hours after <code class="inline">staged_at</code>, groups the count by client, and publishes one
|
|
SNS message naming every affected client. One check, two real failure modes: a client's Frappe scheduler has
|
|
silently stopped calling Phase B at all, <em>or</em> Phase B is repeatedly failing to push a specific record
|
|
(which — by design — never changes that record's status, so it looks identical to "nobody's asking" from here).
|
|
</p>
|
|
</section>
|
|
|
|
<section id="frappe-side">
|
|
<div class="kicker">06 — Frappe integration</div>
|
|
<h2 class="h">The Frappe side</h2>
|
|
<p class="section-intro">What lives in the client's own Canofy/Frappe app (<code class="inline">canofy_tcs</code>), not in this AWS stack.</p>
|
|
|
|
<figure class="diagram">
|
|
<pre class="mermaid">
|
|
flowchart LR
|
|
HOOKS["hooks.py<br/>scheduler_events.daily"] --> JOB["trigger_nsdl_sync()"]
|
|
JOB --> CFG[("API Configuration - Canofy<br/>doc: NSDL SEZ DELIVER PENDING")]
|
|
CFG -.reads url / api_key / api_secret<br/>+ X-Frappe-Client header.-> JOB
|
|
JOB --> NSR["NSDL Sync Run<br/>insert(status=Triggered)"]
|
|
JOB --> POST["make_post_request()"]
|
|
POST --> AWS["AWS API Gateway"]
|
|
AWS -. later, async .-> NSR2["Phase B updates the<br/>same NSDL Sync Run doc"]
|
|
</pre>
|
|
<figcaption>Fig. 5 — Everything this integration needs — endpoint, credentials, and this site's own client identity —
|
|
lives in one existing doctype record; nothing lives in <code class="inline">site_config.json</code>.</figcaption>
|
|
</figure>
|
|
|
|
<div class="grid2">
|
|
<div class="card">
|
|
<h4>NSDL Sync Run (new DocType)</h4>
|
|
<p>One audit row per trigger: <code class="inline">run_date</code>, <code class="inline">client_name</code>,
|
|
<code class="inline">status</code> (Triggered → Delivered / No Pending Records / Failed),
|
|
<code class="inline">records_delivered</code>, <code class="inline">pdfs_attached</code>,
|
|
<code class="inline">error_message</code>. Created by the Frappe job, updated later by Phase B.</p>
|
|
<div class="foot">canofy_tcs/import_documents/doctype/nsdl_sync_run/</div>
|
|
</div>
|
|
<div class="card">
|
|
<h4>API Configuration - Canofy (existing DocType, reused)</h4>
|
|
<p>One record named <code class="inline">NSDL SEZ DELIVER PENDING</code> holds the deployed API URL, the
|
|
API key/secret, <em>and</em> this site's <code class="inline">X-Frappe-Client</code> value inside its
|
|
existing "Additional Static Headers" JSON field — no new field, no file-system config.</p>
|
|
<div class="foot">canofy_tcs/canofy_tcs/doctype/api_configuration___canofy/</div>
|
|
</div>
|
|
</div>
|
|
<p style="margin-top:16px">
|
|
Deliberately <strong>not</strong> routed through the existing <code class="inline">Canofy API Call</code>
|
|
auto-dispatch controller — that controller only ever fires for API configs whose name/service contains
|
|
<code class="inline">"EWAY"</code> (hardcoded), so it would silently do nothing for an NSDL config without
|
|
editing shared, already-in-production code. <code class="inline">nsdl_sync_scheduled_job.py</code> instead
|
|
reuses the two genuinely generic pieces: <code class="inline">frappe.integrations.utils.make_post_request</code>
|
|
and the <code class="inline">API Configuration - Canofy</code> doctype as a config store.
|
|
</p>
|
|
</section>
|
|
|
|
<section id="configuration">
|
|
<div class="kicker">07 — Reference</div>
|
|
<h2 class="h">Configuration reference</h2>
|
|
<p class="section-intro">Every value that shapes this system's behavior, and where it lives.</p>
|
|
|
|
<h3 class="h">CloudFormation parameters — human-confirmed, never guessed</h3>
|
|
<div class="tbl-wrap">
|
|
<table>
|
|
<thead><tr><th>Parameter</th><th>Purpose</th><th>Current (TEST)</th></tr></thead>
|
|
<tbody>
|
|
<tr><td class="k">Environment</td><td>TEST or PROD — suffixes every named resource so both stacks can coexist.</td><td class="num">TEST</td></tr>
|
|
<tr><td class="k">S3RetentionDays</td><td>Days before a raw NSDL export expires from S3. No default — pending a real SEZ/customs recordkeeping minimum.</td><td class="num">90</td></tr>
|
|
<tr><td class="k">DailyScheduleCronExpression</td><td>EventBridge schedule for Phase A. No default — needs a confirmed business UTC time.</td><td class="num">cron(0 9 * * ? *)</td></tr>
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
|
|
<h3 class="h">PhaseAFunction environment</h3>
|
|
<div class="tbl-wrap">
|
|
<table>
|
|
<thead><tr><th>Variable</th><th>Value / source</th><th>Notes</th></tr></thead>
|
|
<tbody>
|
|
<tr><td class="k">RAW_FILES_BUCKET</td><td class="mono">!Ref RawFilesBucket</td><td>Physical S3 bucket name</td></tr>
|
|
<tr><td class="k">RUN_CONTROL_TABLE</td><td class="mono">!Ref RunControlTable</td><td>Per-day idempotency</td></tr>
|
|
<tr><td class="k">STAGED_RECORDS_TABLE</td><td class="mono">!Ref StagedRecordsTable</td><td>Shared with Phase B</td></tr>
|
|
<tr><td class="k">RUN_GATE_TABLE</td><td class="mono">!Ref RunGateTable</td><td>Per-client QC summary view</td></tr>
|
|
<tr><td class="k">NSDL_BOE_SECRET_NAME</td><td class="mono">!Ref NsdlBoeSecrets (ARN)</td><td>boto3 accepts ARN or name</td></tr>
|
|
<tr><td class="k">NSDL_BASE_URL</td><td class="mono">sezonline-ndml.co.in/User/Login.aspx</td><td>fw_verify's proven default</td></tr>
|
|
<tr><td class="k">NSDL_MODULE_BE / NSDL_MODULE_DTA</td><td class="mono">17 / 26</td><td>NSDL's own module dropdown values</td></tr>
|
|
<tr><td class="k">NSDL_ACCOUNT_KEY</td><td class="mono">siddhartha_ftwz</td><td>Key into the secret's NSDL_ACCOUNTS block</td></tr>
|
|
<tr><td class="k">MAX_LOGIN_ATTEMPTS / MAX_CAPTCHA_READ_ATTEMPTS</td><td class="mono">8 / 6</td><td>Retry ceilings</td></tr>
|
|
<tr><td class="k">STEP2_QC_STRICT</td><td class="mono">true</td><td>Withhold a client/pipeline's records on any failed check</td></tr>
|
|
<tr><td class="k">HOME</td><td class="mono">/tmp</td><td>Real Lambda sandbox user's own $HOME is read-only — see Hardening</td></tr>
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
|
|
<h3 class="h">PhaseBFunction environment</h3>
|
|
<div class="tbl-wrap">
|
|
<table>
|
|
<thead><tr><th>Variable</th><th>Value / source</th><th>Notes</th></tr></thead>
|
|
<tbody>
|
|
<tr><td class="k">STAGED_RECORDS_TABLE / NSDL_BOE_SECRET_NAME / NSDL_BASE_URL / NSDL_ACCOUNT_KEY / MAX_LOGIN_ATTEMPTS / MAX_CAPTCHA_READ_ATTEMPTS</td><td colspan="2">Same as Phase A.</td></tr>
|
|
<tr><td class="k">STAGED_TTL_DAYS</td><td class="mono">60</td><td>TTL applied only at ATTACHED</td></tr>
|
|
<tr><td class="k">PDF_PAUSE_EVERY / PDF_PAUSE_SECONDS</td><td class="mono">15 / 120</td><td>Throttle to avoid NSDL session expiry</td></tr>
|
|
<tr><td class="k">MAX_PDF_ATTACHES_PER_INVOCATION</td><td class="mono">30</td><td>Caps one invocation's PDF work under the 900s ceiling</td></tr>
|
|
<tr><td class="k">HOME</td><td class="mono">/tmp</td><td>Same real-Lambda constraint as Phase A</td></tr>
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
|
|
<h3 class="h">Secrets Manager shape — <code class="inline">nsdl-boe-automation-secrets-{Environment}</code></h3>
|
|
<pre class="code">{
|
|
"NSDL_ACCOUNTS": {
|
|
"siddhartha_ftwz": {
|
|
"NSDL_USERNAME": "",
|
|
"NSDL_PASSWORD": "",
|
|
"entity_id": "" // NSDL's own SEZ-unit ID, shown per-row in every export
|
|
}
|
|
},
|
|
"CLIENTS": {
|
|
"eap": {
|
|
"client_iec_codes": ["300014953", "0300014953"],
|
|
"FRAPPE_URL": "https://eapx.canofy.tech",
|
|
"api_key": "",
|
|
"api_secret": ""
|
|
}
|
|
}
|
|
}</pre>
|
|
<p>Real credentials are never generated or guessed by this codebase — populated post-deploy via
|
|
<code class="inline">aws secretsmanager put-secret-value</code>. An IEC listed under two clients aborts the
|
|
whole Phase A run rather than resolving silently.</p>
|
|
</section>
|
|
|
|
<section id="hardening">
|
|
<div class="kicker">08 — Hardening</div>
|
|
<h2 class="h">Hardening notes — Docker-local vs. real Lambda</h2>
|
|
<p class="section-intro">
|
|
Every one of these passed cleanly under <code class="inline">sam local invoke</code> and then failed the
|
|
first time it ran as the actually-deployed Lambda — the gap between a normal Docker container and AWS
|
|
Lambda's sandboxed execution environment. Fixed and verified against real invocations, in the order found.
|
|
</p>
|
|
<div class="tbl-wrap">
|
|
<table>
|
|
<thead><tr><th>#</th><th>Symptom</th><th>Root cause</th><th>Fix</th></tr></thead>
|
|
<tbody>
|
|
<tr><td class="num">1</td><td>numpy bool/int not JSON-serializable, crashing a DynamoDB write</td><td><code class="inline">_dynamo_safe()</code> only handled <code class="inline">float</code></td><td>Extended to <code class="inline">np.bool_</code> / <code class="inline">np.integer</code></td></tr>
|
|
<tr><td class="num">2</td><td>Lambda return-value marshaling error</td><td>DynamoDB's <code class="inline">Decimal</code> + raw numpy QC values in the response body</td><td><code class="inline">make_json_safe()</code> + explicit <code class="inline">int(version)</code></td></tr>
|
|
<tr><td class="num">3</td><td><code class="inline">BrowserContext.new_page</code>: target closed</td><td>Lambda's real <code class="inline">/dev/shm</code> is far smaller than a local container's</td><td><code class="inline">--disable-dev-shm-usage</code></td></tr>
|
|
<tr><td class="num">4</td><td>Playwright rejects a raw <code class="inline">--user-data-dir</code> arg</td><td><code class="inline">.launch()</code> doesn't accept that flag directly</td><td>Switched to <code class="inline">launch_persistent_context()</code></td></tr>
|
|
<tr><td class="num">5</td><td><code class="inline">DropAllCapabilitiesOnCurrentThread()</code> check failed</td><td>Chromium's zygote fork model isn't permitted in Lambda's container</td><td><code class="inline">--no-zygote --disable-gpu</code></td></tr>
|
|
<tr><td class="num">6</td><td><code class="inline">[Errno 30] Read-only file system: /home/sbx_user1051</code></td><td>Lambda runs as a non-root sandbox user; only <code class="inline">/tmp</code> is writable, but EasyOCR/torch/Chromium all assume a writable <code class="inline">$HOME</code></td><td><code class="inline">HOME=/tmp</code> set globally as a Lambda env var</td></tr>
|
|
<tr><td class="num">7</td><td><code class="inline">Object of type Decimal is not JSON serializable</code> pushing to Frappe</td><td>boto3 always deserializes DynamoDB numbers as <code class="inline">Decimal</code> on read</td><td><code class="inline">make_json_safe()</code> extended for <code class="inline">Decimal</code>, applied at the read boundary</td></tr>
|
|
<tr><td class="num">8</td><td>DTA Sale records never got a PDF</td><td>Wrong assumption baked into an earlier build — NSDL's print flow is identical for BE and DTA</td><td>Both pipelines now wait at <code class="inline">PUSHED</code> and share one PDF-attach path</td></tr>
|
|
<tr><td class="num">9</td><td>Incomplete data for "today"</td><td>NSDL's own export lags one day behind</td><td><code class="inline">to_date = run_date - 1</code></td></tr>
|
|
<tr><td class="num">10</td><td>900s hard timeout on a large PDF backlog</td><td>~9s/PDF plus periodic 120s pauses adds up fast</td><td><code class="inline">MAX_PDF_ATTACHES_PER_INVOCATION</code> cap; excess safely deferred (idempotent resume)</td></tr>
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
</section>
|
|
|
|
<section id="setup-checklist">
|
|
<div class="kicker">09 — Setup</div>
|
|
<h2 class="h">Setup checklist — what's done, what's left</h2>
|
|
<ol class="checklist">
|
|
<li><span class="num">1</span><div class="body"><b>Deploy the AWS stack <span class="tag">done</span></b><span><code class="inline">sam build && sam deploy</code> with <code class="inline">S3RetentionDays</code> and <code class="inline">DailyScheduleCronExpression</code> supplied.</span></div></li>
|
|
<li><span class="num">2</span><div class="body"><b>Populate real secrets <span class="tag">done</span></b><span>NSDL TEST credentials + the eap client's Frappe URL/API key/secret, via <code class="inline">put-secret-value</code>.</span></div></li>
|
|
<li><span class="num">3</span><div class="body"><b>Verify Phase A + Phase B end-to-end <span class="tag">done</span></b><span>Real NSDL login, real export, real Frappe push, real PDF attach — all verified against the actual deployed Lambdas, not just local invokes.</span></div></li>
|
|
<li><span class="num">4</span><div class="body"><b>Add NSDL Sync Run + register the scheduled job <span class="tag">done — local dev bench</span></b><span>Migrated onto the local <code class="inline">frappe.local</code> dev site. Not yet deployed to whatever actually hosts <code class="inline">eapx.canofy.tech</code> in production.</span></div></li>
|
|
<li><span class="num">5</span><div class="body"><b>Deploy those same file changes to production Frappe <span class="tag">outstanding</span></b><span>Real production hosting path for <code class="inline">canofy_tcs</code> is not yet identified from this session.</span></div></li>
|
|
<li><span class="num">6</span><div class="body"><b>Create the API Configuration - Canofy record <span class="tag">outstanding</span></b><span>Named <code class="inline">NSDL SEZ DELIVER PENDING</code>, with the real deployed API URL, the real API Gateway key value, and <code class="inline">{"X-Frappe-Client": "eap"}</code> in its headers field.</span></div></li>
|
|
<li><span class="num">7</span><div class="body"><b>Subscribe to AlertsTopic <span class="tag">outstanding</span></b><span>Nobody receives Watchdog/error alarms until a real email or Slack endpoint is subscribed.</span></div></li>
|
|
</ol>
|
|
</section>
|
|
|
|
<section id="open-items">
|
|
<div class="kicker">10 — Open items</div>
|
|
<h2 class="h">Known gaps, worth deciding on deliberately</h2>
|
|
<div class="grid2">
|
|
<div class="card">
|
|
<h4>No self-healing against a manually-deleted Frappe doc</h4>
|
|
<p>DynamoDB's <code class="inline">ATTACHED</code> status is permanent ground truth — if a Frappe record is
|
|
deleted by hand afterward, nothing re-checks or re-creates it automatically.</p>
|
|
</div>
|
|
<div class="card">
|
|
<h4>MAX_PDF_ATTACHES_PER_INVOCATION=30 is a first estimate</h4>
|
|
<p>Conservative, based on one real 64-record/31-PDF run. Worth revisiting once real multi-day,
|
|
multi-client volume data exists.</p>
|
|
</div>
|
|
<div class="card">
|
|
<h4>Cold-start EasyOCR model download</h4>
|
|
<p>A fresh execution environment's <code class="inline">/tmp</code> starts empty, so every cold start
|
|
re-downloads the OCR detection model (~minutes) before it can even attempt a CAPTCHA.</p>
|
|
</div>
|
|
<div class="card">
|
|
<h4>Unmapped IEC volume</h4>
|
|
<p>Real NSDL data in this window included large row counts under IECs not yet configured for any
|
|
onboarded client — correctly skipped, but worth checking whether any belong to a client to onboard.</p>
|
|
</div>
|
|
</div>
|
|
</section>
|
|
|
|
<footer>
|
|
Generated from the deployed <code class="inline">nsdl-sez-boe-automation-test</code> stack and its source
|
|
under <code class="inline">D:\Client Projects\EAP\NSDL_SEZ_AUTOMATION\nsdl-sez-boe-automation</code>.
|
|
Every figure and code block above was verified against a real running system, not derived from the build spec alone.
|
|
</footer>
|
|
</main>
|
|
</div>
|
|
</div>
|
|
|
|
<script src="https://cdnjs.cloudflare.com/ajax/libs/mermaid/10.9.1/mermaid.min.js"></script>
|
|
<script>
|
|
function initMermaid(){
|
|
var isDark = false;
|
|
var t = document.documentElement.getAttribute('data-theme');
|
|
if (t === 'dark') isDark = true;
|
|
else if (t !== 'light') isDark = window.matchMedia('(prefers-color-scheme: dark)').matches;
|
|
mermaid.initialize({
|
|
startOnLoad:true,
|
|
theme: isDark ? 'dark' : 'neutral',
|
|
themeVariables: {
|
|
fontFamily:"'IBM Plex Mono', monospace",
|
|
fontSize:'13px'
|
|
},
|
|
securityLevel:'loose'
|
|
});
|
|
}
|
|
initMermaid();
|
|
</script>
|
|
|
|
<!--claude-mermaid-runtime-begin:3477-->
|
|
<style>.mermaid-diagram{margin-block:4px}.mermaid-diagram svg{display:block;margin:0 auto;max-width:100%;height:auto}</style>
|
|
<script src="/_runtime/mermaid-11.16.1.min.js"></script>
|
|
<script>(function(){
|
|
var CFG={"palettes":{"light":{"surface":"#f4efe4","text":"#42392e","line":"#8a7f6d","border":"#7a6c52","bg":"#fffdf8"},"dark":{"surface":"#262b34","text":"#f2f3f5","line":"#a8adb8","border":"#9aa4b8","bg":"#1f232b"}}};
|
|
if(typeof mermaid==='undefined')return;
|
|
var pres=Array.prototype.slice.call(document.querySelectorAll('pre.mermaid')).filter(function(p){if(p.hasAttribute('data-claude-mermaid-claimed'))return false;p.setAttribute('data-claude-mermaid-claimed','1');return true;});
|
|
if(!pres.length)return;
|
|
var mq=window.matchMedia?window.matchMedia('(prefers-color-scheme: dark)'):null;
|
|
var root=document.documentElement;
|
|
var items=pres.map(function(pre){
|
|
var mount=document.createElement('div');mount.className='mermaid-diagram';
|
|
return {pre:pre,mount:mount,src:pre.textContent||''};
|
|
});
|
|
var seq=0;
|
|
var renderGen=0;
|
|
var lastKey='';
|
|
function pageBg(fallback){
|
|
var els=[document.body,document.documentElement];
|
|
for(var i=0;i<els.length;i++){
|
|
var c=els[i]&&getComputedStyle(els[i]).backgroundColor;
|
|
if(c&&c!=='transparent'&&c!=='rgba(0, 0, 0, 0)')return c;
|
|
}
|
|
return fallback;
|
|
}
|
|
function render(){
|
|
var theme=root.getAttribute('data-theme');
|
|
var dark=theme==='dark'||(!!(mq&&mq.matches)&&theme!=='light');
|
|
var pal=dark?CFG.palettes.dark:CFG.palettes.light;
|
|
var bg=pageBg(pal.bg);
|
|
var key=(dark?'d':'l')+'|'+bg;
|
|
if(key===lastKey)return;
|
|
lastKey=key;
|
|
var gen=++renderGen;
|
|
var font=getComputedStyle(document.body).fontFamily||'sans-serif';
|
|
var nat={useMaxWidth:false};
|
|
mermaid.initialize({
|
|
startOnLoad:false,securityLevel:'strict',theme:'base',
|
|
flowchart:nat,sequence:nat,er:nat,state:nat,class:nat,pie:nat,
|
|
gantt:nat,journey:nat,timeline:nat,gitGraph:nat,mindmap:nat,xyChart:nat,
|
|
quadrantChart:nat,sankey:nat,c4:nat,requirement:nat,block:nat,
|
|
packet:nat,kanban:nat,architecture:nat,radar:nat,
|
|
themeVariables:{background:bg,mainBkg:pal.surface,primaryColor:pal.surface,
|
|
primaryTextColor:pal.text,lineColor:pal.line,primaryBorderColor:pal.border,
|
|
nodeBorder:pal.border,clusterBorder:pal.border,edgeLabelBackground:bg,
|
|
clusterBkg:'rgba(127,127,127,0.07)',titleColor:pal.text,
|
|
darkMode:dark,rowOdd:bg,rowEven:'rgba(127,127,127,0.07)',
|
|
attributeBackgroundColorOdd:bg,attributeBackgroundColorEven:'rgba(127,127,127,0.07)',
|
|
fontSize:'16px',fontFamily:font},
|
|
themeCSS:'.node rect, .node circle, .node polygon, .node path, .cluster rect { stroke-width: 2px; }'
|
|
});
|
|
items.forEach(function(it){
|
|
var id='claude-mermaid-'+seq++;
|
|
mermaid.render(id,it.src).then(function(r){
|
|
if(gen!==renderGen)return;
|
|
var prev=it.pre.previousElementSibling;
|
|
if(prev&&prev.className==='mermaid-diagram'&&prev!==it.mount)return;
|
|
it.mount.innerHTML=r.svg;
|
|
if(!it.mount.parentNode)it.pre.parentNode.insertBefore(it.mount,it.pre);
|
|
it.pre.style.display='none';
|
|
},function(){
|
|
var scratch=document.getElementById(id);
|
|
if(scratch)scratch.parentNode.removeChild(scratch);
|
|
scratch=document.getElementById('d'+id);
|
|
if(scratch)scratch.parentNode.removeChild(scratch);
|
|
if(gen!==renderGen)return;
|
|
if(it.mount.parentNode)it.mount.parentNode.removeChild(it.mount);
|
|
it.pre.style.display='';
|
|
});
|
|
});
|
|
}
|
|
render();
|
|
if(mq&&mq.addEventListener)mq.addEventListener('change',render);
|
|
if(typeof MutationObserver!=='undefined')new MutationObserver(render).observe(root,{attributes:true,attributeFilter:['data-theme']});
|
|
})();</script>
|
|
<!--claude-mermaid-runtime-end-->
|
|
|
|
</body></html>
|