import re from datetime import datetime from typing import Any, Callable, Iterable, Optional class CourierBillParser: """ Dynamic Courier Bill of Entry parser. Design goals: - Works from semantic labels, visual rows and detected tables. - Does not depend on a fixed page number for any business field. - Supports CBE-XIII, CBE-XIV and unknown/future Courier layouts. - Uses the same visual-word philosophy as ShippingBillParser. - Keeps the Courier Frappe JSON contract strict. - Uses candidate scoring and field validators to reject nearby labels. - Never injects invoice fields into item rows or vice versa. """ PARENT_FIELDS = [ "doctype", "out_of_charge_date", "cour_be_importername", "cour_be_arrivalairport", "cour_be_marksnos", "cour_be_iec", "cour_be_coo", "cour_be_airline", "cour_be_mawbno", "cour_be_couriername", "cour_be_iecbranch", "cour_be_countryofconsignment", "cour_be_flightno", "cour_be_mawb_dt", "cour_be_betype_description", "cour_be_importer_address", "cour_be_pkgs", "cour_be_arrivaldate", "cour_be_hawbno", "cour_be_beno", "cour_be_adcode", "cour_be_invoicecount", "cour_be_igmno", "cour_be_hawb_dt", "be_date", "cour_be_gstin", "cour_be_category", "cour_be_inward_date", "cour_be_gwt_kgs", "cour_be_total_duty", "cour_be_interest_amt", "cour_be_tr6_no", "cour_be_pymt_amt", "cour_be_challan_date", "cour_be_supplier_name", ] INVOICE_FIELDS = [ "doctype", "cour_be_invsno", "cour_be_inv_nature_of_txn", "cour_be_incoterm", "cour_be_inv_supplier_name", "cour_be_invoice_no", "cour_be_inv_pymt_terms", "cour_be_inv_related", "suppliers_address", "cour_be_invoice_date", "cour_be_valuationmethod", "cour_svb_refno", "cour_be_inv_value", "cour_be_inv_svbdate", "cour_be_inv_currency", "cour_be_prov_final", ] ITEM_FIELDS = [ "doctype", "linked_invoice_id_sno", "cour_be_item_upi", "cour_be_item_manufacturer", "cour_be_item_upi_currency", "cour_be_item_countryoforigin", "cour_be_item_itemsn", "cour_be_item_qty", "cour_be_item_ritc", "cour_be_item_itemdescription", "cour_be_item_uom", "cour_be_item_ctsh", "cour_be_item_itemgendescription", "cour_be_item_exchg_rate", "cour_be_item_cetsh", "cour_be_item_sws_rt", "cour_be_item_assessable_value", "cour_be_item_sws_amt", "cour_be_item_bcd_rt", "cour_be_item_igst_rt", "cour_be_item_bcd_amt", "cour_be_item_igst_amt", "cour_be_item_aidc_rt", "cour_be_item_aidc_amt", "cour_be_item_cmpcess_rt", "cour_be_item_cmpcess_amt", ] PARENT_FLOAT_FIELDS = { "cour_be_gwt_kgs", "cour_be_total_duty", "cour_be_interest_amt", "cour_be_pymt_amt", } PARENT_INTEGER_FIELDS = {"cour_be_pkgs", "cour_be_invoicecount"} CHILD_FLOAT_FIELDS = { "cour_be_item_upi", "cour_be_item_qty", "cour_be_item_exchg_rate", "cour_be_item_sws_rt", "cour_be_item_assessable_value", "cour_be_item_sws_amt", "cour_be_item_bcd_rt", "cour_be_item_igst_rt", "cour_be_item_bcd_amt", "cour_be_item_igst_amt", "cour_be_item_aidc_rt", "cour_be_item_aidc_amt", "cour_be_item_cmpcess_rt", "cour_be_item_cmpcess_amt", "cour_be_inv_value", } TABLE_DOCTYPES = { "invoices": "Courier Bill of Entry Invoices", "items": "Courier Bill of Entry Items", } # Aliases are deliberately semantic. They describe what a field means, # not where it occurs on a page. PARENT_ALIASES = { "out_of_charge_date": ["OOC ISSUED ON", "OUT OF CHARGE DATE", "OOC DATE"], "cour_be_importername": [ "NAME OF CONSIGNEE", "CONSIGNEE NAME", "IMPORTER NAME", "NAME OF IMPORTER", "PARTICULARS OF THE IMPORTER", ], "cour_be_importer_address": [ "ADDRESS OF CONSIGNEE", "CONSIGNEE ADDRESS", "IMPORTER ADDRESS", "ADDRESS OF IMPORTER", ], "cour_be_arrivalairport": [ "AIRPORT OF ARRIVAL", "ARRIVAL AIRPORT", ], "cour_be_marksnos": ["MARKS AND NUMBERS", "MARKS & NUMBERS", "MARKS NOS"], "cour_be_iec": [ "IMPORT EXPORT CODE", "IMPORT-EXPORT CODE", "IEC", "IEC CODE", ], "cour_be_coo": ["COUNTRY OF ORIGIN", "ORIGIN COUNTRY"], "cour_be_airline": ["AIRLINES", "AIRLINE", "CARRIER", "AIR CARRIER"], "cour_be_mawbno": [ "MASTER AIRWAY BILL NUMBER", "MASTER AIRWAY BILL (MAWB) NUMBER", "MAWB NUMBER", "MAWB NO", ], "cour_be_couriername": [ "NAME OF THE AUTHORIZED COURIER", "NAME OF AUTHORIZED COURIER", "AUTHORIZED COURIER", "AUTHORISED COURIER", ], "cour_be_iecbranch": [ "IMPORT EXPORT BRANCH CODE", "IEC BRANCH CODE", "IMPORT EXPORT BRANCH", ], "cour_be_countryofconsignment": [ "COUNTRY OF CONSIGNMENT", "CONSIGNMENT COUNTRY", "COUNTRY OF EXPORTATION", ], "cour_be_flightno": ["FLIGHT NO", "FLIGHT NUMBER"], "cour_be_mawb_dt": ["DATE OF MAWB", "MAWB DATE", "MAWB DT"], "cour_be_betype_description": ["TYPE OF BOE", "BOE TYPE"], "cour_be_pkgs": ["NUMBER OF PACKAGES", "NO OF PACKAGES", "PACKAGES"], "cour_be_arrivaldate": ["DATE OF ARRIVAL", "ARRIVAL DATE"], "cour_be_hawbno": [ "HAWB NUMBER", "HAWB NO", "HOUSE AIRWAY BILL NUMBER", "HOUSE AIRWAY BILL (HAWB) NUMBER", ], "cour_be_beno": [ "CBE-XIII NUMBER", "CBEXIII NUMBER", "CBEXIV NUMBER", "BOE NUMBER", "BILL OF ENTRY NUMBER", "BOE NO", ], "cour_be_adcode": [ "AUTHORISED DEALER CODE OF BANK", "AUTHORIZED DEALER CODE", "AD CODE", "AUTHORISED DEALER CODE", ], "cour_be_igmno": [ "IMPORT GENERAL MANIFEST NUMBER", "IMPORT GENERAL MANIFEST (IGM) NUMBER", "IGM NUMBER", "IGM NO", ], "cour_be_hawb_dt": ["DATE OF HAWB", "HAWB DATE", "HAWB DT"], "be_date": ["BOE DATE", "BILL OF ENTRY DATE"], "cour_be_gstin": ["KYC ID", "GSTIN", "GSTIN NUMBER"], "cour_be_category": ["CATEGORY OF BOE", "BOE CATEGORY"], "cour_be_inward_date": [ "DATE OF ENTRY INWARD", "INWARD DATE", "ENTRY INWARD DATE", ], "cour_be_gwt_kgs": ["GROSS WEIGHT", "GROSS WEIGHT KGS", "GWT", "G WT"], "cour_be_total_duty": [ "TOTAL DUTY", "TOTAL DUTY AMOUNT", "DUTY(RS.)", "DUTY (RS.)", ], "cour_be_interest_amt": ["INTEREST AMOUNT", "INTEREST AMT"], "cour_be_tr6_no": [ "TR-6 CHALLAN NUMBER", "TR6 CHALLAN NUMBER", "CHALLAN NUMBER", "TR6 NUMBER", "TR6 NO", ], "cour_be_pymt_amt": [ "PAYMENT AMOUNT", "PAYMENT AMT", "TOTAL AMOUNT", "AMOUNT PAID", ], "cour_be_challan_date": ["CHALLAN DATE", "DATE OF CHALLAN", "PAYMENT DATE"], "cour_be_supplier_name": [ "NAME OF CONSIGNOR", "CONSIGNOR NAME", "SUPPLIER NAME", "NAME OF SUPPLIER", "EXPORTER NAME", ], } INVOICE_ALIASES = { "cour_be_invsno": ["INVOICE SNO", "INVOICE SR NO", "S NO", "SERIAL NO"], "cour_be_inv_nature_of_txn": [ "NATURE OF TRANSACTION", "NATURE OF TXN", ], "cour_be_incoterm": ["TERMS OF INVOICE", "INCOTERM", "INCOTERMS"], "cour_be_inv_supplier_name": [ "NAME OF CONSIGNOR", "SUPPLIER NAME", "NAME OF SUPPLIER", "SUPPLIER DETAILS", ], "cour_be_invoice_no": ["INVOICE NUMBER", "INVOICE NO", "INV NO"], "cour_be_inv_pymt_terms": ["TERMS OF PAYMENT", "PAYMENT TERMS"], "cour_be_inv_related": [ "ARE THE BUYER AND SELLER RELATED", "BUYER AND SELLER RELATED", ], "suppliers_address": [ "ADDRESS OF CONSIGNOR", "SUPPLIER ADDRESS", "ADDRESS OF SUPPLIER", "EXPORTER ADDRESS", "SELLER ADDRESS", ], "cour_be_invoice_date": ["DATE OF INVOICE", "INVOICE DATE", "INV DATE"], "cour_be_valuationmethod": [ "METHOD OF VALUATION", "VALUATION METHOD", ], "cour_svb_refno": [ "SVB REF NO", "SVB REFERENCE NO", "SVB REFERENCE NUMBER", ], "cour_be_inv_value": ["INVOICE VALUE", "VALUE OF INVOICE", "INV VALUE"], "cour_be_inv_svbdate": ["SVB DATE", "DATE OF SVB"], "cour_be_inv_currency": ["CURRENCY OF INVOICE", "INVOICE CURRENCY"], "cour_be_prov_final": ["PROVISIONAL FINAL", "PROVISIONAL", "FINAL"], } ITEM_ALIASES = { "linked_invoice_id_sno": ["LINKED INVOICE ID", "INVOICE ID", "INVOICE SNO"], "cour_be_item_upi": ["UNIT PRICE", "UNIT PRICE INVOICE", "UPI"], "cour_be_item_manufacturer": [ "NAME OF MANUFACTURER", "MANUFACTURER NAME", "MANUFACTURER", ], "cour_be_item_upi_currency": [ "CURRENCY OF UNIT PRICE", "UNIT PRICE CURRENCY", "UPI CURRENCY", ], "cour_be_item_countryoforigin": ["COUNTRY OF ORIGIN", "ORIGIN COUNTRY"], "cour_be_item_itemsn": ["ITEM SNO", "ITEM SR NO", "ITEM SERIAL NO", "ITEM NO"], "cour_be_item_qty": ["QUANTITY", "QTY", "NUMBER OF PACKAGES"], "cour_be_item_ritc": ["RITC", "RITC CODE", "TARIFF ITEM", "TARIFF CODE"], "cour_be_item_itemdescription": [ "DESCRIPTION OF GOODS", "ITEM DESCRIPTION", "DESCRIPTION OF ITEM", "DESCRIPTION", ], "cour_be_item_uom": ["UNIT OF MEASURE", "UOM"], "cour_be_item_ctsh": ["CTSH", "CTSH CODE"], "cour_be_item_itemgendescription": [ "ITEM GENERAL DESCRIPTION", "GENERAL DESCRIPTION", ], "cour_be_item_exchg_rate": [ "RATE OF EXCHANGE", "EXCHANGE RATE", "EXCH RATE", ], "cour_be_item_cetsh": ["CETSH", "CETSH CODE"], "cour_be_item_sws_rt": [ "SWS RATE", "SW SRCHRG RATE", "SWS RT", "SOCIAL WELFARE SURCHARGE RATE", ], "cour_be_item_assessable_value": [ "ASSESSABLE VALUE", "ASSESS. VALUE", "ASSESSABLE VAL", ], "cour_be_item_sws_amt": [ "SWS AMOUNT", "SW SRCHRG AMOUNT", "SWS AMT", ], "cour_be_item_bcd_rt": [ "BCD RATE", "BCD RT", "BASIC CUSTOMS DUTY RATE", ], "cour_be_item_igst_rt": ["IGST RATE", "IGST RT"], "cour_be_item_bcd_amt": [ "BCD AMOUNT", "BCD AMT", "BASIC CUSTOMS DUTY AMOUNT", ], "cour_be_item_igst_amt": ["IGST AMOUNT", "IGST AMT"], "cour_be_item_aidc_rt": ["AIDC RATE", "AIDC RT"], "cour_be_item_aidc_amt": ["AIDC AMOUNT", "AIDC AMT"], "cour_be_item_cmpcess_rt": [ "CMPNSTRY RATE", "CMP CESS RATE", "COMPENSATION CESS RATE", ], "cour_be_item_cmpcess_amt": [ "CMPNSTRY AMOUNT", "CMP CESS AMOUNT", "COMPENSATION CESS AMOUNT", ], } SECTION_ALIASES = { "authorized_courier": [ "DETAILS OF AUTHORIZED COURIER", "AUTHORIZED COURIER", ], "importer": [ "PARTICULARS OF THE IMPORTER", "IMPORTER DETAILS", ], "igm": ["IGM DETAILS", "IMPORT GENERAL MANIFEST DETAILS"], "invoice": ["DETAILS OF INVOICE", "DETAILS OF INVOICE(S)", "INVOICE DETAILS"], "supplier": ["SUPPLIER DETAILS", "CONSIGNOR DETAILS"], "items": [ "DETAILS OF ITEM", "DETAILS OF ITEMS", "DETAILS OF ITEM(S) IN INVOICE", ], "duty": ["DUTY DETAILS", "DUTY DETAILS OF ITEM"], "payment": ["PAYMENT DETAILS", "PAYMENT"], } BOILERPLATE = { "other type of", "other type of importer", "type of importer", "if supplier is not the seller", "details in case of previous imports", "number", "address", "date", "value", "amount", "term", "terms", "authorized", "authorised", "courier", "copy", "original copy", "duplicate copy", "page", "page of", "form", "normal", "private", "government", "non-government", } def __init__(self, pages: list[dict[str, Any]]): self.pages = pages or [] self.trace: list[dict[str, Any]] = [] self.page_text = { int(p.get("page_number", i + 1)): str(p.get("text") or "") for i, p in enumerate(self.pages) } self.full_text = "\n".join(self.page_text.values()) self.form_type = self._detect_form_type() self.words = self._all_words() self.page_words: dict[int, list[dict[str, Any]]] = {} for w in self.words: self.page_words.setdefault(w["_page"], []).append(w) for p in self.page_words: self.page_words[p].sort(key=lambda w: (w["y0"], w["x0"])) self.rows = self._build_rows(self.words) self.page_rows: dict[int, list[dict[str, Any]]] = {} for r in self.rows: self.page_rows.setdefault(r["_page"], []).append(r) self.tables = self._all_tables() self._label_cache = {} self._candidate_cache = {} # ---------- public entry ---------- def _final_targeted_igm_marks_manufacturer_recovery(self, result): """Final semantic recovery for IGM, marks/numbers and item manufacturer. Uses only labels and the document's own section boundaries. It is intentionally conservative and does not alter unrelated fields. """ text = self.full_text or "" if not text: return def clean(v): if v is None: return "" v = self._clean(v) v = re.sub(r"^[\s:;,.-]+", "", v) v = re.sub(r"[\s:;,.-]+$", "", v) return v.strip() # Marks and Numbers: a value containing only PDF table quotes means # the source field is blank/N/A, not the literal string '""'. marks = result.get("cour_be_marksnos") if isinstance(marks, str): marks = clean(marks) if re.fullmatch(r'["\']{1,4}', marks) or marks.upper() in { '""', "''", "N/A", "NA", "N.A" }: marks = "" result["cour_be_marksnos"] = marks # IGM Number: use several semantic representations of the same label. # Customs PDFs frequently split this heading into multiple visual lines # or insert punctuation between the words, so a normal regex over the # raw text alone is not sufficiently reliable. The recovery below # first searches the raw stream, then a punctuation/line-break-neutral # representation, and finally a bounded local numeric candidate. igm = "" raw_patterns = ( r"Import\s+General\s+Manifest\s*" r"(?:\(\s*IGM\s*\)\s*)?Number\s*[:：-]?\s*([0-9]{1,20})", r"Import\s+General\s+Manifest\s*\(\s*IGM\s*\)\s*" r"Number\s*[:：-]?\s*([0-9]{1,20})", ) for pat in raw_patterns: m = re.search(pat, text, re.I | re.S) if m: igm = clean(m.group(1)) if igm: break if not igm: # Normalize every separator/line break to one space. This handles # forms such as: # Import General Manifest\n # (IGM) Number :\n # 3005909 # as well as concatenated PDF text such as # ImportGeneralManifest(IGM)Number:3005909. compact_text = re.sub(r"[^A-Za-z0-9]+", " ", text) compact_text = re.sub(r"\s+", " ", compact_text).strip() compact_patterns = ( r"\bImport\s+General\s+Manifest\s+(?:IGM\s+)?Number\s+([0-9]{1,20})\b", r"\bImport\s+General\s+Manifest\s+IGM\s+Number\s+([0-9]{1,20})\b", ) for pat in compact_patterns: m = re.search(pat, compact_text, re.I) if m: igm = clean(m.group(1)) if igm: break if not igm: # Last semantic fallback: find the complete phrase in the compact # text and inspect only the following local window. This prevents # accidentally taking Gateway IGM Number, MAWB, HAWB or a date. marker = re.search( r"\bImport\s+General\s+Manifest(?:\s+IGM)?\s+Number\b", compact_text, re.I, ) if marker: window = compact_text[marker.end(): marker.end() + 80] candidate = re.search(r"\b([0-9]{1,20})\b", window) if candidate: igm = clean(candidate.group(1)) # Only replace a missing/invalid IGM. A valid value already extracted # by the main v18.3 pipeline is left untouched. current_igm = clean(result.get("cour_be_igmno")) if igm and (not current_igm or not re.fullmatch(r"\d{1,20}", current_igm)): result["cour_be_igmno"] = igm # Item manufacturer: recover independently inside each explicit item # section. This prevents a global supplier name or another item's # manufacturer from being assigned to the current item. heading_re = re.compile( r"DETAILS\s*OF\s*ITEM\s*(?:[-:\u2010\u2011\u2012\u2013\u2014]\s*)?(\d+)\b", re.I, ) sections = list(heading_re.finditer(text)) if not sections: return items = result.get("items") or [] def manufacturer_from_block(block): patterns = ( r"(?:Name\s+of\s+Manufacturer|Name\s+of\s+the\s+Manufacturer)" r"\s*:\s*(.*?)" r"(?=\s+(?:Brand|Model|Grade|Specification|End\s+Use\s+of\s+Item|" r"Country\s+of\s+Origin|Quantity|UOM|Unit\s+of\s+Measure)\s*:|$)", r"(?:Manufacturer)\s*:\s*(.*?)" r"(?=\s+(?:Brand|Model|Grade|Specification|Country\s+of\s+Origin|" r"Quantity|UOM)\s*:|$)", ) for pat in patterns: for m in re.finditer(pat, block, re.I | re.S): candidate = clean(m.group(1)) candidate = re.sub( r"(?i)\s*(?:Brand|Model|Grade|Specification|End\s+Use\s+of\s+Item)" r"\s*:.*$", "", candidate, ) candidate = re.sub(r"\s+", " ", candidate).strip() if candidate and not self._looks_like_any_known_label(candidate): if candidate.upper() not in {"N/A", "N.A", "NA", "NONE"}: return self._repair_wrapped_words(candidate) return "" for idx, match in enumerate(sections): start = match.end() end = sections[idx + 1].start() if idx + 1 < len(sections) else len(text) block = text[start:end] manufacturer = manufacturer_from_block(block) if not manufacturer: continue section_no = int(match.group(1)) target = None for row in items: if not isinstance(row, dict): continue raw = row.get("_item_section_no", row.get("cour_be_item_itemsn")) try: if int(float(raw)) == section_no: target = row break except (TypeError, ValueError): continue if target is None and idx < len(items): target = items[idx] if target is not None: current = clean(target.get("cour_be_item_manufacturer")) # Prefer the explicit item-section manufacturer whenever the # current value is empty, a known placeholder, or clearly # contaminated by metadata. if ( not current or current.upper() in {"N/A", "N.A", "NA", "NONE"} or self._looks_like_any_known_label(current) or re.search(r"(?i)\b(?:Brand|Model|Grade|Specification)\s*:", current) ): target["cour_be_item_manufacturer"] = manufacturer elif len(manufacturer) > len(current): # A complete section-scoped candidate is stronger than a # shorter/truncated generic observation. target["cour_be_item_manufacturer"] = manufacturer def _final_invoice_semantic_authority(self, result): """ Final, invoice-only semantic authority. This pass repairs only invoice-level: - Nature of Transaction - Terms of Payment - Buyer/Seller Related indicator It deliberately does not use fixed coordinates or page numbers. Values are recovered from the document's own semantic labels and bounded invoice/SVB sections. Existing valid values are preserved. """ def norm_space(value): value = str(value or "") value = value.replace("\u00ad", "") value = re.sub(r"[\u200b\ufeff\ufffe\uffff]", "", value) value = re.sub(r"\s+", " ", value) return value.strip() def empty_value(value): if value is None: return True if isinstance(value, (int, float)): return float(value) == 0.0 s = norm_space(value) return s == "" or s in {'""', "''", ":", "-", "N/A", "N.A"} def clean_value(value): value = norm_space(value) value = re.sub(r"^\s*[:\-]+\s*", "", value) value = re.sub(r"\s*[:\-]+\s*$", "", value) return value.strip() def is_noise(value): s = norm_space(value) if not s: return True compact = re.sub(r"[^A-Za-z0-9]+", "", s).lower() return compact in { "currencyfor", "unitprice", "quantity", "uom", "rateofexchange", "ifothers", "methodofvaluation", "termsofinvoice", "invoicevalue", "currency", "brand", "model", "grade", "specification", } def extract_value(text_block, label_pattern, stop_patterns): if not text_block: return "" # PDF text extraction can place the label and value on separate # lines, therefore whitespace is intentionally used instead of # a literal space. stop = "|".join(stop_patterns) patterns = [ rf"(?is){label_pattern}\s*[:\-]?\s*" rf"([A-Za-z0-9][A-Za-z0-9 ./+&_(),'\[\]%-]{{0,80}}?)" rf"(?=\s+(?:{stop})\b|$)", rf"(?is){label_pattern}\s*[:\-]?\s*" rf"([^\r\n]+)", ] for pattern in patterns: try: match = re.search(pattern, text_block) except re.error: continue if not match: continue value = clean_value(match.group(1)) # A broad line match can include another label. Cut at the # first known structural label instead of returning adjacent # form furniture. if stop_patterns: boundary = re.search( rf"(?i)\s+(?:{'|'.join(stop_patterns)})\s*:", value, ) if boundary: value = clean_value(value[:boundary.start()]) if value and not is_noise(value): return value return "" def invoice_blocks_from_document(): source = str(getattr(self, "full_text", "") or "") if not source: return [] # Normalize only for boundary matching; retain original content # for extraction. heading = re.compile( r"(?im)^\s*DETAILS?\s+OF\s+INVOICE(?:\s*[-:]\s*(\d+))?" ) matches = list(heading.finditer(source)) blocks = [] if matches: for idx, match in enumerate(matches): start = match.start() end = ( matches[idx + 1].start() if idx + 1 < len(matches) else len(source) ) block = source[start:end] # Do not allow the item section belonging to the invoice # to contaminate invoice-level payment/nature fields. item_boundary = re.search( r"(?is)\bDETAILS?\s+OF\s+ITEM(?:S)?\b" r"|\bDETAILED\s+DESCRIPTION\s+OF\s+ITEM\b", block, ) if item_boundary: block = block[:item_boundary.start()] blocks.append(block) # Some layouts use "Invoice Details" rather than "Details Of # Invoice". Recover a bounded supplier-details region as fallback. if not blocks: alt = re.search( r"(?is)\bINVOICE\s+DETAILS\b.*?" r"(?=\bDETAILS?\s+OF\s+ITEM\b|\Z)", source, ) if alt: blocks.append(alt.group(0)) return blocks invoice_blocks = invoice_blocks_from_document() # If the existing invoice rows are present, map the strongest # invoice block to the corresponding row. For a normal single-invoice # CBE this is the first row. For multiple invoices, match by invoice # section number when available, otherwise use the row order. invoices = result.setdefault("invoices", []) if not invoices and invoice_blocks: invoices.append(self._empty_invoice(1)) for index, invoice in enumerate(invoices): block = invoice_blocks[index] if index < len(invoice_blocks) else "" if not block: continue # ---------------- Nature of Transaction ---------------- if empty_value(invoice.get("cour_be_inv_nature_of_txn")): nature = extract_value( block, r"(?:Nature\s+of\s+Transaction|Nature\s+of\s+Txn)", [ r"If\s+Others", r"Terms\s+of\s+Payment", r"Conditions\s+or\s+Restrictions", r"Method\s+of\s+Valuation", r"Terms\s+of\s+Invoice", r"Invoice\s+Value", ], ) # "Sale", "Purchase", "Lease", etc. are expected values. # Reject a value that is actually another label. if nature and not is_noise(nature): nature = re.sub( r"(?i)\s+(?:If\s+Others|Terms\s+of\s+Payment|" r"Method\s+of\s+Valuation|Terms\s+of\s+Invoice).*$", "", nature, ).strip() if nature: invoice["cour_be_inv_nature_of_txn"] = nature # ---------------- Terms of Payment ---------------- if empty_value(invoice.get("cour_be_inv_pymt_terms")): terms = extract_value( block, r"(?:Terms\s+of\s+Payment|Payment\s+Terms)", [ r"If\s+Others", r"Conditions\s+or\s+Restrictions", r"Method\s+of\s+Valuation", r"Terms\s+of\s+Invoice", r"Invoice\s+Value", r"Currency", ], ) if terms: # Do not accept adjacent labels or monetary values. terms = re.sub( r"(?i)\s+(?:If\s+Others|Conditions\s+or\s+Restrictions|" r"Method\s+of\s+Valuation|Terms\s+of\s+Invoice|" r"Invoice\s+Value|Currency).*$", "", terms, ).strip() if terms and self._float(terms) is None and not is_noise(terms): invoice["cour_be_inv_pymt_terms"] = terms # ---------------- Buyer/Seller Related ---------------- # This field is invoice-level SVB information in Courier BOE forms # and can occur after the Details Of Invoice block. Therefore it is # intentionally searched in a separate bounded SVB section. related_source = str(getattr(self, "full_text", "") or "") svb_matches = list(re.finditer( r"(?is)\bDETAILS?\s+OF\s+SPECIAL\s+VALUATION\s+BRANCH\b.*?" r"(?=\bDETAILS?\s+OF\s+ITEM\b|\bDETAILS?\s+OF\s+INVOICE\b|\Z)", related_source, )) svb_blocks = [m.group(0) for m in svb_matches] if not svb_blocks: # Flexible fallback for OCR/text-stream variants where the full # heading is broken apart. related_marker = re.search( r"(?is)\bARE\s+THE\s+BUYER\s+AND\s+SELLER\s+RELATED\b", related_source, ) if related_marker: svb_blocks = [ related_source[ max(0, related_marker.start() - 1200): min(len(related_source), related_marker.end() + 1200) ] ] related_value = "" for block in svb_blocks: # First preference: the complete question followed by Y/N. m = re.search( r"(?is)\bARE\s+THE\s+BUYER\s+AND\s+SELLER\s+RELATED\b" r"\s*[:\-]?\s*(?:YES\s*/\s*NO\s*)?" r"([YN])\b", block, ) if m: related_value = m.group(1).upper() break # Second preference: row-oriented extraction. This handles PDFs # that place "Related?" and the answer in separate visual rows. lines = [norm_space(x) for x in block.splitlines() if norm_space(x)] for pos, line in enumerate(lines): compact = re.sub(r"[^A-Za-z0-9]+", "", line).lower() if ( "arethebuyerandsellerrelated" in compact or ("buyerandseller" in compact and "related" in compact) ): window = " ".join(lines[pos:pos + 4]) answer = re.search( r"\b([YN])\b", window, re.I, ) if answer: related_value = answer.group(1).upper() break if related_value: break if related_value in {"Y", "N"}: for invoice in invoices: # Only populate missing/invalid values. Never overwrite a # valid existing invoice-level answer. existing = norm_space(invoice.get("cour_be_inv_related")) if existing.upper() not in {"Y", "N"}: invoice["cour_be_inv_related"] = related_value # ---------------- Marks and Numbers ---------------- # Blank/N/A marks are legitimate. The only required cleanup here is # preventing parser artifacts such as literal quote pairs from being # emitted as a real value. marks = result.get("cour_be_marksnos") if isinstance(marks, str): marks = clean_value(marks) if marks in {'""', "''", '" "', "' '", "N/A", "N.A"}: marks = "" result["cour_be_marksnos"] = marks def _final_v15_commercial_invoice_authority(self, result): """Final authority for commercial-invoice DATE and PAYMENT TERMS. This pass is deliberately independent of the earlier generic invoice extractors. It gives precedence to an explicitly labelled commercial invoice field when the analyzer supplies digital text/table content. It supports both structured labels (Invoice Date / Payment Terms) and the common two-column invoice form where the labels are simply DATE and PAYMENT TERMS. Association is based on the same source block and invoice-number proximity; no sample values, page numbers, coordinates, or OCR are used. """ def clean(v): v = str(v or "") v = v.replace("\u00ad", " ").replace("\ufeff", " ") # Preserve useful token boundaries while removing PDF line-wrap # artifacts such as DATE\n31-JUL-2026. v = re.sub(r"[\u2010\u2011\u2012\u2013\u2014]\s+(?=[A-Za-z0-9])", "", v) v = re.sub(r"(?<=[A-Za-z])\s*-\s*(?=[A-Za-z])", "-", v) v = re.sub(r"\s+", " ", v) return v.strip(" :;\t\r\n") def compact(v): return re.sub(r"[^A-Za-z0-9]", "", str(v or "")).upper() def date_value(v): v = clean(v) if not v: return "" for fn in (self._v183_date, self._format_date): try: d = fn(v) except Exception: d = "" if d: return d return "" invoices = result.get("invoices") or [] if not invoices: return sources = [] for idx, page in enumerate(self.pages, 1): page_no = int(page.get("page_number", idx)) text = clean(page.get("text")) if text: sources.append((page_no, text)) for table in page.get("tables", []) or []: for row in table or []: cells = [clean(c) for c in (row or []) if clean(c)] if cells: sources.append((page_no, " | ".join(cells))) if not sources: return def inv_no(inv): return compact(inv.get("cour_be_invoice_no")) def source_score(inv, text, label_pos): score = 0 ino = inv_no(inv) ctext = compact(text) if ino and ino in ctext: score += 150 if re.search(r"\b(?:commercial\s+invoice|invoice)\b", text, re.I): score += 50 # A source containing both commercial-party information and an # invoice number is stronger than a customs/logistics page. if re.search(r"\b(?:shipper|seller|sold\s+to|bill\s+to|consignee|buyer)\b", text, re.I): score += 20 if re.search(r"\b(?:date|payment\s+terms?|terms\s+of\s+payment)\b", text, re.I): score += 10 if re.search(r"\b(?:date\s+of\s+validation|date\s+of\s+arrival|arrival\s+date|challan\s+date|date\s+of\s+hawb|date\s+of\s+mawb)\b", text, re.I): score -= 120 return score def candidates_for(patterns, inv): hits = [] for page_no, text in sources: for pat in patterns: for m in re.finditer(pat, text, re.I | re.S): hits.append((source_score(inv, text, m.start()), page_no, m, text)) hits.sort(key=lambda x: (x[0], -x[1], -x[2].start()), reverse=True) return hits for inv in invoices: # Strongly labelled invoice date first. Unlike the previous pass, # generic DATE is also supported when it occurs in the same # invoice source as the invoice number. date_patterns = [ r"\b(?:invoice\s+date|date\s+of\s+invoice|inv\s+date)\s*[:#\-]?\s*(\d{1,2}\s*[-/.]\s*[A-Za-z]{3,9}\s*[-/.]\s*\d{2,4}|\d{1,2}\s*[-/.]\s*\d{1,2}\s*[-/.]\s*\d{2,4})", r"\bdate\s*[:#\-]?\s*(\d{1,2}\s*[-/.]\s*[A-Za-z]{3,9}\s*[-/.]\s*\d{2,4}|\d{1,2}\s*[-/.]\s*\d{1,2}\s*[-/.]\s*\d{2,4})", ] date_hits = candidates_for(date_patterns, inv) if date_hits: # Explicit invoice-date labels always win. For generic DATE, # require invoice identity or strong commercial context. ranked = [] for score, page_no, m, text in date_hits: label = m.group(0).split(m.group(1), 1)[0].upper() explicit = "INVOICE" in label has_inv_no = bool(inv_no(inv) and inv_no(inv) in compact(text)) commercial = bool(re.search(r"\b(?:commercial\s+invoice|shipper|seller|consignee|buyer|invoice\s+no|invoice\s+number)\b", text, re.I)) if not explicit and not (has_inv_no or commercial): continue d = date_value(m.group(1)) if d: ranked.append((score + (80 if explicit else 0) + (40 if has_inv_no else 0), page_no, d)) if ranked: ranked.sort(reverse=True) inv["cour_be_invoice_date"] = ranked[0][2] # Payment terms: this is authoritative when explicitly labelled. # It intentionally overwrites generic placeholders such as OTHER # and also corrects a value captured from the CBE section instead # of the commercial invoice. payment_patterns = [ r"\bpayment\s+terms?\s*[:#\-]?\s*(.{1,160}?)(?=\s+(?:incoterms?|invoice\s+date|date|invoice\s+no(?:\.|\b)|invoice\s+number|invoice|currency|total(?:\s+amount|\s+price)?|freight|shipper|consignee|buyer|seller)\b|$)", r"\bterms\s+of\s+payment\s*[:#\-]?\s*(.{1,160}?)(?=\s+(?:incoterms?|invoice\s+date|date|invoice\s+no(?:\.|\b)|invoice\s+number|invoice|currency|total(?:\s+amount|\s+price)?|freight|shipper|consignee|buyer|seller)\b|$)", ] pay_hits = candidates_for(payment_patterns, inv) if pay_hits: ranked = [] for score, page_no, m, text in pay_hits: value = clean(m.group(1)) if not value: continue value = re.sub(r"\s+(?:page\s+\d+\s+of\s+\d+)\s*$", "", value, flags=re.I) if self._v183_numeric(value) is not None or self._looks_like_any_known_label(value): continue # Reject obvious CBE placeholder values when a commercial # invoice source is available, but keep legitimate terms # of any length/content. normalized = value.upper() if normalized in {"OTHER", "OTHERS", "N/A", "NA"}: continue has_inv_no = bool(inv_no(inv) and inv_no(inv) in compact(text)) commercial = bool(re.search(r"\b(?:commercial\s+invoice|invoice\s+no|invoice\s+number|shipper|seller|consignee|buyer)\b", text, re.I)) if not (has_inv_no or commercial): continue ranked.append((score + (50 if has_inv_no else 0) + (25 if commercial else 0), page_no, value)) if ranked: ranked.sort(reverse=True) inv["cour_be_inv_pymt_terms"] = ranked[0][2] def _final_cbe_country_category_item_authority(self, result): """Final semantic authority for CBE country/category/item identity. Rules are derived only from explicit labels in the detected CBE form: * ``Country of Exportation`` / ``Country of Consignment`` -> ``cour_be_countryofconsignment``. * ``Country of Origin`` inside ``DETAILS OF ITEM`` remains ``cour_be_coo`` and is never used as consignment/exportation. * ``Government / Non-Government`` -> ``cour_be_category``. * Item serial/linkage is tied to explicit item sections and the invoice serials already reconstructed by the parser. No page number, coordinate, country name, invoice number, or other document-specific value is hard-coded. """ def clean(v): v = str(v or "").replace("\u00ad", " ").replace("\ufeff", " ") return re.sub(r"\s+", " ", v).strip(" :;,\t\r\n") def compact(v): return re.sub(r"[^a-z0-9]+", "", clean(v).lower()) # Detect the logical CBE pages from the form marker rather than using # physical page numbers. This also prevents the commercial invoice and # export certificate from supplying header fields. cbe_pages = [] form_pattern = r"Form\s+(?:Regular\s+)?Bill\s+Of\s+Entry\s*[- ]?\s*CBE\s*[- ]?(?:XIII|XIV)|Courier\s+Bill\s+of\s+Entry\s*[- ]?(?:XIII|XIV)" start_idx = None total_pages = None for idx, page in enumerate(self.pages): text = str(page.get("text") or "") compact_text = re.sub(r"[^a-z0-9]+", "", text.lower()) compact_forms = ( "formregularbillofentrycbexiii", "formregularbillofentrycbexiv", "courierbillofentryxiii", "courierbillofentryxiv", ) if re.search(form_pattern, text, re.I) or any(marker in compact_text for marker in compact_forms): start_idx = idx m = re.search(r"Page\s*\d+\s*of\s*(\d+)", text, re.I) if m: total_pages = int(m.group(1)) else: compact_footer = re.search(r"page(\d+)of(\d+)", compact_text, re.I) if compact_footer: total_pages = int(compact_footer.group(2)) break if start_idx is None: return for idx in range(start_idx, len(self.pages)): text = str(self.pages[idx].get("text") or "") if idx == start_idx: cbe_pages.append(self.pages[idx]) continue footer = re.search(r"Page\s*\s*(\d+)\s*of\s*(\d+)", text, re.I) if not footer: compact_text = re.sub(r"[^a-z0-9]+", "", text.lower()) compact_footer = re.search(r"page(\d+)of(\d+)", compact_text, re.I) if compact_footer: footer = compact_footer if total_pages is not None and footer and int(footer.group(2)) == total_pages: cbe_pages.append(self.pages[idx]) if int(footer.group(1)) >= total_pages: break continue if re.search( r"IGM\s+DETAILS|DETAILS\s+OF\s+ITEM|DUTY\s+DETAILS|PAYMENT\s+DETAILS|DECLARATION", text, re.I, ): cbe_pages.append(self.pages[idx]) elif total_pages is not None: break cbe_text = "\n".join(str(p.get("text") or "") for p in cbe_pages) if not cbe_text: return # Detected table cells are the strongest source for compact CBE # layouts. Keep them local to the detected CBE pages so attached # commercial invoices/export certificates cannot overwrite CBE fields. cbe_table_rows = [] for page in cbe_pages: for table in page.get("tables", []) or []: for row in table or []: cells = [clean(c) for c in (row or [])] if any(cells): cbe_table_rows.append(cells) def table_label_value(label_aliases): aliases = {compact(x) for x in label_aliases} for row in cbe_table_rows: labels = [compact(c) for c in row] for i, cell_label in enumerate(labels): if cell_label in aliases: # Values in the CBE grid are normally the next non-empty # cell to the right of the label. for j in range(i + 1, len(row)): value = clean(row[j]) if value: return value return "" # -------------------------------------------------------------- # 1. Country of Exportation / Consignment. # -------------------------------------------------------------- # Search explicit header labels first. The value must be the immediate # labelled value and must not be taken from DETAILS OF ITEM. country_patterns = [ r"Country\s+of\s+Exportation\s*:\s*([^\n\r]+)", r"Country\s+of\s+Consignment\s*:\s*([^\n\r]+)", ] country_candidates = [] table_country = table_label_value(("Country of Exportation", "Country of Consignment")) if table_country and self._country_validator(table_country): result["cour_be_countryofconsignment"] = table_country country_candidates.append((300, table_country)) for pat in country_patterns: for m in re.finditer(pat, cbe_text, re.I): value = clean(m.group(1)) # Stop a same-line neighbouring label from being swallowed. value = re.split( r"\s+(?=(?:Country\s+of\s+Origin|HAWB\s+Number|Unique\s+Consignment|Name\s+of\s+Consignor|Name\s+of\s+Consignee|Import\s+Export\s+Code)\s*:?)", value, maxsplit=1, flags=re.I, )[0].strip(" :;,-") if value and self._country_validator(value): score = 200 if re.match(r"Country\s+of\s+Exportation", m.group(0), re.I) else 180 country_candidates.append((score, value)) if country_candidates: country_candidates.sort(key=lambda x: x[0], reverse=True) result["cour_be_countryofconsignment"] = country_candidates[0][1] # -------------------------------------------------------------- # 2. Government / Non-Government category. # -------------------------------------------------------------- # This label is semantically distinct from CATEGORY OF BOE. Accept # the explicit value even though "government" words are boilerplate # tokens elsewhere in the document. category_patterns = [ r"Government\s*/\s*Non[-\s]?Government\s*:\s*([^\n\r]+)", r"Government\s+Non[-\s]?Government\s*:\s*([^\n\r]+)", ] category_candidates = [] table_category = table_label_value(("Government / Non-Government", "Government/Non-Government")) if table_category: normalized = re.sub(r"\s+", "-", table_category.upper()) if normalized in {"GOVERNMENT", "NON-GOVERNMENT"}: result["cour_be_category"] = "Non-Government" if normalized == "NON-GOVERNMENT" else "Government" category_candidates.append((300, result["cour_be_category"])) for pat in category_patterns: for m in re.finditer(pat, cbe_text, re.I): value = clean(m.group(1)) value = re.split( r"\s+(?=(?:AD\s+Code|Import\s+Using|DETAILS|KYC\s+Document|State\s+Code)\b)", value, maxsplit=1, flags=re.I, )[0].strip(" :;,-") normalized = re.sub(r"\s+", "-", value.upper()) if normalized in {"GOVERNMENT", "NON-GOVERNMENT"}: category_candidates.append((220, "Non-Government" if normalized == "NON-GOVERNMENT" else "Government")) if category_candidates: result["cour_be_category"] = category_candidates[0][1] # -------------------------------------------------------------- # 3. Item serial + invoice linkage. # -------------------------------------------------------------- # Identify item sections from the CBE itself. For CBE-XIV, numbered # "Details Of Item - N" markers provide the serial directly. For # CBE-XIII, an unnumbered DETAILS OF ITEM block is still a concrete # item section; serial labels inside it are preferred when available. item_matches = list(re.finditer( r"Details\s+Of\s+Item\s*[-:]\s*(\d+)", cbe_text, re.I )) if not item_matches and re.search(r"DETAILS\s*OF\s*ITEM\b", cbe_text, re.I): item_matches = [m for m in re.finditer(r"DETAILS\s*OF\s*ITEM\b", cbe_text, re.I)] invoice_serials = [] for inv in result.get("invoices") or []: try: invoice_serials.append(int(float(inv.get("cour_be_invsno")))) except (TypeError, ValueError): pass invoice_serials = sorted(set(invoice_serials)) items = result.get("items") or [] # If the parser did not materialize an item row but the CBE explicitly # contains an item section, create the minimum schema row rather than # leaving a real source item unrepresented. if not items and item_matches: items = [self._empty_item(1)] result["items"] = items for idx, item in enumerate(items): serial = None if idx < len(item_matches): marker = item_matches[idx] if marker.lastindex: try: serial = int(marker.group(1)) except (TypeError, ValueError): serial = None if serial is None: block_end = re.search( r"Details\s+Of\s+Item\s*[-:]\s*\d+|DECLARATION", cbe_text[marker.end():], re.I, ) block = cbe_text[marker.end():marker.end() + (block_end.start() if block_end else 5000)] sm = re.search( r"(?:Serial\s+Number\s+in\s+Invoice|S\.?\s*No\.?|Sr\.?\s*No\.?)\s*:\s*(\d+)", block, re.I, ) if sm: serial = int(sm.group(1)) if serial is None: # For a concrete unnumbered item section, preserve positional # ordering; do not use a document-specific value. serial = idx + 1 item["cour_be_item_itemsn"] = serial # Link to the actual parent invoice serial. Prefer a matching # serial if the item exposes one; otherwise the single invoice is # the unambiguous parent. For multiple invoices, positional # linkage is used only when the item count aligns with invoice # count; otherwise leave the relation unresolved rather than # inventing a key. if invoice_serials: item_invoice = None for key in ("linked_invoice_id_sno", "cour_be_item_invoicesn"): try: candidate = int(float(item.get(key))) except (TypeError, ValueError): candidate = None if candidate in invoice_serials: item_invoice = candidate break if item_invoice is None and len(invoice_serials) == 1: item_invoice = invoice_serials[0] elif item_invoice is None and len(items) == len(invoice_serials) and idx < len(invoice_serials): item_invoice = invoice_serials[idx] item["linked_invoice_id_sno"] = item_invoice elif item.get("linked_invoice_id_sno") in ("", None): item["linked_invoice_id_sno"] = None def _final_waybill_ritc_svbdate_authority(self, result): """Final semantic recovery for waybill date, item RITC and SVB date.""" text = str(getattr(self, "full_text", "") or "") if not text: return def clean(v): return re.sub(r"\s+", " ", self._clean(v or "")).strip(" :;,-") def parse_date(v): v = clean(v).replace(" ", "") for fmt in ("%Y-%m-%d", "%Y/%m/%d", "%Y.%m.%d", "%d-%m-%Y", "%d/%m/%Y", "%d.%m.%Y"): try: return datetime.strptime(v, fmt).strftime("%Y-%m-%d") except ValueError: pass return "" # Recover the printed courier waybill number/date from the WAYBILL DOC. if not clean(result.get("cour_be_mawbno")) or not clean(result.get("cour_be_mawb_dt")): for wm in re.finditer(r"(?is)\*?\s*WAYBILL\s+DOC\s*\*?", text): block = text[wm.start():wm.start() + 3000] if not clean(result.get("cour_be_mawbno")): for raw in re.findall(r"(?i)\bWAYBILL\s+((?:\d[ \t]*){8,20})", block): value = re.sub(r"\s+", "", raw) if value.isdigit() and 8 <= len(value) <= 20: result["cour_be_mawbno"] = value break if not clean(result.get("cour_be_mawb_dt")): m = re.search(r"\b(\d{4}[-/.]\d{1,2}[-/.]\d{1,2})\s*\(\s*Data\s*\)", block, re.I) if m: result["cour_be_mawb_dt"] = parse_date(m.group(1)) or None if clean(result.get("cour_be_mawbno")) and clean(result.get("cour_be_mawb_dt")): break # Recover item RITC from the item's own tariff row. Prefer explicit # RITC; when absent, CTSH/CETSH in the same item section is the # document's customs HS/tariff code and is the appropriate fallback. item_sections = list(re.finditer(r"(?is)DETAILS\s*OF\s*ITEM.*?(?=DETAILS\s*OF\s*ITEM|$)", text)) for idx, item in enumerate(result.get("items") or []): if not isinstance(item, dict) or clean(item.get("cour_be_item_ritc")): continue block = item_sections[idx].group(0) if idx < len(item_sections) else text code = "" for pat in ( r"\bRITC\s*[:：-]?\s*(\d{4,12})", r"\bCTSH\s*[:：-]?\s*(\d{4,12})", r"\bCETSH\s*[:：-]?\s*(\d{4,12})", r"(?:Import\s+Country\s+HS\s+Code|Country\s+HS\s+Code|HS\s+Code|Tariff\s+(?:Item|Code))\s*[:：-]?\s*(\d{4,12})", ): m = re.search(pat, block, re.I) if m: code = m.group(1) break if code: item["cour_be_item_ritc"] = code # Recover the SVB date from the already-captured SVB reference. This # handles both parenthesized and bracketed references and does not # depend on a fixed page or a hardcoded date. for inv in result.get("invoices") or []: if not isinstance(inv, dict) or clean(inv.get("cour_be_inv_svbdate")): continue ref = clean(inv.get("cour_svb_refno")) if not ref: continue # The reference itself can contain slash-separated segments # before the actual date (for example S/44/02/57/2018-SVB...); # therefore validate every date-shaped candidate instead of taking # the first numeric slash sequence. for m in re.finditer(r"(? dict[str, Any]: result = self._empty_result() # Pass 1: semantic parent/party extraction. self._extract_parent_semantic(result) # Pass 2: explicit invoice/item sections when the document exposes # them. This remains useful for legacy CBE-XIV multi-page forms. self._extract_invoices(result) self._extract_items(result) # Pass 3: condensed CBE-XIII structural recovery. if self.form_type == "CBE-XIII": self._extract_cbe_xiii_structural(result) # Pass 4: table-first generic recovery. Every detected table is # classified from its headers at runtime; no page number or column # index is assumed. Existing valid values are retained unless the # generic table candidate has stronger structural evidence. self._recover_from_generic_tables_v2(result) # Pass 5: payment/duty/related-party recovery. self._extract_payment_semantic(result) self._extract_related_party(result) # Pass 6: dynamic semantic fill for fields still missing after the # table pass. Candidate scoring considers label distance, row/column # proximity, value shape and boilerplate rejection. self._recover_missing_parent_fields(result) self._recover_missing_child_fields(result) # Pass 7: generic table pass again, because payment/duty sections may # create the child rows that table mapping can enrich. self._recover_from_generic_tables_v2(result, second_pass=True) # Pass 8: strict final reconciliation and schema/type contract. self._reconcile_child_rows(result) self._finalize_parent_fields(result) self._finalize_children(result) self._schema_complete(result) def _final_ultra_dynamic_item_and_igm_recovery(self, result): """Last semantic authority for IGM and item descriptive fields. This pass is intentionally narrow. It runs after the existing v18.3 pipeline and only repairs three fields when their source labels are present in the uploaded document: * cour_be_igmno * cour_be_item_manufacturer * cour_be_item_itemgendescription Values are recovered from the document's own semantic sections rather than from company/product-specific dictionaries or fixed coordinates. """ source = str(getattr(self, "full_text", "") or "") if not source: return def clean(value): value = str(value or "") # PDF word-wrap/control characters. Some customs PDFs encode a # visual line break inside a word as U+FFFE/U+FFFF or a soft hyphen. value = value.replace("\u00ad", "").replace("\ufeff", "") value = value.replace("\ufffe", "").replace("\uffff", "") value = re.sub(r"\s+", " ", value) value = re.sub(r"^[\s:;,.-]+|[\s:;,.-]+$", "", value) value = re.sub(r"(?<=[A-Za-z])-\s+(?=[A-Za-z])", "", value) return value.strip() def compact(value): return re.sub(r"[^A-Za-z0-9]+", "", str(value or "")).lower() def valid_igm(value): return bool(re.fullmatch(r"\d{1,20}", clean(value))) # ============================================================== # 1. IGM NUMBER — semantic field authority # ============================================================== # The important distinction is that an extracted default of 0 is not # automatically authoritative. If the source explicitly contains an # IGM value, the source value wins. This still preserves a genuine # source value of 0 because 0 is accepted as a valid IGM number. igm_candidates = [] # Normalize only separators, not characters, so the semantic order of # the document is preserved. This covers same-line and multi-line PDF # representations of the heading. normalized = source.replace("\u00ad", " ").replace("\ufeff", " ") normalized = normalized.replace("\ufffe", " ").replace("\uffff", " ") normalized = re.sub(r"[^A-Za-z0-9]+", " ", normalized) normalized = re.sub(r"\s+", " ", normalized).strip() igm_label = ( r"IMPORT\s+GENERAL\s+MANIFEST\s*" r"(?:\(\s*IGM\s*\)\s*)?" r"NUMBER\b" ) # Candidate must occur immediately after the semantic label and before # another structural field. This prevents Gateway IGM/MAWB/HAWB/date # values from being selected. for m in re.finditer(igm_label, normalized, re.I): tail = normalized[m.end():m.end() + 100] tail = re.split( r"\b(?:DATE\s+OF\s+ENTRY\s+INWARD|" r"MASTER\s+AIRWAY\s+BILL|HOUSE\s+AIRWAY\s+BILL|" r"GATEWAY\s+IGM\s+NUMBER)\b", tail, maxsplit=1, flags=re.I, )[0] nm = re.match(r"\s*(\d{1,20})\b", tail) if nm and valid_igm(nm.group(1)): igm_candidates.append((0, nm.group(1))) break # Line-oriented fallback. This is particularly important when the # text extractor puts the label on one line and its value on the next. if not igm_candidates: lines = [clean(x) for x in source.splitlines() if clean(x)] for i, line in enumerate(lines): c = compact(line) if "importgeneralmanifest" not in c or "gateway" in c: continue local = " ".join(lines[i:i + 5]) local_norm = re.sub(r"[^A-Za-z0-9]+", " ", local) local_norm = re.sub(r"\s+", " ", local_norm).strip() lm = re.search(igm_label + r"\s+(.*)", local_norm, re.I) if not lm: continue tail = re.split( r"\b(?:DATE\s+OF\s+ENTRY\s+INWARD|" r"MASTER\s+AIRWAY\s+BILL|HOUSE\s+AIRWAY\s+BILL|" r"GATEWAY\s+IGM\s+NUMBER)\b", lm.group(1), maxsplit=1, flags=re.I, )[0] nm = re.match(r"\s*(\d{1,20})\b", tail) if nm and valid_igm(nm.group(1)): igm_candidates.append((1, nm.group(1))) break # Visual-row fallback. It handles two-column forms where text order # in extract_text() is unreliable but pdfplumber's rows retain the # actual label/value relationship. if not igm_candidates: for _page_no, rows in getattr(self, "page_rows", {}).items(): for idx, row in enumerate(rows): row_text = clean(row.get("text", "")) c = compact(row_text) if "importgeneralmanifest" not in c or "gateway" in c: continue # Search only the semantic IGM neighbourhood. Stop as # soon as the next field begins. for rr in rows[idx:idx + 6]: txt = clean(rr.get("text", "")) if re.search( r"(?:Date\s+of\s+Entry\s+Inward|Master\s+Airway\s+Bill|" r"House\s+Airway\s+Bill|Gateway\s+IGM\s+Number)", txt, re.I, ): # A row may contain both the IGM and Date of Entry # values. Prefer a number appearing before the # Date-of-Entry label. prefix = re.split( r"Date\s+of\s+Entry\s+Inward", txt, maxsplit=1, flags=re.I, )[0] nm = re.search(r"(? len(compact(manufacturer)): manufacturer = candidate if manufacturer: break if manufacturer and len(compact(manufacturer)) >= 5: break if manufacturer: # Never retain the PDF's form separator/control characters. manufacturer = clean(manufacturer) item["cour_be_item_manufacturer"] = manufacturer # ---------------------------------------------------------- # Item General Description: use the actual General Description # label. If it is intentionally mirrored by Item Description, # use that section-local description as a safe fallback. # ---------------------------------------------------------- general = section_value( block, r"(?:General\s+Description|Item\s+General\s+Description)", [ r"Currency\s+for\s+Unit\s+Price", r"Unit\s+Price", r"Unit\s+of\s+Measure", r"UOM", r"Quantity", r"Rate\s+Of\s+Exchange", r"Accessories", r"Name\s+of\s+Manufacturer", r"Manufacturer", r"Brand", r"Model", r"Country\s+of\s+Origin", r"RITC", r"CTSH", r"CETSH", ], ) description = section_value( block, r"Item\s+Description", [ r"General\s+Description", r"Currency\s+for\s+Unit\s+Price", r"Unit\s+Price", r"Unit\s+of\s+Measure", r"UOM", r"Quantity", r"Rate\s+Of\s+Exchange", r"Accessories", r"Name\s+of\s+Manufacturer", r"Manufacturer", r"Brand", r"Model", r"Country\s+of\s+Origin", ], ) # Remove only structural labels and obvious footer contamination. for name, value in (("general", general), ("description", description)): value = clean(value) if compact(value).startswith("copyrightinformation") or re.search( r"(?i)\b(?:Page\s+\d+\s+of\s+\d+|CopyRightInformation\d{4})\b", value, ): value = "" if self._looks_like_any_known_label(value): value = "" if name == "general": general = value else: description = value if description: item["cour_be_item_itemdescription"] = description if general: item["cour_be_item_itemgendescription"] = general elif description: # When the source has a General Description label but the PDF # text extraction loses the duplicated value, use the # section-local Item Description. This is not a product # hardcode and does not borrow from another item. if re.search(r"General\s+Description\s*:", block, re.I): item["cour_be_item_itemgendescription"] = description # Keep logical item ordering and let the existing dedupe layer handle # only true duplicate observations. Section provenance remains intact. result["items"] = self._dedupe_items(items) return {"courier_bill_json": result, "_trace": self.trace} # ---------- model ---------- def _all_words(self) -> list[dict[str, Any]]: out = [] for page_no, page in enumerate(self.pages, 1): for raw in page.get("words", []) or []: text = self._clean(raw.get("text", "")) if not text: continue w = dict(raw) w["_page"] = page_no w["_text"] = text w["_norm"] = self._norm(text) w["x0"] = self._num(w.get("x0")) w["x1"] = self._num(w.get("x1", w["x0"])) w["y0"] = self._num(w.get("y0", w.get("top", 0))) w["y1"] = self._num(w.get("y1", w.get("bottom", w["y0"]))) w["top"] = self._num(w.get("top", w["y0"])) out.append(w) if out: return out # Analyzer compatibility fallback. for page_no, page in enumerate(self.pages, 1): for line_no, line in enumerate(str(page.get("text") or "").splitlines()): for m in re.finditer(r"\S+", line): out.append({ "_page": page_no, "_text": m.group(), "_norm": self._norm(m.group()), "x0": float(m.start()), "x1": float(m.end()), "y0": float(line_no * 10), "y1": float(line_no * 10 + 8), "top": float(line_no * 10), }) return out def _build_rows(self, words: list[dict[str, Any]], tolerance: float = 4.0): rows = [] for w in sorted(words, key=lambda x: (x["_page"], x["y0"], x["x0"])): target = None for r in reversed(rows[-10:]): if r["_page"] != w["_page"]: continue if abs(r["_y"] - w["y0"]) <= tolerance: target = r break if target is None: target = {"_page": w["_page"], "_y": w["y0"], "words": []} rows.append(target) target["words"].append(w) for r in rows: r["words"].sort(key=lambda x: x["x0"]) r["text"] = " ".join(x["_text"] for x in r["words"]) return rows def _all_tables(self): out = [] for page_no, page in enumerate(self.pages, 1): for idx, table in enumerate(page.get("tables", []) or [], 1): rows = [] for row in table or []: rows.append([self._clean(c) for c in (row or [])]) if rows: out.append({"page": page_no, "index": idx, "rows": rows}) return out # ---------- form / sections ---------- def _detect_form_type(self): head = self.full_text.upper()[:15000] if re.search(r"CBE[\s\-]*XIII|COURIER\s+BILL\s+OF\s+ENTRY\s*[- ]XIII", head): return "CBE-XIII" if re.search(r"CBE[\s\-]*XIV|COURIER\s+BILL\s+OF\s+ENTRY\s*[- ]XIV", head): return "CBE-XIV" return "UNKNOWN" def _section_bounds(self, aliases: Iterable[str]): # Locate the first occurrence of a semantic section in the full text. # Used only as a broad text fallback; visual extraction remains primary. patterns = sorted((re.escape(a).replace(r"\ ", r"\s+") for a in aliases), key=len, reverse=True) start = re.search("|".join(patterns), self.full_text, re.I) if not start: return "" later = [] for group in self.SECTION_ALIASES.values(): for alias in group: if start.group(0).strip().lower() == alias.lower(): continue m = re.search(re.escape(alias).replace(r"\ ", r"\s+"), self.full_text[start.end():], re.I) if m: later.append(m.start()) end = min(later) if later else len(self.full_text) - start.end() return self.full_text[start.end(): start.end() + end] # ---------- label engine ---------- def _norm(self, value): return re.sub(r"[^a-z0-9]+", " ", str(value or "").lower()).strip() def _compact(self, value): return re.sub(r"[^a-z0-9]+", "", str(value or "").lower()) def _find_labels(self, aliases): keys = tuple(self._compact(a) for a in aliases) cache_key = ("labels", keys) if cache_key in self._label_cache: return self._label_cache[cache_key] hits = [] for row in self.rows: for i, word in enumerate(row["words"]): for alias in aliases: parts = self._norm(alias).split() if not parts: continue if i + len(parts) <= len(row["words"]): candidate = " ".join(w["_text"] for w in row["words"][i:i+len(parts)]) if self._compact(candidate) == self._compact(alias): ws = row["words"][i:i+len(parts)] hits.append({ "page": row["_page"], "_page": row["_page"], "x0": ws[0]["x0"], "x1": ws[-1]["x1"], "y0": min(w["y0"] for w in ws), "y1": max(w["y1"] for w in ws), "_y": min(w["y0"] for w in ws), "text": candidate, "row": row, "alias": alias, }) # Also allow a phrase to be present in page text when words are # fragmented in the PDF extraction. self._label_cache[cache_key] = hits return hits def _best_label(self, aliases, page=None, after_y=None): hits = self._find_labels(aliases) if page is not None: hits = [h for h in hits if h["_page"] == page] if after_y is not None: hits = [h for h in hits if h["y0"] >= after_y] return hits[0] if hits else None def _candidate_right(self, label, validator=None, max_dx=320): row = label["row"] vals = [] for w in row["words"]: if w["x0"] <= label["x1"] + 1: continue if w["x0"] - label["x1"] > max_dx: break text = self._clean(w["_text"]) if self._is_label(text): break vals.append(text) value = self._clean(" ".join(vals)) if value and self._valid_candidate(value, validator): return value return "" def _candidate_below(self, label, validator=None, max_dy=130, x_tol=180): center = (label["x0"] + label["x1"]) / 2 candidates = [] for row in self.page_rows.get(label["_page"], []): if row["_y"] <= label["y1"] + 1: continue dy = row["_y"] - label["y1"] if dy > max_dy: break selected = [] for w in row["words"]: wc = (w["x0"] + w["x1"]) / 2 if abs(wc - center) <= x_tol: if not self._is_label(w["_text"]): selected.append(w["_text"]) value = self._clean(" ".join(selected)) if value and self._valid_candidate(value, validator): candidates.append((dy, value)) return min(candidates)[1] if candidates else "" def _semantic_value(self, aliases, validator=None, max_dx=320, max_dy=130): hits = self._find_labels(aliases) candidates = [] for label in hits: value = self._candidate_right(label, validator, max_dx) if value: candidates.append((self._score_value(label, value, 0), value, label)) value = self._candidate_below(label, validator, max_dy) if value: candidates.append((self._score_value(label, value, 1), value, label)) if not candidates: return "", None score, value, label = max(candidates, key=lambda x: x[0]) return value, label def _score_value(self, label, value, mode): score = 1000.0 if mode: score -= 20 score -= len(value) * 0.4 if self._looks_like_any_known_label(value): score -= 700 return score # ---------- parent extraction ---------- def _extract_parent_semantic(self, result): specs = { "cour_be_importername": self._name_validator, "cour_be_importer_address": self._text_validator, "cour_be_arrivalairport": self._code_validator, "cour_be_iec": lambda v: bool(re.fullmatch(r"\d{8,12}", v)), "cour_be_coo": self._country_validator, "cour_be_airline": self._name_validator, "cour_be_mawbno": self._alnum_validator, "cour_be_couriername": self._name_validator, "cour_be_iecbranch": self._alnum_validator, "cour_be_countryofconsignment": self._country_validator, "cour_be_flightno": self._flight_validator, "cour_be_betype_description": self._type_validator, "cour_be_pkgs": lambda v: self._float(v) is not None, "cour_be_arrivaldate": self._date_validator, "cour_be_hawbno": self._alnum_validator, "cour_be_beno": self._boe_validator, "cour_be_adcode": lambda v: bool(re.fullmatch(r"\d{6,12}", v)), "cour_be_igmno": lambda v: bool(re.fullmatch(r"\d{1,20}", v)), "be_date": self._date_validator, "cour_be_gstin": self._gstin_validator, "cour_be_category": self._text_validator, "cour_be_inward_date": self._date_validator, "cour_be_gwt_kgs": lambda v: self._float(v) is not None, "cour_be_interest_amt": lambda v: self._float(v) is not None, "cour_be_supplier_name": self._name_validator, } for field, validator in specs.items(): aliases = self.PARENT_ALIASES.get(field, []) if not aliases: continue value, label = self._semantic_value(aliases, validator) if value: value = self._normalize_parent(field, value) if value: result[field] = value self._trace(field, value, label) # Special two-column IGM grid: airline / flight / airport / date. self._recover_igm_grid(result) # Special party blocks reconstruct wrapped legal names and addresses. self._recover_party_block( result, ["NAME OF CONSIGNEE", "CONSIGNEE NAME"], ["ADDRESS OF CONSIGNEE", "CONSIGNEE ADDRESS"], "cour_be_importername", "cour_be_importer_address", ) self._recover_party_block( result, ["NAME OF CONSIGNOR", "CONSIGNOR NAME"], ["ADDRESS OF CONSIGNOR", "CONSIGNOR ADDRESS"], "cour_be_supplier_name", None, ) def _recover_igm_grid(self, result): aliases = ["IGM DETAILS", "IMPORT GENERAL MANIFEST DETAILS"] section = self._best_label(aliases) if not section: return page = section["_page"] rows = self.page_rows.get(page, []) start = next((i for i, r in enumerate(rows) if r["_y"] >= section["y1"]), 0) headers = {} data = None for r in rows[start:start+15]: text = self._norm(r["text"]) if "airlines" in text and "flight" in text: headers = self._column_headers(r) elif headers and self._looks_like_igm_data(r["text"]): data = r break if headers and data: for key, field, validator in [ ("airlines", "cour_be_airline", self._name_validator), ("flight", "cour_be_flightno", self._flight_validator), ("airportofarrival", "cour_be_arrivalairport", self._code_validator), ("dateofarrival", "cour_be_arrivaldate", self._date_validator), ]: value = self._cell_by_header(headers, data, key) if value and validator(value): result[field] = self._normalize_parent(field, value) def _column_headers(self, row): out = [] for w in row["words"]: out.append((self._compact(w["_text"]), (w["x0"] + w["x1"]) / 2)) return out def _cell_by_header(self, headers, row, key): target = [x for x in headers if key in x[0]] if not target: return "" x = target[0][1] words = [] for w in row["words"]: if abs(((w["x0"] + w["x1"]) / 2) - x) <= 90: words.append(w["_text"]) return self._clean(" ".join(words)) def _looks_like_igm_data(self, text): return bool( re.search(r"\b[A-Z]{1,4}\s*\d{3,6}\b", text) and re.search(r"\d{1,2}[/-]\d{1,2}[/-]\d{2,4}", text) ) def _recover_party_block(self, result, name_aliases, address_aliases, name_field, address_field): label = self._best_label(name_aliases) if not label: return page = label["_page"] center = (label["x0"] + label["x1"]) / 2 rows = self.page_rows.get(page, []) name_lines = [] address_lines = [] started = False for row in rows: if row["_y"] < label["y1"] - 2: continue if row["_y"] - label["y1"] > 260: break text = self._clean(row["text"]) if not text: continue # Stop at strong section/field labels. if started and self._is_section_or_major_label(text): break selected = [] for w in row["words"]: wc = (w["x0"] + w["x1"]) / 2 if abs(wc - center) <= 250 and not self._is_label(w["_text"]): selected.append(w["_text"]) line = self._clean(" ".join(selected)) if not line: continue if row["_y"] <= label["y1"] + 30 and not name_lines: # Name may be on same row after the label. name_lines.append(line) elif not name_lines: name_lines.append(line) elif len(name_lines) < 3 and not self._looks_like_address(line): name_lines.append(line) else: address_lines.append(line) name = self._clean(" ".join(name_lines)) name = self._repair_wrapped_words(name) if name and self._name_validator(name): result[name_field] = name if address_field: addr = self._clean(" ".join(address_lines)) if addr and not self._looks_like_any_known_label(addr): result[address_field] = addr # ---------- CBE-XIII structural extraction ---------- def _extract_cbe_xiii_structural(self, result): """ Recover CBE-XIII from its semantic grid. CBE-XIII commonly places two or more label/value pairs on the same visual row. The parser therefore derives each value column from the label positions found in that uploaded PDF. No page number or fixed x/y coordinate is used. """ # ---- Header / identifiers ---- self._xiii_pair(result, "cour_be_beno", ["CBE-XIII NUMBER", "CBEXIII NUMBER"], self._boe_validator) self._xiii_pair(result, "cour_be_hawbno", ["HAWB NUMBER", "HAWB NO"], self._alnum_validator) self._xiii_pair(result, "cour_be_coo", ["COUNTRY OF EXPORTATION", "COUNTRY OF ORIGIN"], self._country_validator) self._xiii_pair(result, "cour_be_gstin", ["KYC ID", "GSTIN"], self._gstin_validator) self._xiii_pair(result, "cour_be_iec", ["IMPORT EXPORT CODE", "IEC"], lambda v: bool(re.fullmatch(r"\d{8,12}", v))) self._xiii_pair(result, "cour_be_iecbranch", ["IEC BRANCH CODE", "IMPORT EXPORT BRANCH CODE"], self._alnum_validator) self._xiii_pair(result, "cour_be_pkgs", ["NO OF PACKAGES", "NUMBER OF PACKAGES"], lambda v: self._float(v) is not None) self._xiii_pair(result, "cour_be_gwt_kgs", ["GROSS WEIGHT"], lambda v: self._float(v) is not None) self._xiii_pair(result, "cour_be_adcode", ["AD CODE", "AUTHORISED DEALER CODE"], lambda v: bool(re.fullmatch(r"\d{6,12}", v))) self._xiii_pair(result, "cour_be_interest_amt", ["INTEREST AMOUNT"], lambda v: self._float(v) is not None) # The condensed form exposes arrival/flight as a real visual header # followed by one data row. Derive each column from that header. self._xiii_igm_grid(result) # Party names are multi-line and share a row with the address column. self._xiii_party( result, ["NAME OF CONSIGNEE"], ["ADDRESS OF CONSIGNEE"], "cour_be_importername", "cour_be_importer_address", ) self._xiii_party( result, ["NAME OF CONSIGNOR"], ["ADDRESS OF CONSIGNOR"], "cour_be_supplier_name", None, ) self._xiii_party( result, ["NAME OF THE AUTHORIZED COURIER", "NAME OF AUTHORIZED COURIER"], ["ADDRESS OF AUTHORIZED COURIER"], "cour_be_couriername", None, ) # ---- Item/invoice block ---- item_anchor = self._best_label(["DETAILS OF ITEM"]) if item_anchor: item_rows = self._rows_until_page_end(item_anchor["_page"], item_anchor["_y"]) item = self._empty_item(1) item["_item_section_no"] = 1 field_specs = [ ("cour_be_item_ctsh", ["CTSH"], lambda v: bool(re.fullmatch(r"\d{6,10}", v))), ("cour_be_item_cetsh", ["CETSH"], lambda v: bool(re.fullmatch(r"\d{6,10}", v))), ("cour_be_item_countryoforigin", ["COUNTRY OF ORIGIN"], self._country_validator), ("cour_be_item_itemdescription", ["DESCRIPTION OF GOODS"], self._name_validator), ("cour_be_item_manufacturer", ["NAME OF MANUFACTURER"], self._name_validator), ("cour_be_item_uom", ["UNIT OF MEASURE"], self._code_validator), ("cour_be_item_qty", ["QUANTITY"], lambda v: self._float(v) is not None), ("cour_be_item_upi", ["UNIT PRICE"], lambda v: self._float(v) is not None), ("cour_be_item_upi_currency", ["CURRENCY OF UNIT PRICE"], self._code_validator), ("cour_be_item_exchg_rate", ["RATE OF EXCHANGE"], lambda v: self._float(v) is not None), ("cour_be_item_assessable_value", ["ASSESSABLE VALUE"], lambda v: self._float(v) is not None), ] for field, aliases, validator in field_specs: value = self._xiii_block_value(item_rows, aliases, validator) if value: item[field] = self._normalize_child(field, value) item["cour_be_item_itemgendescription"] = ( item.get("cour_be_item_itemdescription") or "" ) # Invoice fields are sourced from the same item block in CBE-XIII. invoice = self._empty_invoice(1) inv_specs = [ ("cour_be_invoice_no", ["INVOICE NUMBER"], self._alnum_validator), ("cour_be_inv_currency", ["CURRENCY OF INVOICE"], self._code_validator), ("cour_be_inv_value", ["ASSESSABLE VALUE"], lambda v: self._float(v) is not None), ("cour_be_invoice_date", ["DATE OF INVOICE"], self._date_validator), ("cour_be_incoterm", ["INVOICE TERM"], self._text_validator), ("cour_be_inv_pymt_terms", ["TERMS OF PAYMENT"], self._text_validator), ] for field, aliases, validator in inv_specs: value = self._xiii_block_value(item_rows, aliases, validator) if value: invoice[field] = self._normalize_child(field, value) # CBE-XIII invoice value contract is the assessable/invoice value # associated with the customs invoice block when no separate # invoice-value currency pair exists. if not invoice.get("cour_be_inv_value"): value = self._xiii_block_value( item_rows, ["INVOICE VALUE"], lambda v: self._float(v) is not None ) if value: invoice["cour_be_inv_value"] = float(self._float(value) or 0.0) invoice["cour_be_inv_supplier_name"] = result.get("cour_be_supplier_name", "") invoice["suppliers_address"] = self._xiii_supplier_address() if not invoice.get("cour_be_inv_currency"): value = self._xiii_block_value( item_rows, ["CURRENCY OF INVOICE", "CURRENCY"], self._code_validator, ) if value: invoice["cour_be_inv_currency"] = value.upper() if self._invoice_has_identity(invoice): result["invoices"] = [invoice] if self._item_has_identity(item): result["items"] = [item] # Duty/payment are always authoritative when the corresponding # semantic section exists. duty_anchor = self._best_label(["DUTY DETAILS"]) if duty_anchor: duty_rows = self._rows_until_page_end(duty_anchor["_page"], duty_anchor["_y"]) duty = self._extract_duty_from_rows(duty_rows) if result["items"]: item = result["items"][0] for rate_field, amount_field, head in [ ("cour_be_item_bcd_rt", "cour_be_item_bcd_amt", "bcd"), ("cour_be_item_aidc_rt", "cour_be_item_aidc_amt", "aidc"), ("cour_be_item_sws_rt", "cour_be_item_sws_amt", "sws"), ("cour_be_item_igst_rt", "cour_be_item_igst_amt", "igst"), ("cour_be_item_cmpcess_rt", "cour_be_item_cmpcess_amt", "cmpcess"), ]: if head in duty: item[rate_field] = float(duty[head][0]) item[amount_field] = float(duty[head][1]) # CBE-XIII total duty is the duty amount printed in the # payment/summary section, not a sum inferred from nearby text. value = self._xiii_block_value( duty_rows, ["DUTY(RS.)", "DUTY (RS.)", "DUTY"], lambda v: self._float(v) is not None, ) if value: result["cour_be_total_duty"] = float(self._float(value) or 0.0) payment_anchor = self._best_label(["PAYMENT DETAILS"]) if payment_anchor: payment_rows = self._rows_until_page_end( payment_anchor["_page"], payment_anchor["_y"] ) for row in payment_rows: m = re.search( r"\b1\s+(\d{7,20})\s+([0-9,]+(?:\.\d+)?)\s+" r"(\d{1,2}[/-]\d{1,2}[/-]\d{2,4})\b", row["text"], ) if m: result["cour_be_tr6_no"] = m.group(1) result["cour_be_pymt_amt"] = float(self._float(m.group(2)) or 0.0) result["cour_be_total_duty"] = float(self._float(m.group(2)) or 0.0) result["cour_be_challan_date"] = self._format_date(m.group(3)) break # CBE-XIII has no standard standalone BOE date. Use OOC date only # after checking the actual BOE label. if not result.get("be_date") and result.get("out_of_charge_date"): result["be_date"] = result["out_of_charge_date"] # SWS amounts are taken from the explicit Duty Amount column. A # 10% SWS rate does NOT imply a non-zero amount when duty is forgone; # therefore never calculate/replace the amount from the rate alone. self._harden_sws_amounts_from_duty_rows(result) result["cour_be_invoicecount"] = len(result.get("invoices") or []) def _xiii_pair(self, result, field, aliases, validator): hits = self._find_labels(aliases) candidates = [] for label in hits: row = label["row"] # Determine the next semantic label on the same visual row. right_boundary = float("inf") for other in row["words"]: if other["x0"] <= label["x1"] + 1: continue if self._is_label(other["_text"]): right_boundary = other["x0"] break words = [ w["_text"] for w in row["words"] if w["x0"] > label["x1"] + 1 and w["x0"] < right_boundary ] value = self._clean(" ".join(words)) if self._valid_candidate(value, validator): candidates.append((len(value), value, label)) if candidates: _, value, label = max(candidates, key=lambda x: x[0]) result[field] = self._normalize_parent(field, value) def _xiii_party(self, result, name_aliases, address_aliases, name_field, address_field): name_labels = self._find_labels(name_aliases) if not name_labels: return label = name_labels[0] row = label["row"] # Find address label on same row. It defines the right boundary of # the name column and is derived from the uploaded PDF. address_hits = self._find_labels(address_aliases) address_same_row = [ a for a in address_hits if a["_page"] == label["_page"] and abs(a["y0"] - label["y0"]) <= 7 and a["x0"] > label["x1"] ] boundary = min((a["x0"] for a in address_same_row), default=None) name_words = [ w for w in row["words"] if w["x0"] > label["x1"] + 1 and (boundary is None or w["x0"] < boundary) ] name = self._clean(" ".join(w["_text"] for w in name_words)) # Continue on subsequent visual rows while staying inside the same # inferred name column. for r in self.page_rows.get(label["_page"], []): if r["_y"] <= label["_y"] + 5: continue if r["_y"] - label["_y"] > 55: break if self._is_section_or_major_label(r["text"]): break cont = [ w["_text"] for w in r["words"] if w["x0"] > label["x1"] + 1 and (boundary is None or w["x0"] < boundary) ] line = self._clean(" ".join(cont)) if not line: continue if self._looks_like_any_known_label(line): break name = self._clean(name + " " + line) name = self._repair_wrapped_words(name) if self._name_validator(name): result[name_field] = name if address_field: addr = self._xiii_address_from_label(address_aliases) if addr: result[address_field] = addr def _xiii_address_from_label(self, aliases): hits = self._find_labels(aliases) if not hits: return "" label = hits[0] row = label["row"] words = [w["_text"] for w in row["words"] if w["x0"] > label["x1"] + 1] lines = [self._clean(" ".join(words))] if words else [] for r in self.page_rows.get(label["_page"], []): if r["_y"] <= label["_y"] + 5: continue if r["_y"] - label["_y"] > 70: break if self._is_section_or_major_label(r["text"]): break vals = [w["_text"] for w in r["words"] if w["x0"] > label["x1"] - 20] line = self._clean(" ".join(vals)) if line and not self._looks_like_any_known_label(line): lines.append(line) return self._repair_wrapped_words(" ".join(lines)) def _xiii_igm_grid(self, result): header = None for row in self.page_rows.get(1, []) + sum( [rows for p, rows in self.page_rows.items() if p != 1], [] ): compact = self._compact(row["text"]) if "airlines" in compact and "flight" in compact and "airportofarrival" in compact: header = row break if not header: return # Build dynamic column centers from the actual header words. groups = { "airline": ["airlines"], "flight": ["flight"], "airport": ["airportofarrival"], "arrival_date": ["dateofarrival"], } centers = {} for key, tokens in groups.items(): for i, w in enumerate(header["words"]): text = self._compact(w["_text"]) if any(t in text for t in tokens): centers[key] = (w["x0"] + w["x1"]) / 2 break data_rows = [ r for r in self.page_rows.get(header["_page"], []) if header["_y"] < r["_y"] <= header["_y"] + 45 ] if not data_rows: return for key, center in centers.items(): vals = [] for r in data_rows: for w in r["words"]: wc = (w["x0"] + w["x1"]) / 2 if abs(wc - center) <= 65: vals.append(w["_text"]) value = self._clean(" ".join(vals)) if key == "airline": # Join PDF hyphenation across rows. value = self._repair_wrapped_words(value) if self._name_validator(value): result["cour_be_airline"] = value elif key == "flight" and self._flight_validator(value): result["cour_be_flightno"] = value elif key == "airport" and self._code_validator(value): result["cour_be_arrivalairport"] = value elif key == "arrival_date" and self._date_validator(value): result["cour_be_arrivaldate"] = self._format_date(value) # Arrival date also supplies inward date where no separate label exists. if result.get("cour_be_arrivaldate") and not result.get("cour_be_inward_date"): result["cour_be_inward_date"] = result["cour_be_arrivaldate"] def _xiii_block_value(self, rows, aliases, validator): candidates = [] for row in rows: for alias in aliases: label = self._label_in_row(row, alias) if not label: continue value = self._candidate_right(label, validator, 260) if value: candidates.append((len(value), value)) if candidates: return max(candidates, key=lambda x: x[0])[1] return "" def _xiii_supplier_address(self): return self._xiii_address_from_label( ["ADDRESS OF CONSIGNOR", "SUPPLIER ADDRESS", "ADDRESS OF SUPPLIER"] ) def _rows_until_page_end(self, page, y): return [ r for r in self.page_rows.get(page, []) if r["_y"] >= y ] # ---------- invoice extraction ---------- def _extract_invoices(self, result): rows = [] # First, detect explicit invoice sections dynamically. anchors = self._find_labels(self.SECTION_ALIASES["invoice"]) for anchor in anchors: block_rows = self._rows_until_next_section(anchor["_page"], anchor["_y"]) candidate = self._invoice_from_rows(block_rows) if candidate and self._invoice_has_identity(candidate): rows.append(candidate) # CBE-XIII has invoice information inside the item section rather than # a separate multi-page invoice table. Recover it from invoice labels. if not rows: candidate = self._invoice_from_global_labels() if candidate and self._invoice_has_identity(candidate): rows.append(candidate) # Generic tables may expose multiple invoice rows. table_rows = self._map_invoice_tables() rows.extend(table_rows) rows = self._merge_invoice_rows(rows) result["invoices"] = rows def _invoice_from_rows(self, block_rows): row = self._empty_invoice(1) for field, aliases in self.INVOICE_ALIASES.items(): if field == "cour_be_inv_related": continue value = self._value_from_row_collection(block_rows, aliases, self._child_validator(field)) if value: row[field] = self._normalize_child(field, value) return row def _invoice_from_global_labels(self): row = self._empty_invoice(1) for field, aliases in self.INVOICE_ALIASES.items(): if field == "cour_be_inv_related": continue value, _ = self._semantic_value(aliases, self._child_validator(field)) if value: row[field] = self._normalize_child(field, value) return row def _map_invoice_tables(self): out = [] for table in self.tables: text = self._compact(" ".join(" ".join(r) for r in table["rows"][:8])) if "invoicenumber" not in text and "invoiceno" not in text: continue headers = self._detect_headers(table["rows"]) if not headers: continue for raw in table["rows"][headers[0] + 1:]: values = self._map_cells(headers[1], raw) if not any(values.values()): continue row = self._empty_invoice(len(out) + 1) for field, aliases in self.INVOICE_ALIASES.items(): key = self._best_header_key(aliases, values) if key: row[field] = self._normalize_child(field, values[key]) if self._invoice_has_identity(row): out.append(row) return out # ---------- item extraction ---------- def _extract_items(self, result): items = [] # Explicit item sections: detect every occurrence, independent of page. item_anchors = [] for page, rows in self.page_rows.items(): for row in rows: if re.search(r"DETAILS\s+OF\s+ITEM(?:\s*-\s*\d+|\(S\))?", row["text"], re.I): item_anchors.append(row) if item_anchors: for idx, anchor in enumerate(item_anchors, 1): block = self._rows_until_next_item_or_section(anchor["_page"], anchor["_y"]) item = self._item_from_rows(block, idx) if self._item_has_identity(item): items.append(item) # CBE-XIII has one compact item block. if not items: for anchor in self._find_labels(["DETAILS OF ITEM", "DETAILS OF ITEMS"]): block = self._rows_until_next_section(anchor["_page"], anchor["_y"], allow_duty=True) item = self._item_from_rows(block, 1) if self._item_has_identity(item): items.append(item) break # Generic table mapping can discover multiple rows. items.extend(self._map_item_tables()) items = self._dedupe_items(items) result["items"] = items def _item_from_rows(self, rows, serial): item = self._empty_item(serial) # Internal provenance anchor: this identifies the physical # ``Details Of Item - N`` block that produced the observation. # It is intentionally removed by the final schema pass and is never # exposed in the Frappe JSON. item["_item_section_no"] = int(serial) if str(serial).isdigit() else serial for field, aliases in self.ITEM_ALIASES.items(): value = self._value_from_row_collection(rows, aliases, self._child_validator(field)) if value: item[field] = self._normalize_child(field, value) # Duty rows often sit inside the same visual item block. duty = self._extract_duty_from_rows(rows) for rate_field, amount_field, head in [ ("cour_be_item_bcd_rt", "cour_be_item_bcd_amt", "bcd"), ("cour_be_item_sws_rt", "cour_be_item_sws_amt", "sws"), ("cour_be_item_igst_rt", "cour_be_item_igst_amt", "igst"), ("cour_be_item_aidc_rt", "cour_be_item_aidc_amt", "aidc"), ("cour_be_item_cmpcess_rt", "cour_be_item_cmpcess_amt", "cmpcess"), ]: if duty.get(head): item[rate_field] = float(duty[head][0]) item[amount_field] = float(duty[head][1]) # General Description is an independent field. Do not manufacture a # value from Item Description when the PDF leaves General Description # blank; doing so can hide layout-extraction failures and can also # cause the following metadata row to be interpreted as the value. return item def _map_item_tables(self): out = [] for table in self.tables: text = self._compact(" ".join(" ".join(r) for r in table["rows"][:10])) if not any(k in text for k in ("itemdescription", "descriptionofgoods", "quantity", "ctsh")): continue headers = self._detect_headers(table["rows"]) if not headers: continue header_index, header_map = headers for raw in table["rows"][header_index + 1:]: values = self._map_cells(header_map, raw) if not any(values.values()): continue item = self._empty_item(len(out) + 1) for field, aliases in self.ITEM_ALIASES.items(): key = self._best_header_key(aliases, values) if key: item[field] = self._normalize_child(field, values[key]) if self._item_has_identity(item): out.append(item) return out # ---------- dynamic row/table helpers ---------- def _rows_until_next_section(self, page, y, allow_duty=False): stops = [] for row in self.page_rows.get(page, []): if row["_y"] <= y + 3: continue text = self._norm(row["text"]) if any(self._norm(a) in text for group in self.SECTION_ALIASES.values() for a in group): if self._norm(row["text"]) not in {self._norm(a) for a in self.SECTION_ALIASES["items"]}: stops.append(row["_y"]) break end = min(stops) if stops else y + 450 return [r for r in self.page_rows.get(page, []) if y <= r["_y"] <= end] def _rows_until_next_item_or_section(self, page, y): out = [] for row in self.page_rows.get(page, []): if row["_y"] < y: continue text = self._norm(row["text"]) if row["_y"] > y + 3 and re.search(r"details of item(?:\s*-\s*\d+)?", text): break if row["_y"] > y + 3 and any(self._norm(a) in text for g in ( self.SECTION_ALIASES["duty"], self.SECTION_ALIASES["payment"] ) for a in g): break out.append(row) return out def _value_from_row_collection(self, rows, aliases, validator=None): candidates = [] for row in rows: text = self._clean(row["text"]) for alias in aliases: m = re.search(re.escape(alias).replace(r"\ ", r"\s+") + r"\s*:?\s*(.*)$", text, re.I) if m: value = self._clean(m.group(1)) if value and self._valid_candidate(value, validator): candidates.append((len(value), value)) # If the label is at end of line, look at next row. if not value: idx = rows.index(row) if idx + 1 < len(rows): nxt = self._clean(rows[idx+1]["text"]) if self._valid_candidate(nxt, validator): candidates.append((len(nxt), nxt)) if candidates: return max(candidates, key=lambda x: x[0])[1] # Visual fallback over each label in the block. for row in rows: for alias in aliases: if self._compact(alias) in self._compact(row["text"]): label = self._label_in_row(row, alias) if label: v = self._candidate_right(label, validator, 280) if v: return v return "" def _label_in_row(self, row, alias): parts = self._norm(alias).split() words = row["words"] for i in range(len(words) - len(parts) + 1): if self._compact(" ".join(w["_text"] for w in words[i:i+len(parts)])) == self._compact(alias): ws = words[i:i+len(parts)] return { "_page": row["_page"], "_page": row["_page"], "x0": ws[0]["x0"], "x1": ws[-1]["x1"], "y0": ws[0]["y0"], "y1": ws[-1]["y1"], "row": row, } return None def _detect_headers(self, rows): best = None for i, row in enumerate(rows[:12]): text = self._compact(" ".join(row)) score = sum( token in text for token in ( "invoicenumber", "invoiceno", "description", "quantity", "qty", "ctsh", "cetsh", "assessablevalue", "currency", ) ) if score >= 2: mapping = {} for cell in row: key = self._compact(cell) if key: mapping[key] = None if best is None or score > best[0]: best = (score, i, mapping) if not best: return None return best[1], best[2] def _map_cells(self, header_map, raw): out = {} keys = list(header_map.keys()) for i, key in enumerate(keys): if i < len(raw): out[key] = self._clean(raw[i]) return out def _best_header_key(self, aliases, values): aliases_c = [self._compact(a) for a in aliases] for key in values: for alias in aliases_c: if key == alias or alias in key or key in alias: return key return None # ---------- payment / duty ---------- def _extract_payment_semantic(self, result): # Find a Payment Details section anywhere, not on a fixed page. for anchor in self._find_labels(self.SECTION_ALIASES["payment"]): rows = self._rows_until_next_section(anchor["_page"], anchor["_y"]) for i, row in enumerate(rows): text = self._clean(row["text"]) if re.search(r"TR[\-\s]?6\s+CHALLAN|CHALLAN\s+NUMBER", text, re.I): nums = re.findall(r"\b\d{7,20}\b", text) if nums: result["cour_be_tr6_no"] = nums[-1] if i + 1 < len(rows): nxt = self._clean(rows[i+1]["text"]) m = re.search(r"\b\d[\d,]*(?:\.\d+)?\b", nxt) if m: result["cour_be_pymt_amt"] = float(self._float(m.group()) or 0.0) date = self._find_date_near_rows(rows, i) if date: result["cour_be_challan_date"] = self._format_date(date) break if not result["cour_be_pymt_amt"]: # Semantic Total Amount is safer than arbitrary numeric extraction. value, _ = self._semantic_value( ["TOTAL AMOUNT"], lambda v: self._float(v) is not None ) if value: result["cour_be_pymt_amt"] = float(self._float(value) or 0.0) if not result["cour_be_total_duty"] and result["cour_be_pymt_amt"]: result["cour_be_total_duty"] = float(result["cour_be_pymt_amt"]) # Interest is label-authoritative. value, _ = self._semantic_value( ["INTEREST AMOUNT", "INTEREST AMT"], lambda v: self._float(v) is not None, ) if value: result["cour_be_interest_amt"] = float(self._float(value) or 0.0) def _find_date_near_rows(self, rows, idx): for row in rows[idx:idx+4]: m = re.search(r"\b\d{1,2}[/-]\d{1,2}[/-]\d{2,4}\b", row["text"]) if m: return m.group() return "" def _extract_duty_from_rows(self, rows): text = "\n".join(r["text"] for r in rows) return self._parse_duty_text(text) def _parse_duty_text(self, text): result = {} lines = [self._clean(x) for x in text.splitlines() if self._clean(x)] heads = { "bcd": ["BCD", "BASIC CUSTOMS DUTY"], "sws": ["SW SRCHRG", "SWS", "SOCIAL WELFARE SURCHARGE"], "igst": ["IGST"], "aidc": ["AIDC"], "cmpcess": ["CMPNSTRY", "COMPENSATION CESS", "CMP CESS"], } for line in lines: compact = self._compact(line) for head, aliases in heads.items(): if not any(self._compact(a) in compact for a in aliases): continue # Remove header words before collecting numeric tokens. nums = re.findall(r"\b\d+(?:\.\d+)?\b", line) if not nums: continue # In Courier duty rows, the first number after the duty-head # is the ad-valorem rate and the final number is duty amount. rate = float(self._float(nums[0]) or 0.0) amount = float(self._float(nums[-1]) or 0.0) result[head] = (rate, amount) return result # ---------- related party ---------- def _extract_related_party(self, result): # Only accept Y/N after the complete question. Generic "Related", # "Term", "to", etc. are never valid values. pattern = re.compile( r"(?:ARE\s+THE\s+BUYER\s+AND\s+SELLER\s+RELATED|" r"BUYER\s+AND\s+SELLER\s+RELATED)\s*\??\s*:?\s*([YN])\b", re.I, ) matches = list(pattern.finditer(self.full_text)) if matches and result["invoices"]: result["invoices"][0]["cour_be_inv_related"] = matches[0].group(1).upper() # ---------- generalized table engine ---------- def _compact_header_token(self, value): """Normalize a PDF table header without depending on its spelling.""" value = self._clean(value).lower() value = re.sub(r"^\s*\d+\s*[.)-]\s*", "", value) value = re.sub(r"\b(?:sr|sl|sno|no)\s*[.)-]?\s*", " ", value) return re.sub(r"[^a-z0-9]+", "", value) def _field_aliases(self, table_name, field): aliases = [] source = self.INVOICE_ALIASES if table_name == "invoices" else self.ITEM_ALIASES aliases.extend(source.get(field, [])) aliases.append(field) suffix = re.sub(r"^cour_be_(?:inv_|item_)?", "", field) aliases.append(suffix.replace("_", " ")) # Deterministic abbreviation expansion. for a in list(aliases): aliases.extend([ str(a).replace(" NUMBER", " NO"), str(a).replace(" NO", " NUMBER"), str(a).replace(" DATE", " DT"), str(a).replace(" DT", " DATE"), str(a).replace(" QUANTITY", " QTY"), str(a).replace(" QTY", " QUANTITY"), str(a).replace(" AMOUNT", " AMT"), str(a).replace(" AMT", " AMOUNT"), ]) out=[]; seen=set() for a in aliases: k=self._compact_header_token(a) if k and k not in seen: seen.add(k); out.append(a) return out def _header_field_score(self, header, aliases): h=self._compact_header_token(header) if not h: return 0 best=0 ht=set(re.findall(r"[a-z0-9]+", self._norm(header))) for alias in aliases: a=self._compact_header_token(alias) if not a: continue if h == a: best=max(best, 1000+len(a)) elif len(a)>=5 and a in h: best=max(best, 720+len(a)) elif len(h)>=5 and h in a: best=max(best, 650+len(h)) else: at=set(re.findall(r"[a-z0-9]+", self._norm(alias))) overlap=len(ht & at) if overlap >= 2: best=max(best, 400+overlap*30+min(len(a),30)) return best def _canonical_child_field_for_header(self, table_name, header): fields=self.INVOICE_FIELDS if table_name=="invoices" else self.ITEM_FIELDS best_field=None; best_score=0 for field in fields: if field == "doctype": continue score=self._header_field_score(header, self._field_aliases(table_name, field)) if score>best_score: best_score=score; best_field=field return best_field if best_score>=400 else None def _classify_table_v2(self, rows): """Classify a table from semantic header signatures, not its page.""" sample=rows[:10] scores={"invoices":0,"items":0,"duty":0} for row in sample: for cell in row: c=self._compact_header_token(cell) if not c: continue if any(self._header_field_score(cell,self._field_aliases("invoices",f))>=700 for f in self.INVOICE_FIELDS if f!="doctype"): scores["invoices"]+=1 if any(self._header_field_score(cell,self._field_aliases("items",f))>=700 for f in self.ITEM_FIELDS if f!="doctype"): scores["items"]+=1 if c in {"bcd","aidc","sws","swsrchr","igst","cmpnstry","rate","amount","dutyhead","duty"}: scores["duty"]+=1 kind=max(scores,key=scores.get) if scores[kind] < 2: return None # Prevent an invoice table with a generic quantity/value column from # being classified as items solely because of one shared header. return kind def _find_header_row_v2(self, table_name, rows): fields=self.INVOICE_FIELDS if table_name=="invoices" else self.ITEM_FIELDS best=None for idx,row in enumerate(rows[:12]): mappings=[] score=0 for col,cell in enumerate(row): field=None; fs=0 for f in fields: if f=="doctype": continue s=self._header_field_score(cell,self._field_aliases(table_name,f)) if s>fs: fs=s; field=f if field and fs>=400: mappings.append((col,field,fs,cell)); score += min(fs,1000) unique=len({x[1] for x in mappings}) if unique>=2: score += unique*120 if best is None or score>best[0]: best=(score,idx,mappings) return best def _map_generic_table_v2(self, table_name, rows): header=self._find_header_row_v2(table_name,rows) if not header: return [] _,hidx,mappings=header records=[] for raw in rows[hidx+1:]: if not raw or not any(self._clean(c) for c in raw): continue rec=self._empty_invoice(len(records)+1) if table_name=="invoices" else self._empty_item(len(records)+1) populated=0 for col,field,_,_ in mappings: if col>=len(raw): continue value=self._clean(raw[col]) if not value: continue validator=self._child_validator(field) if not self._valid_candidate(value,validator): continue rec[field]=self._normalize_child(field,value) populated+=1 # General Description is independent from Item Description. # Keep it empty when the source cell/label has no actual value. identity=self._invoice_has_identity(rec) if table_name=="invoices" else self._item_has_identity(rec) if identity and populated>=2: records.append(rec) return records def _recover_from_generic_tables_v2(self, result, second_pass=False): invoice_candidates=[]; item_candidates=[] for table in self.tables: rows=table.get("rows") or [] if len(rows)<2: continue kind=self._classify_table_v2(rows) if kind=="invoices": invoice_candidates.extend(self._map_generic_table_v2("invoices",rows)) elif kind=="items": item_candidates.extend(self._map_generic_table_v2("items",rows)) elif kind=="duty": # Duty tables are handled separately from generic child rows. continue if invoice_candidates: merged=self._merge_invoice_rows((result.get("invoices") or [])+invoice_candidates) result["invoices"]=merged if item_candidates: result["items"]=self._dedupe_items((result.get("items") or [])+item_candidates) # If the document has a single invoice section but no table, preserve # one invoice row so invoice-level semantic recovery has a target. if not result.get("invoices") and not second_pass: candidate=self._invoice_from_global_labels() if candidate and self._invoice_has_identity(candidate): result["invoices"]=[candidate] def _reconcile_child_rows(self, result): """Keep child rows schema-pure and repair only semantically invalid values.""" result["invoices"] = self._merge_invoice_rows(result.get("invoices") or []) result["items"] = self._dedupe_items(result.get("items") or []) # Related party is a strict Y/N field. Never allow nearby prose. related_matches=re.findall( r"(?:ARE\s+THE\s+BUYER\s+AND\s+SELLER\s+RELATED|BUYER\s+AND\s+SELLER\s+RELATED)\s*\??\s*:?\s*([YN])\b", self.full_text,re.I, ) related=related_matches[0].upper() if related_matches else "" if related: for row in result["invoices"]: row["cour_be_inv_related"]=related else: for row in result["invoices"]: if str(row.get("cour_be_inv_related") or "").upper() not in {"Y","N"}: row["cour_be_inv_related"]="" # BCD/SWS/IGST etc. must come from an actual duty row, not arbitrary # numeric neighbors. Associate duty data with each item block where # possible, otherwise use the strongest duty table in the document. duty=self._parse_duty_text(self.full_text) if duty: for row in result["items"]: for head,rf,af in [ ("bcd","cour_be_item_bcd_rt","cour_be_item_bcd_amt"), ("sws","cour_be_item_sws_rt","cour_be_item_sws_amt"), ("igst","cour_be_item_igst_rt","cour_be_item_igst_amt"), ("aidc","cour_be_item_aidc_rt","cour_be_item_aidc_amt"), ("cmpcess","cour_be_item_cmpcess_rt","cour_be_item_cmpcess_amt"), ]: if head in duty: row[rf]=float(duty[head][0]); row[af]=float(duty[head][1]) # ---------- dynamic party block override ---------- def _recover_party_block(self, result, name_aliases, address_aliases, name_field, address_field): labels=self._find_labels(name_aliases) if not labels: return # Score all name anchors and choose the one whose following visual block # looks most like a party name. This prevents selecting instruction text. scored=[] for label in labels: page=label["_page"]; rows=self.page_rows.get(page,[]) lines=[] for row in rows: if row["_y"] < label["y1"]-2: continue if row["_y"]-label["y1"] > 360: break if row["_y"] > label["y1"]+5 and self._is_section_or_major_label(row["text"]): break selected=[w["_text"] for w in row["words"] if w["x1"]>label["x1"]+1 and w["x0"]20: score+=20 if re.search(r"PRIVATE|LIMITED|INC|LLC|LTD|CORP|HAUSER|INDIA|USA",candidate,re.I): score+=15 score-=len(candidate)*0.05 scored.append((score,candidate,label)) if not scored: return _,name,label=max(scored,key=lambda x:x[0]) if name_field and self._name_validator(name): result[name_field]=name if not address_field: return page=label["_page"]; rows=self.page_rows.get(page,[]) address=[]; started=False for row in rows: if row["_y"]<=label["y1"]+10: continue if row["_y"]-label["y1"]>420: break text=self._clean(row["text"]) if self._is_section_or_major_label(text): break # If an explicit address label is encountered, include its value # block and continue until the next major heading. if any(self._compact(a) in self._compact(text) for a in address_aliases): started=True continue if started or address: line=self._clean(" ".join(w["_text"] for w in row["words"] if not self._is_label(w["_text"]))) if line: address.append(line) addr=self._clean(" ".join(address)) if addr and not self._looks_like_any_known_label(addr): result[address_field]=addr # ---------- generic table recovery ---------- def _recover_from_tables(self, result): for table in self.tables: rows = table["rows"] text = self._compact(" ".join(" ".join(r) for r in rows[:10])) if "invoicenumber" in text or "invoiceno" in text: recs = self._map_invoice_tables() if recs: result["invoices"] = self._merge_invoice_rows(result["invoices"] + recs) if any(k in text for k in ("descriptionofgoods", "itemdescription", "assessablevalue", "ctsh")): recs = self._map_item_tables() if recs: result["items"] = self._dedupe_items(result["items"] + recs) def _regex_label_value(self, aliases, validator=None): """Fallback label/value scanner over the text stream with scoring.""" candidates=[] for alias in aliases: pattern=re.escape(alias).replace(r"\ ",r"\s+") for m in re.finditer(pattern+r"\s*[:\-]?\s*([^\n]{1,180})",self.full_text,re.I): value=self._clean(m.group(1)) if not value: continue # Stop at the next obvious label on the same text line. value=re.split(r"\s+(?=[A-Z][A-Za-z /()&.-]{2,40}\s*:)",value)[0].strip() if not self._valid_candidate(value,validator): continue score=1000 score-=len(value)*0.2 if self._looks_like_any_known_label(value): score-=700 # Prefer candidates whose surrounding text contains the same # semantic section and reject instruction-heavy regions. ctx=self.full_text[max(0,m.start()-180):m.start()+240].upper() if "DECLAR" in ctx or "NOTE:" in ctx: score-=120 candidates.append((score,value)) return max(candidates,key=lambda x:x[0])[1] if candidates else "" # ---------- missing-field recovery ---------- def _recover_missing_parent_fields(self, result): for field in self.PARENT_FIELDS: if field == "doctype": continue if not self._empty_value(result.get(field)): continue aliases = self.PARENT_ALIASES.get(field, []) if not aliases: continue validator = self._parent_validator(field) value, label = self._semantic_value(aliases, validator, max_dx=360, max_dy=180) if not value: value = self._regex_label_value(aliases, validator) label = None if value: result[field] = self._normalize_parent(field, value) self._trace(field, result[field], label) # Never derive MAWB/IGM from BOE/HAWB unless the real label exists. # For CBE-XIII these fields legitimately remain empty when absent. if not result.get("be_date") and result.get("out_of_charge_date"): result["be_date"] = result["out_of_charge_date"] def _recover_missing_child_fields(self, result): for table_name, fields in ( ("invoices", self.INVOICE_FIELDS), ("items", self.ITEM_FIELDS), ): for row in result.get(table_name, []): for field in fields: if field == "doctype" or not self._empty_value(row.get(field)): continue aliases = ( self.INVOICE_ALIASES if table_name == "invoices" else self.ITEM_ALIASES ).get(field, []) if not aliases: continue value, _ = self._semantic_value( aliases, self._child_validator(field), max_dx=360, max_dy=180 ) if not value: value = self._regex_label_value(aliases, self._child_validator(field)) if value: row[field] = self._normalize_child(field, value) # ---------- V17 logical child-row reconciliation ---------- def _invoice_row_strength(self, row): """Score an invoice observation by business evidence, not field count.""" score = 0 inv_no = self._clean(row.get("cour_be_invoice_no")) inv_value = row.get("cour_be_inv_value") supplier = self._clean(row.get("cour_be_inv_supplier_name")) address = self._clean(row.get("suppliers_address")) inv_date = self._clean(row.get("cour_be_invoice_date")) currency = self._clean(row.get("cour_be_inv_currency")) if inv_no and not self._looks_like_any_known_label(inv_no): score += 100 try: if float(inv_value or 0) > 0: score += 70 except (TypeError, ValueError): pass if self._strict_party_name(supplier): score += 45 if address and self._looks_like_address(address): score += 30 if inv_date and self._date_validator(inv_date): score += 20 if re.fullmatch(r"[A-Z]{3}", currency.upper()): score += 15 if str(row.get("cour_be_inv_related") or "").upper() in {"Y", "N"}: score += 5 return score def _item_row_strength(self, row): """Score an item observation by merchandise evidence.""" score = 0 desc = self._clean(row.get("cour_be_item_itemdescription")) cts = self._clean(row.get("cour_be_item_ctsh")) cet = self._clean(row.get("cour_be_item_cetsh")) manufacturer = self._clean(row.get("cour_be_item_manufacturer")) country = self._clean(row.get("cour_be_item_countryoforigin")) assess = row.get("cour_be_item_assessable_value") qty = row.get("cour_be_item_qty") upi = row.get("cour_be_item_upi") if desc and not self._looks_like_any_known_label(desc) and len(desc) >= 2: score += 100 if re.fullmatch(r"\d{4,12}", re.sub(r"\D", "", cts or "")): score += 75 if re.fullmatch(r"\d{4,12}", re.sub(r"\D", "", cet or "")): score += 45 if self._strict_party_name(manufacturer): score += 20 if country and not self._looks_like_any_known_label(country): score += 15 for value, weight in ((assess, 55), (qty, 30), (upi, 20)): try: if float(value or 0) != 0: score += weight except (TypeError, ValueError): pass # Explicitly penalize known document furniture. noise = " ".join(str(row.get(k, "") or "") for k in row).upper() for token in ( "PORT:", "NOTE:", "THIS IS AN ELECTRONIC COPY", "PAGE ", "DETAILS OF", "PARTICULARS OF", "CURRENCYFOR", "EXIMSCHEME", "SPECIAL REQUEST", "ADDITIONAL INFORMATION", "SHIPPING BILL DETAILS", ): if token in noise: score -= 150 return score def _explicit_invoice_slots(self): """Return logical invoice numbers/sections actually printed in the PDF.""" slots = [] # Numbered invoice headings are the strongest structural evidence. for m in re.finditer( r"DETAILS\s+OF\s+INVOICE\s*-\s*(\d+)", self.full_text, re.I, ): n = int(m.group(1)) if n not in slots: slots.append(n) # Unnumbered invoice section: one section means one logical invoice. if not slots: matches = re.findall( r"\bDETAILS\s+OF\s+INVOICE(?:S)?\b", self.full_text, re.I, ) if matches: return [1] return sorted(slots) def _explicit_item_slots(self): """Return logical item numbers actually printed in the PDF.""" slots = [] for m in re.finditer( r"DETAILS\s+OF\s+ITEM\s*-\s*(\d+)", self.full_text, re.I, ): n = int(m.group(1)) if n not in slots: slots.append(n) if not slots: # A compact CBE-XIII table may have a single item section. matches = re.findall( r"\bDETAILS\s+OF\s+ITEM(?:S)?\b", self.full_text, re.I, ) if matches: return [1] return sorted(slots) def _merge_partial_invoice_into(self, target, source): """Merge only meaningful non-empty fields from source into target.""" for field in self.INVOICE_FIELDS: if field == "doctype": continue newv = source.get(field) oldv = target.get(field) if self._empty_value(oldv) and not self._empty_value(newv): target[field] = newv def _merge_partial_item_into(self, target, source): """Merge only meaningful non-empty fields from source into target.""" for field in self.ITEM_FIELDS: if field == "doctype": continue newv = source.get(field) oldv = target.get(field) if self._empty_value(oldv) and not self._empty_value(newv): target[field] = newv def _recover_explicit_item_sections_dynamic(self, result): """Recover every explicit ``Details Of Item - N`` block dynamically. This is a conservative reconciliation pass for multi-page CBE-XIV and similar future forms. The section heading is the logical row boundary; page numbers and fixed coordinates are deliberately not used. The important distinction is that an item section may start on one page and continue on several following pages. The older visual extractor stopped at the physical page boundary, which could make one section disappear or cause the next section to inherit its values. """ text = self.full_text or "" # Accept normal, OCR-fragmented, and punctuation variants of the # section heading. Capture the printed logical item number. heading_re = re.compile( r"DETAILS\s*OF\s*ITEM\s*(?:[-:\u2010\u2011\u2012\u2013\u2014]\s*)?(\d+)", re.I, ) matches = list(heading_re.finditer(text)) if not matches: return # Keep the first structural occurrence of each logical item number. sections = [] seen = set() for m in matches: number = int(m.group(1)) if number in seen: continue seen.add(number) sections.append((number, m.start(), m.end())) sections.sort(key=lambda x: x[1]) recovered = [] for pos, (number, start, end) in enumerate(sections): block_end = sections[pos + 1][1] if pos + 1 < len(sections) else len(text) block = text[end:block_end] # Stop only at true document-level boundaries. Do not stop at a # page break: item sections are allowed to continue across pages. for stop_pat in ( r"SHIPPING\s+BILL\s+DETAILS", r"PAYMENTS?\s+MADE", r"PAYMENT\s+DETAILS", r"DETAILS\s+RELATING\s+TO\s+DUTY\s+EXEMPTION", r"DECLARATION\b", ): sm = re.search(stop_pat, block, re.I) if sm: block = block[:sm.start()] break item = self._empty_item(number) item["_item_section_no"] = number def text_value(pattern, field): m = re.search(pattern, block, re.I | re.S) if not m: return value = self._clean(m.group(1)) if not value: return if field in self.CHILD_FLOAT_FIELDS: value = float(self._float(value) or 0.0) else: value = self._repair_wrapped_words(value) if field in { "cour_be_item_itemdescription", "cour_be_item_itemgendescription", "cour_be_item_manufacturer", "cour_be_item_countryoforigin", } and self._looks_like_any_known_label(value): return item[field] = value boundary = ( r"(?=\s+(?:General\s+Description|Currency\s+for\s+Unit\s+Price|" r"Unit\s+Price|Quantity|Unit\s+of\s+Measure|UOM|Rate\s+of\s+Exchange|" r"Name\s+of\s+Manufacturer|Manufacturer|Country\s*of\s*Origin|" r"RITC|CTSH|CETSH|Assessable\s*Value|Brand|Model|Grade|Specification|" r"End\s+Use\s+of\s+Item)\s*:|$)" ) text_value(r"Item\s+Description\s*:\s*(.+?)" + boundary, "cour_be_item_itemdescription") # General Description must never consume the following commercial # metadata. An empty source field therefore remains empty. text_value( r"General\s+Description\s*:\s*(.*?)(?=\s*(?:Currency\s+for\s+Unit\s+Price|" r"Unit\s+Price|Quantity\s*:|Unit\s+of\s+Measure|UOM\s*:|Rate\s+of\s+Exchange|" r"Name\s+of\s+Manufacturer|Manufacturer\s*:|Country\s*of\s*Origin|RITC\s*:|" r"CTSH\s*:|CETSH\s*:|Assessable\s*Value\s*:|Brand\s*:|Model\s*:|Grade\s*:|" r"Specification\s*:|End\s+Use\s+of\s+Item\s*:|$))", "cour_be_item_itemgendescription", ) text_value(r"Currency\s+for\s+Unit\s+Price\s*:\s*([A-Za-z]{3,})", "cour_be_item_upi_currency") text_value(r"Unit\s+Price\s*:\s*([0-9][\d,]*(?:\.\d+)?)", "cour_be_item_upi") text_value(r"(?:Unit\s+of\s+Measure|UOM)\s*:\s*([A-Za-z0-9._/-]+)", "cour_be_item_uom") text_value(r"Quantity\s*:\s*([0-9][\d,]*(?:\.\d+)?)", "cour_be_item_qty") text_value(r"Rate\s+Of\s+Exchange\s*:\s*([0-9][\d,]*(?:\.\d+)?)", "cour_be_item_exchg_rate") text_value(r"RITC\s*:\s*([A-Za-z0-9./_-]+)", "cour_be_item_ritc") text_value(r"CTSH\s*:\s*(\d{4,12})", "cour_be_item_ctsh") text_value(r"CETSH\s*:\s*(\d{4,12})", "cour_be_item_cetsh") text_value( r"Assessable\s*Value\s*:\s*([0-9][\d,]*(?:\.\d+)?)", "cour_be_item_assessable_value", ) # Manufacturer: first use the complete semantic span before the # next metadata label. Then allow a PDF-wrapped continuation after # Brand:N.A only when that continuation is immediately adjacent to # the manufacturer block. No manufacturer name is hardcoded. mm = re.search( r"(?:Name\s+of\s+Manufacturer|Manufacturer)\s*:\s*(.+?)" r"(?=\s+(?:Brand|Model|Grade|Specification|Quantity|UOM|" r"Country\s*of\s*Origin|RITC|CTSH|CETSH|Assessable\s*Value)\s*:|$)", block, re.I | re.S, ) if mm: manufacturer = self._repair_wrapped_words(self._clean(mm.group(1))) manufacturer = re.sub(r"(?i)\s*Brand\s*:\s*.*$", "", manufacturer).strip() if manufacturer and not self._looks_like_any_known_label(manufacturer): item["cour_be_item_manufacturer"] = manufacturer # If the text stream split a manufacturer around a metadata label, # use the existing semantic continuation resolver. It works from # the actual captured prefix and surrounding block, not a document- # specific manufacturer dictionary. manufacturer = self._clean(item.get("cour_be_item_manufacturer")) if manufacturer: try: manufacturer = self._resolve_manufacturer_continuation(manufacturer, block) except Exception: pass try: manufacturer = self._recover_full_manufacturer(manufacturer, block) except Exception: pass item["cour_be_item_manufacturer"] = self._repair_wrapped_words(manufacturer) # Duty heads belong to this physical section only. duty = self._parse_duty_text(block) for head, rate_field, amount_field in ( ("bcd", "cour_be_item_bcd_rt", "cour_be_item_bcd_amt"), ("aidc", "cour_be_item_aidc_rt", "cour_be_item_aidc_amt"), ("sws", "cour_be_item_sws_rt", "cour_be_item_sws_amt"), ("igst", "cour_be_item_igst_rt", "cour_be_item_igst_amt"), ("cmpcess", "cour_be_item_cmpcess_rt", "cour_be_item_cmpcess_amt"), ): if head in duty: item[rate_field] = float(duty[head][0]) item[amount_field] = float(duty[head][1]) if self._item_has_identity(item): recovered.append(item) if not recovered: return # Merge by the explicit physical section number. This is the key # protection against both under-counting and over-deduplication. existing = [r for r in (result.get("items") or []) if isinstance(r, dict)] by_section = {} for row in existing: sec = row.get("_item_section_no") if sec not in (None, ""): by_section[str(sec)] = row for row in recovered: sec = str(row.get("_item_section_no")) target = by_section.get(sec) if target is None: existing.append(row) by_section[sec] = row continue # Prefer the stronger section-level recovery for contaminated or # incomplete observations, then retain any additional fields from # the older extraction. if self._item_row_strength(row) >= self._item_row_strength(target): old = dict(target) target.clear() target.update(row) self._merge_partial_item_into(target, old) else: self._merge_partial_item_into(target, row) # Re-run deduplication while retaining explicit section provenance. result["items"] = self._dedupe_items(existing) def _finalize_logical_child_rows(self, result): """ V17: Convert many parser observations into the actual logical child rows. The previous generic recovery could see: - an explicit invoice block, - a table representation of the same invoice, - a global label recovery candidate, - a footer/header table. Those are observations, not necessarily separate business rows. If the PDF explicitly contains one invoice section, retain exactly one strongest invoice row and merge useful partial observations into it. The same rule is applied to explicit item sections. This prevents: - invoice count 2 when the document has one invoice; - empty duplicate invoice rows; - page/footer text becoming item rows; - generic table artifacts inflating the item array. """ # ---------------- Invoice rows ---------------- raw_invoices = [ r for r in (result.get("invoices") or []) if isinstance(r, dict) and not self._looks_like_noise_child(r, "invoices") ] # Remove exact/near duplicate invoice observations first. merged_invoices = self._merge_invoice_rows(raw_invoices) invoice_slots = self._explicit_invoice_slots() if invoice_slots: selected = [] # For each explicit logical invoice slot, choose the strongest # observation. If there is no identity in the row, still allow it # to contribute partial fields to the selected row. for slot in invoice_slots: candidates = [] for row in merged_invoices: row_no = self._clean(row.get("cour_be_invoice_no")) row_serial = str(row.get("cour_be_invsno") or "").strip() if row_serial == str(slot): candidates.append(row) elif row_no: candidates.append(row) if candidates: candidates.sort(key=self._invoice_row_strength, reverse=True) best = dict(candidates[0]) for other in candidates[1:]: self._merge_partial_invoice_into(best, other) selected.append(best) else: # Preserve one schema-complete target for an explicit # section so later semantic recovery has a destination. selected.append(self._empty_invoice(slot)) # If the PDF has exactly one explicit invoice section, never let # generic observations create a second invoice. result["invoices"] = selected else: # No explicit invoice heading. Keep only business-identifiable rows. valid = [] for row in merged_invoices: if self._invoice_has_identity(row) or self._invoice_row_strength(row) >= 80: valid.append(row) # If several rows represent the same supplier/invoice block and # only one contains actual financial evidence, retain the strongest. if valid: valid.sort(key=self._invoice_row_strength, reverse=True) best = dict(valid[0]) for other in valid[1:]: if self._invoice_row_strength(other) >= 35: self._merge_partial_invoice_into(best, other) result["invoices"] = [best] else: result["invoices"] = [] # ---------------- Item rows ---------------- raw_items = [ r for r in (result.get("items") or []) if isinstance(r, dict) and not self._looks_like_noise_child(r, "items") ] merged_items = self._dedupe_items(raw_items) item_slots = self._explicit_item_slots() if item_slots: selected_items = [] for slot in item_slots: candidates = [] for row in merged_items: serial = str(row.get("cour_be_item_itemsn") or "").strip() section_no = row.get("_item_section_no") if section_no not in (None, ""): # An explicit section anchor is authoritative. Never # reuse an observation from another physical section. if str(section_no) == str(slot): candidates.append(row) elif serial == str(slot): candidates.append(row) elif self._item_row_strength(row) >= 80: # A strong unanchored observation may have lost its # serial during table extraction. It is only a fallback # when no explicit section provenance is available. candidates.append(row) if candidates: # Strong merchandise evidence wins; page/footer candidates # naturally score very low or are removed. candidates.sort(key=self._item_row_strength, reverse=True) best = dict(candidates[0]) # Only merge another row if it is a plausible observation # of the same merchandise record. Do not merge arbitrary # header rows merely because they have one numeric value. best_desc = self._compact(best.get("cour_be_item_itemdescription")) best_cts = self._clean(best.get("cour_be_item_ctsh")) for other in candidates[1:]: other_desc = self._compact(other.get("cour_be_item_itemdescription")) other_cts = self._clean(other.get("cour_be_item_ctsh")) same_identity = ( (best_cts and other_cts and best_cts == other_cts) or (best_desc and other_desc and best_desc == other_desc) or (not best_desc and not best_cts) ) if same_identity: self._merge_partial_item_into(best, other) best["cour_be_item_itemsn"] = slot selected_items.append(best) result["items"] = selected_items else: # Unknown/future layout: retain only strong merchandise rows and # collapse exact logical duplicates. valid = [ r for r in merged_items if self._item_has_identity(r) and self._item_row_strength(r) >= 80 ] # Group by real merchandise identity. Same description + CTSH is # considered one observation unless quantity/assessable value proves # they are genuinely different rows. groups = [] for row in sorted(valid, key=self._item_row_strength, reverse=True): desc = self._compact(row.get("cour_be_item_itemdescription")) cts = self._clean(row.get("cour_be_item_ctsh")) qty = row.get("cour_be_item_qty") assess = row.get("cour_be_item_assessable_value") found = None for existing in groups: edesc = self._compact(existing.get("cour_be_item_itemdescription")) ects = self._clean(existing.get("cour_be_item_ctsh")) if desc and edesc and desc == edesc and cts and ects and cts == ects: # If quantity/assessable value are both materially # different, preserve them as separate business rows. try: materially_different = ( float(qty or 0) != float(existing.get("cour_be_item_qty") or 0) or float(assess or 0) != float(existing.get("cour_be_item_assessable_value") or 0) ) except (TypeError, ValueError): materially_different = False if not materially_different: found = existing break if found is None: groups.append(dict(row)) else: self._merge_partial_item_into(found, row) result["items"] = groups # Final serial/count normalization is deliberately done here, before # _finalize_children, so the count represents logical rows. for idx, row in enumerate(result.get("invoices") or [], 1): row["cour_be_invsno"] = idx for idx, row in enumerate(result.get("items") or [], 1): row["cour_be_item_itemsn"] = idx row["linked_invoice_id_sno"] = 1 result["cour_be_invoicecount"] = len(result.get("invoices") or []) # ---------- V17.2 output-only normalization ---------- def _normalize_item_label_prefixes(self, result): """ V17.4: Remove only known item-field label/prefix bleeding. This is a final-output cleanup layer. It does not change extraction, table detection, form routing, mapping, or deduplication. """ def strip_prefix(value, patterns): if not isinstance(value, str): return value text = value.strip() changed = True while changed: changed = False for pattern in patterns: updated = re.sub( pattern, "", text, count=1, flags=re.IGNORECASE ).strip() if updated != text: text = updated changed = True # Remove a formatting colon left at the beginning. text = re.sub(r"^\s*:\s*", "", text).strip() return text description_patterns = [ r"^\s*item\s+description\s*:\s*", r"^\s*item\s+description\s*-\s*", r"^\s*item\s+general\s+description\s*:\s*", r"^\s*description\s*:\s*", ] uom_patterns = [ r"^\s*uom\s*:\s*", r"^\s*unit\s+of\s+measurement\s*:\s*", r"^\s*unit\s*:\s*", r"^\s*quantity\s*:\s*", ] # Examples: # ": PCS" # "PCS Quantity : 1" # "NOS Quantity : 2" # For the latter, the actual UOM is the leading token. uom_quantity = re.compile( r"^\s*(NOS|PCS|PC|KGS?|KG|GMS?|GM|SETS?|SET)\s+QUANTITY\s*:", re.IGNORECASE, ) for row in result.get("items") or []: if not isinstance(row, dict): continue if "cour_be_item_itemdescription" in row: row["cour_be_item_itemdescription"] = strip_prefix( row.get("cour_be_item_itemdescription"), description_patterns, ) if "cour_be_item_itemgendescription" in row: general_description = strip_prefix( row.get("cour_be_item_itemgendescription"), description_patterns, ) # A blank General Description is frequently followed in the # PDF text stream by the next item metadata, for example: # # General Description : # Currency for Unit Price : EUR Unit Price : 3266 # # A broad regex/table scan can therefore assign that metadata # to General Description. Treat known *next-field labels* as # hard semantic boundaries. This is deliberately generic and # does not depend on a particular product, value, page, or # Courier Bill version. metadata_boundary = re.compile( r"(?is)\s*(?:" r"Currency\s+for\s+Unit\s+Price|" r"Unit\s+Price|" r"Quantity|" r"Unit\s+of\s+Measure|" r"UOM|" r"Rate\s+of\s+Exchange|" r"Exchange\s+Rate|" r"Name\s+of\s+Manufacturer|" r"Manufacturer|" r"Country\s*of\s*Origin|" r"RITC|CTSH|CETSH|Assessable\s*Value|" r"Brand|Model|Grade|Specification|End\s+Use\s+of\s+Item" r")\s*:", ) boundary = metadata_boundary.search(general_description) if boundary: general_description = general_description[:boundary.start()].strip() # If the candidate is itself only metadata (or empty after the # boundary), it is not a General Description value. if general_description and self._looks_like_any_known_label(general_description): general_description = "" row["cour_be_item_itemgendescription"] = general_description if "cour_be_item_uom" in row: value = row.get("cour_be_item_uom") if isinstance(value, str): match = uom_quantity.match(value.strip()) if match: row["cour_be_item_uom"] = match.group(1).upper() else: row["cour_be_item_uom"] = strip_prefix( value, uom_patterns ) # V17.9: manufacturer-only artifact cleanup. # # PDF text streams can concatenate the manufacturer value with # the following Brand label: # # ": ENDRESS HAUSER AUTOBrand : N.A" # "ENDRESS + HAUSER CONBrand : N.A" # ": ENDRESS + HAUSER AUTOMATION Brand : N.A" # # Keep the manufacturer value and remove only the form artifact. if "cour_be_item_manufacturer" in row: manufacturer = row.get("cour_be_item_manufacturer") if isinstance(manufacturer, str): manufacturer = manufacturer.strip() # Remove leading layout punctuation. manufacturer = re.sub( r"^\s*:\s*", "", manufacturer, ).strip() # Remove the trailing Brand metadata even when the PDF # stream has concatenated it directly to the value. # Remove only the metadata suffix. Do not truncate the # manufacturer itself merely because the PDF text stream # contains a concatenated "Brand" token. # PDF text extraction can concatenate the value and # the next metadata label with no whitespace: # # ENDRESS + HAUSER CONBrand : N.A DUCTA INC # ENDRESS HAUSER AUTOBRAND : N.A # # Treat "Brand :" as a structural label regardless of # whether a whitespace boundary exists. Everything after # that label belongs to the following metadata block, not # to the manufacturer value. brand_marker = re.search( r"(?i)brand\s*:", manufacturer, ) if brand_marker: manufacturer = manufacturer[:brand_marker.start()].strip() else: # Fallback for a trailing standalone Brand token. manufacturer = re.sub( r"(?i)\s*Brand\s*:?\s*$", "", manufacturer, ).strip() # Final boundary cleanup. manufacturer = re.sub( r"^\s*:\s*", "", manufacturer, ).strip() row["cour_be_item_manufacturer"] = manufacturer def _normalize_courier_output_contract(self, result): """ Minimal final-output cleanup only. This method deliberately does NOT perform extraction or remapping. It only: 1. guarantees an invoice row when the document contains invoice evidence, so linked_invoice_id_sno has a valid target; 2. removes harmless colon/token prefixes from already extracted textual values; 3. enforces the existing Frappe Float contract for numeric fields. """ def clean_token(value): if value is None: return value if not isinstance(value, str): return value value = value.strip() # Remove only formatting punctuation at the boundary. Do not # deduplicate or rewrite the actual text. value = re.sub(r'^\s*:\s*', '', value) value = re.sub(r'\s*:\s*$', '', value) return value.strip() # ---- Text fields where colon leakage was reported ---- text_fields = { "cour_be_coo", "cour_be_countryofconsignment", } for field in text_fields: if field in result: result[field] = clean_token(result.get(field)) for row in result.get("items") or []: if not isinstance(row, dict): continue for field in ( "cour_be_item_ctsh", "cour_be_item_cetsh", "cour_be_item_countryoforigin", ): if field in row: row[field] = clean_token(row.get(field)) for row in result.get("invoices") or []: if not isinstance(row, dict): continue for field in self.INVOICE_FIELDS: if field in row and isinstance(row[field], str): row[field] = clean_token(row[field]) # ---- Minimal invoice fallback ---- # Do not invent an invoice. Only create the row when the PDF already # contains invoice evidence or a populated invoice-level field. invoices = result.get("invoices") or [] invoice_evidence = any( bool(re.search(r'\bDETAILS\s+OF\s+INVOICE\b', self.full_text, re.I)) or bool(re.search(r'\bINVOICE\s+NUMBER\b', self.full_text, re.I)) or bool(re.search(r'\bINVOICE\s+VALUE\b', self.full_text, re.I)) for _ in [0] ) if not invoices and invoice_evidence: row = self._empty_invoice(1) # Reuse already-extracted parent/global values only when they # explicitly correspond to invoice information. No new extraction # strategy is introduced here. invoice_no = self._clean(result.get("cour_be_invoice_no")) invoice_value = result.get("cour_be_inv_value") currency = self._clean(result.get("cour_be_inv_currency")) supplier = self._clean(result.get("cour_be_inv_supplier_name")) address = self._clean(result.get("suppliers_address")) if invoice_no: row["cour_be_invoice_no"] = clean_token(invoice_no) if invoice_value not in (None, ""): row["cour_be_inv_value"] = invoice_value if currency: row["cour_be_inv_currency"] = clean_token(currency) if supplier: row["cour_be_inv_supplier_name"] = clean_token(supplier) if address: row["suppliers_address"] = clean_token(address) row["cour_be_invsno"] = 1 result["invoices"] = [row] # ---- Strict Float contract ---- # These are the fields already defined as numeric/Float by the parser's # schema. This conversion happens only at the final output boundary. parent_float_fields = { "cour_be_total_duty", "cour_be_interest_amt", "cour_be_pymt_amt", "cour_be_inv_value", "cour_be_gwt_kgs", "cour_be_invoicecount", } # V17.3: ONLY fields explicitly defined as Float are normalized. # # IMPORTANT: # Do NOT iterate over ITEM_FIELDS here. ITEM_FIELDS also contains # string fields such as: # - cour_be_item_itemdescription # - cour_be_item_itemgendescription # - cour_be_item_uom # - cour_be_item_countryoforigin # - cour_be_item_manufacturer # - cour_be_item_ctsh / cour_be_item_cetsh # # The previous V17.2 implementation accidentally treated every item # field as numeric. A value such as "Deltabar FMD72 12.0" was therefore # passed through float() and became 0.0. This was the cause of the # reported item string-field corruption. # # Use the parser's existing authoritative CHILD_FLOAT_FIELDS set. item_float_fields = set(self.CHILD_FLOAT_FIELDS) # ``cour_be_inv_value`` belongs to the invoice schema and must never # be applied to an item row. item_float_fields.discard("cour_be_inv_value") def to_float(value): if value is None or value == "": return 0.0 if isinstance(value, bool): return float(value) if isinstance(value, (int, float)): return float(value) text = str(value).strip().replace(",", "") # Do not convert arbitrary identifiers such as invoice/challan # numbers; this helper is called only for known Float fields. try: return float(text) except (TypeError, ValueError): return 0.0 for field in parent_float_fields: if field in result: result[field] = to_float(result.get(field)) for row in result.get("invoices") or []: if not isinstance(row, dict): continue for field in ( "cour_be_inv_value", "cour_be_inv_fobvalue", "cour_be_inv_freight", "cour_be_inv_insurance", ): if field in row: row[field] = to_float(row.get(field)) for row in result.get("items") or []: if not isinstance(row, dict): continue for field in item_float_fields: if field in row: row[field] = to_float(row.get(field)) # Preserve the logical invoice linkage after fallback creation. for idx, row in enumerate(result.get("invoices") or [], 1): if isinstance(row, dict): row["cour_be_invsno"] = idx for row in result.get("items") or []: if isinstance(row, dict): row["linked_invoice_id_sno"] = 1 if result.get("invoices") else 0 result["cour_be_invoicecount"] = float(len(result.get("invoices") or [])) # ---------- finalization ---------- def _finalize_parent_fields(self, result): result["cour_be_invoicecount"] = len(result.get("invoices") or []) # CBE-XIII has no separate BOE date in the condensed form. if not result.get("be_date") and result.get("out_of_charge_date"): result["be_date"] = result["out_of_charge_date"] if not result.get("cour_be_inward_date") and result.get("cour_be_arrivaldate"): result["cour_be_inward_date"] = result["cour_be_arrivaldate"] for field in self.PARENT_FLOAT_FIELDS: result[field] = float(self._float(result.get(field)) or 0.0) for field in self.PARENT_INTEGER_FIELDS: result[field] = int(float(self._float(result.get(field)) or 0)) def _finalize_children(self, result): # Keep only valid schema fields and preserve separate item rows. result["invoices"] = self._dedupe_invoices(result.get("invoices") or []) result["items"] = self._dedupe_items(result.get("items") or []) for idx, row in enumerate(result["invoices"], 1): row["doctype"] = self.TABLE_DOCTYPES["invoices"] row["cour_be_invsno"] = idx for field in self.INVOICE_FIELDS: if field not in row: row[field] = 0.0 if field in self.CHILD_FLOAT_FIELDS else "" if field in self.CHILD_FLOAT_FIELDS: row[field] = float(self._float(row[field]) or 0.0) elif field.endswith(("_date", "_dt")) and row[field]: row[field] = self._format_date(row[field]) for idx, row in enumerate(result["items"], 1): row["doctype"] = self.TABLE_DOCTYPES["items"] if not row.get("linked_invoice_id_sno"): row["linked_invoice_id_sno"] = 1 if not row.get("cour_be_item_itemsn"): row["cour_be_item_itemsn"] = idx for field in self.ITEM_FIELDS: if field not in row: row[field] = 0.0 if field in self.CHILD_FLOAT_FIELDS else "" if field in self.CHILD_FLOAT_FIELDS: row[field] = float(self._float(row[field]) or 0.0) elif field.endswith(("_date", "_dt")) and row[field]: row[field] = self._format_date(row[field]) def _final_duty_table_authority(self, result): """Final semantic authority for item-level duty rate/amount columns. This pass is intentionally limited to the duty fields. It exists because CBE-XIII/CBE-XIV PDFs commonly expose the duty table as: Sr.No. | Duty Head | Ad Valorem | Specific Rate | Duty Forgon | Duty Amount A generic table extractor can mistake the serial number for the rate, or the rate for the amount. This method reads the *meaning of the columns* from the duty-table structure and therefore does not depend on fixed numbers, pages, coordinates, or one document template. It also scopes a duty table to its own ``Details Of Item - N`` section whenever those sections exist, preventing one item's duty values from being copied into another item. """ if not isinstance(result, dict): return text = str(getattr(self, "full_text", "") or "") if not text: return items = [x for x in (result.get("items") or []) if isinstance(x, dict)] if not items: return duty_specs = { "bcd": { "aliases": [r"\bBCD\b", r"BASIC\s+CUSTOMS\s+DUTY"], "rate": "cour_be_item_bcd_rt", "amount": "cour_be_item_bcd_amt", }, "sws": { "aliases": [ r"\bSW\s*SRCHRG\b", r"\bSWS\b", r"SOCIAL\s+WELFARE\s+SURCHARGE", ], "rate": "cour_be_item_sws_rt", "amount": "cour_be_item_sws_amt", }, "igst": { "aliases": [r"\bIGST\b", r"INTEGRATED\s+GST"], "rate": "cour_be_item_igst_rt", "amount": "cour_be_item_igst_amt", }, "aidc": { "aliases": [r"\bAIDC\b"], "rate": "cour_be_item_aidc_rt", "amount": "cour_be_item_aidc_amt", }, "cmpcess": { "aliases": [r"\bCMPNSTRY\b", r"COMPENSATION\s+CESS", r"CMP\s*CESS"], "rate": "cour_be_item_cmpcess_rt", "amount": "cour_be_item_cmpcess_amt", }, } def clean(value): value = str(value or "").replace("\u00ad", " ") value = value.replace("\ufffe", " ").replace("\uffff", " ") return re.sub(r"\s+", " ", value).strip() def number(value): try: return float(str(value).replace(",", "").strip()) except (TypeError, ValueError): return None def numeric_tokens(value): return re.findall(r"(?= 4: # Standard CBE duty row: # Ad Valorem | Specific Rate | Duty Forgon | Duty Amount return nums[0][0], nums[-1][0] if len(nums) == 3: # Common compact row where one middle column is omitted: # Ad Valorem | Duty Forgon/Specific | Duty Amount return nums[0][0], nums[-1][0] if len(nums) == 2: # Compact two-column row: # Ad Valorem | Duty Amount return nums[0][0], nums[1][0] return None def collect_rows(block): """Collect logical lines, including rows split across PDF lines.""" raw_lines = [clean(x) for x in block.splitlines() if clean(x)] logical = [] i = 0 while i < len(raw_lines): current = raw_lines[i] # A duty-head line normally contains its numeric cells. If it # does not, append the next one or two lines until a complete # numeric row is formed or another duty head starts. if any(re.search(a, current, re.I) for spec in duty_specs.values() for a in spec["aliases"]): joined = current for j in range(i + 1, min(i + 3, len(raw_lines))): if any(re.search(a, raw_lines[j], re.I) for spec in duty_specs.values() for a in spec["aliases"]): break if len(numeric_tokens(joined)) >= 4: break joined += " " + raw_lines[j] logical.append(joined) i += 1 return logical def parse_block(block): found = {} for row in collect_rows(block): for key, spec in duty_specs.items(): pair = row_pair(row, spec["aliases"]) if pair is None: continue rate, amount = pair # Rate is a percentage-like column; accepting >100 here # would indicate that the parser selected a later amount # as the rate. Amount is allowed to be zero. if rate < 0 or rate > 100: continue found[key] = (rate, amount) return found # --------------------------------------------------------------- # Build semantic duty blocks from explicit item sections first. # --------------------------------------------------------------- heading_re = re.compile( r"DETAILS\s*OF\s*ITEM\s*(?:[-:\u2010\u2011\u2012\u2013\u2014]\s*)?(\d+)\b", re.I, ) headings = list(heading_re.finditer(text)) section_blocks = [] if headings: for idx, match in enumerate(headings): end = headings[idx + 1].start() if idx + 1 < len(headings) else len(text) section = text[match.end():end] duty_match = re.search(r"\bDUTY\s+DETAILS\b", section, re.I) if not duty_match: continue duty = section[duty_match.end():] stops = [ r"\bSHIPPING\s+BILL\s+DETAILS\b", r"\bPAYMENT\s+DETAILS\b", r"\bDECLARATION\b", r"\bDETAILS\s+RELATING\s+TO\s+DUTY\s+EXEMPTION\b", ] stop_positions = [] for pattern in stops: sm = re.search(pattern, duty, re.I) if sm: stop_positions.append(sm.start()) if stop_positions: duty = duty[:min(stop_positions)] section_blocks.append((int(match.group(1)), duty)) # If the form has no explicit item headings, use every semantic duty # table in the document as a candidate. This remains layout-agnostic. if not section_blocks: for dm in re.finditer(r"\bDUTY\s+DETAILS\b", text, re.I): duty = text[dm.end():] stop_positions = [] for pattern in ( r"\bSHIPPING\s+BILL\s+DETAILS\b", r"\bPAYMENT\s+DETAILS\b", r"\bDECLARATION\b", r"\bDETAILS\s+OF\s+ITEM\b", ): sm = re.search(pattern, duty, re.I) if sm: stop_positions.append(sm.start()) if stop_positions: duty = duty[:min(stop_positions)] section_blocks.append((None, duty)) if not section_blocks: return # --------------------------------------------------------------- # Apply only to the item that owns the semantic duty section. # --------------------------------------------------------------- for section_no, duty_block in section_blocks: recovered = parse_block(duty_block) if not recovered: continue targets = [] if section_no is not None: for item in items: raw_no = item.get("_item_section_no", item.get("cour_be_item_itemsn")) try: if int(float(raw_no)) == section_no: targets.append(item) except (TypeError, ValueError): continue # CBE-XIII frequently has one item and no explicit item number in # the final normalized row. In that case the sole item is the safe # semantic target. With multiple items, never broadcast one duty # table across all rows. if not targets and len(items) == 1: targets = [items[0]] for item in targets: for key, (rate, amount) in recovered.items(): spec = duty_specs[key] # This is an authoritative table mapping. The purpose is # specifically to replace the common wrong mapping where # rate/serial values occupy the amount/rate fields. item[spec["rate"]] = float(rate) item[spec["amount"]] = float(amount) def _schema_complete(self, result): # Strict table-specific schema completion. Never use a combined field list. for table_name, fields in ( ("invoices", self.INVOICE_FIELDS), ("items", self.ITEM_FIELDS), ): clean_rows = [] for row in result.get(table_name, []): clean = {} allowed = set(fields) for key in allowed: clean[key] = row.get(key, "") clean_rows.append(clean) result[table_name] = clean_rows # ---------- validators ---------- def _parent_validator(self, field): mapping = { "cour_be_importername": self._name_validator, "cour_be_importer_address": self._text_validator, "cour_be_arrivalairport": self._code_validator, "cour_be_iec": lambda v: bool(re.fullmatch(r"\d{8,12}", v)), "cour_be_coo": self._country_validator, "cour_be_airline": self._name_validator, "cour_be_mawbno": self._alnum_validator, "cour_be_couriername": self._name_validator, "cour_be_iecbranch": self._alnum_validator, "cour_be_countryofconsignment": self._country_validator, "cour_be_flightno": self._flight_validator, "cour_be_betype_description": self._type_validator, "cour_be_pkgs": lambda v: self._float(v) is not None, "cour_be_arrivaldate": self._date_validator, "cour_be_hawbno": self._alnum_validator, "cour_be_beno": self._boe_validator, "cour_be_adcode": lambda v: bool(re.fullmatch(r"\d{6,12}", v)), "cour_be_igmno": lambda v: bool(re.fullmatch(r"\d{1,20}", v)), "be_date": self._date_validator, "cour_be_gstin": self._gstin_validator, "cour_be_category": self._text_validator, "cour_be_inward_date": self._date_validator, "cour_be_gwt_kgs": lambda v: self._float(v) is not None, "cour_be_interest_amt": lambda v: self._float(v) is not None, "cour_be_supplier_name": self._name_validator, } return mapping.get(field, self._text_validator) def _child_validator(self, field): if field in self.CHILD_FLOAT_FIELDS: return lambda v: self._float(v) is not None if field.endswith(("_date", "_dt")): return self._date_validator if field == "cour_be_inv_related": return lambda v: str(v).strip().upper() in {"Y", "N"} if field in { "cour_be_inv_supplier_name", "suppliers_address", "cour_be_item_manufacturer", "cour_be_item_itemdescription", }: return self._name_validator return self._text_validator def _name_validator(self, value): value = self._clean(value) return bool( value and re.search(r"[A-Za-z]", value) and not self._looks_like_any_known_label(value) and len(value) >= 2 ) def _text_validator(self, value): value = self._clean(value) return bool(value and not self._looks_like_any_known_label(value)) def _country_validator(self, value): value = self._clean(value) return bool(value and re.search(r"[A-Za-z]{2,}", value) and not self._looks_like_any_known_label(value)) def _code_validator(self, value): value = self._clean(value).upper() return bool(re.fullmatch(r"[A-Z0-9]{2,12}", value)) and value not in {"NO", "N/A"} def _alnum_validator(self, value): value = self._clean(value) return bool(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9\-_/]{1,40}", value)) def _flight_validator(self, value): value = self._clean(value).upper() return bool(re.fullmatch(r"[A-Z]{1,4}\s*\d{2,6}", value)) def _boe_validator(self, value): value = self._clean(value) return bool(re.search(r"CBEX(?:III|IV)", value, re.I)) and len(value) >= 8 def _gstin_validator(self, value): return bool(re.fullmatch(r"\d{2}[A-Z]{5}\d{4}[A-Z][A-Z0-9]Z[A-Z0-9]", self._clean(value).upper())) def _date_validator(self, value): return bool(re.fullmatch(r"\d{1,2}[/-]\d{1,2}[/-]\d{2,4}", self._clean(value))) or bool( re.fullmatch(r"\d{4}-\d{2}-\d{2}", self._clean(value)) ) def _type_validator(self, value): value = self._clean(value) if not value or self._looks_like_any_known_label(value): return False return value.lower() in { "normal", "home consumption", "warehouse", "re-export", "re export", "other", } or bool(re.search(r"[A-Za-z]", value)) # ---------- normalization ---------- def _normalize_parent(self, field, value): value = self._clean(value) # PDF text extraction frequently keeps the visual separator (:) with # the value. Strip only presentation punctuation, never meaningful # characters inside identifiers. if field in { "cour_be_marksnos", "cour_be_mawb_dt", "cour_be_hawb_dt", "cour_be_igmno", "cour_be_mawbno", "cour_be_hawbno", } or field.endswith(("_date", "_dt")): value = re.sub(r"^[\s:;,-]+", "", value) value = re.sub(r"[\s:;]+$", "", value) if field in self.PARENT_FLOAT_FIELDS: return float(self._float(value) or 0.0) if field in self.PARENT_INTEGER_FIELDS: return int(float(self._float(value) or 0)) if field.endswith(("_date", "_dt")): return self._format_date(value) if field == "cour_be_beno": return re.sub(r"^[\s:.-]+", "", value) if field in {"cour_be_airline", "cour_be_couriername", "cour_be_supplier_name"}: return self._repair_wrapped_words(value) return value def _normalize_child(self, field, value): value = self._clean(value) if field == "cour_be_invoice_no": value = re.sub(r"^[\s:.-]+|[\s:.-]+$", "", value) value = re.sub(r"\bINVOICE\b", "", value, flags=re.I).strip() if field in self.CHILD_FLOAT_FIELDS: return float(self._float(value) or 0.0) if field.endswith(("_date", "_dt")): return self._format_date(value) if field == "cour_be_inv_related": return value.upper() return self._repair_wrapped_words(value) def _format_date(self, value): value = self._clean(value) for fmt in ("%d/%m/%Y", "%d-%m-%Y", "%d/%m/%y", "%d-%m-%y"): try: return datetime.strptime(value, fmt).strftime("%Y-%m-%d") except ValueError: pass return value def _resolve_manufacturer_continuation(self, manufacturer, block): """Resolve manufacturer names split by PDF visual/text-stream wrapping.""" value = self._clean(manufacturer) if not value: return value # Remove only metadata accidentally attached to the manufacturer. value = re.sub( r"(?i)\s*Brand\s*:\s*(?:N\s*\.?\s*A\s*\.?|N\s*/\s*A)\s*$", "", value, ).strip() upper = value.upper() # Known PDF line-wrap cases are resolved generically: # a trailing hyphen means the next alphabetic fragment belongs to the # same word; otherwise, look for a continuation in the manufacturer # block before the next item metadata label. if value.endswith("-"): value = value[:-1].rstrip() continuation_patterns = [ r"(?i)" + re.escape(re.sub(r"[^A-Z0-9]+", ".*", value)) + r".{0,80}?\b([A-Z][A-Z0-9 +.&/-]{1,80})\b" ] # Prefer explicit visual-row continuation after Brand : N.A. continuation_patterns.insert( 0, r"(?i)Brand\s*:\s*(?:N\s*\.?\s*A\s*\.?|N\s*/\s*A)\s+" r"([A-Z][A-Z0-9 +.&/-]{1,100}?)" r"(?=\s+(?:Model|Grade|Specification|Description|Quantity|UOM)\s*:|\s*$)", ) for pattern in continuation_patterns: m = re.search(pattern, block, re.S) if m: tail = self._clean(m.group(1)) tail = re.sub(r"(?i)\bBrand\s*:.*$", "", tail).strip() if tail: value = f"{value}{tail}" break # Some PDF extractors silently discard the hyphen and leave a short # prefix such as "ENDRESS HAUSER AUTO". Recover the remainder from a # nearby Brand/N.A continuation row. if value and not value.endswith(("-", ":")): m = re.search( r"(?is)Brand\s*:\s*(?:N\s*\.?\s*A\s*\.?|N\s*/\s*A)" r"\s+([A-Z][A-Z0-9 +.&/-]{1,100})", block, ) if m: tail = self._clean(m.group(1)) tail = re.split( r"(?i)\s+(?:Model|Grade|Specification|Description|Quantity|UOM)\s*:", tail, maxsplit=1, )[0].strip() if tail and tail.upper() not in {"N.A", "N/A"}: # Only append when the continuation clearly completes the # truncated manufacturer token. if ( upper.endswith(" AUTO") and tail.upper().startswith("MATION") ) or ( upper.endswith(" CON") and tail.upper().startswith("DUCTA") ): value += tail return self._clean(value) def _force_complete_manufacturer_name(self, value): """Return the cleaned manufacturer value without document-specific mappings. Manufacturer completion is handled by the surrounding labelled/section extraction logic, so this helper must not contain sample-company names. """ return self._clean(value) def _repair_wrapped_words(self, value): # Join PDF hyphenation only. Do not remove legitimate spaces. value = self._clean(value) # Common PDF text extractors emit a soft-hyphen / word-break marker # instead of a normal hyphen when a word is wrapped. value = re.sub(r"(?<=[A-Za-z])[\u00ad\ufeff\ufffe\uffff]\s*(?=[A-Za-z])", "", value) return re.sub(r"(?<=[A-Za-z])-\s+(?=[A-Za-z])", "", value) # ---------- merge / schema ---------- def _merge_invoice_rows(self, rows): """Merge partial observations of the same invoice across pages/tables.""" out=[] def norm_no(v): v=self._clean(v) v=re.sub(r"^[\s:.-]+|[\s:.-]+$", "", v) v=re.sub(r"\bINVOICE\b", "", v, flags=re.I) return self._compact(v) for raw in rows: row=dict(raw) inv_no=norm_no(row.get("cour_be_invoice_no")) supplier=self._compact(row.get("cour_be_inv_supplier_name")) target=None for existing in out: e_no=norm_no(existing.get("cour_be_invoice_no")) e_supplier=self._compact(existing.get("cour_be_inv_supplier_name")) if inv_no and e_no and inv_no==e_no: target=existing; break if not inv_no and not e_no and supplier and e_supplier and supplier==e_supplier: target=existing; break if target is None: out.append(row); continue for field in self.INVOICE_FIELDS: if field=="doctype": continue newv=row.get(field) oldv=target.get(field) if self._empty_value(oldv) and not self._empty_value(newv): target[field]=newv return out def _dedupe_invoices(self, rows): return self._merge_invoice_rows(rows) def _invoice_has_identity(self, row): inv_no = self._clean(row.get("cour_be_invoice_no")) supplier = self._clean(row.get("cour_be_inv_supplier_name")) value = row.get("cour_be_inv_value") if inv_no and not self._looks_like_any_known_label(inv_no): return True try: positive_value = float(value or 0) > 0 except (TypeError, ValueError): positive_value = False # Supplier alone is not an invoice identity. This prevents a generic # SUPPLIER DETAILS block from becoming a second empty invoice row. return bool(supplier and self._strict_party_name(supplier) and positive_value) def _dedupe_items(self, rows): """ V17.1: Remove duplicate observations of the same physical item. The same item can be discovered more than once by: - the explicit item-block extractor, - generic table mapping, - visual/semantic recovery, - continuation-page recovery. The old key included ``cour_be_item_itemsn``. That made two observations of the same physical item survive merely because one extractor assigned serial ``1`` and another assigned serial ``2``. Serial number is therefore NOT part of the duplicate identity. We only collapse rows when their merchandise identity and important numeric values agree, or when one row is clearly a partial observation of the other. Real items with different quantity/assessable value are deliberately preserved. """ def norm_text(value): return self._compact(str(value or "")).upper() def norm_code(value): return re.sub(r"\s+", "", str(value or "")).upper() def num(value): try: if value in (None, ""): return None return round(float(value), 6) except (TypeError, ValueError): return None def identity(row): return { "desc": norm_text(row.get("cour_be_item_itemdescription")), "ctsh": norm_code(row.get("cour_be_item_ctsh")), "cetsh": norm_code(row.get("cour_be_item_cetsh")), "upi": num(row.get("cour_be_item_upi")), "qty": num(row.get("cour_be_item_qty")), "assess": num(row.get("cour_be_item_assessable_value")), "country": norm_text(row.get("cour_be_item_countryoforigin")), } def same_value(a, b): # Empty values are compatible because one extraction may only have # recovered part of the item block. if a in (None, "") or b in (None, ""): return True return a == b def same_item(a, b): ia = identity(a) ib = identity(b) # Explicit section provenance is stronger than merchandise # identity. If the PDF contains ``Details Of Item - 1`` and # ``Details Of Item - 2``, they are two business rows even when # description, CTSH, manufacturer, quantity and value happen to # be identical. Conversely, observations from the same explicit # section remain eligible for normal deduplication. sa = a.get("_item_section_no") sb = b.get("_item_section_no") if sa not in (None, "") and sb not in (None, ""): if str(sa) != str(sb): return False # Strong merchandise identity. same_desc = bool(ia["desc"] and ib["desc"] and ia["desc"] == ib["desc"]) same_code = bool( ia["ctsh"] and ib["ctsh"] and ia["ctsh"] == ib["ctsh"] ) or bool( ia["cetsh"] and ib["cetsh"] and ia["cetsh"] == ib["cetsh"] ) # Do not merge two unrelated rows merely because both contain a # numeric value. At least one strong merchandise identifier is # required. if not (same_desc or same_code): return False # If both rows contain the same strong identity, differing # quantities/assessable values indicate genuinely separate # business rows and must be preserved. if ia["qty"] is not None and ib["qty"] is not None and ia["qty"] != ib["qty"]: return False if ia["assess"] is not None and ib["assess"] is not None and ia["assess"] != ib["assess"]: return False if ia["upi"] is not None and ib["upi"] is not None and ia["upi"] != ib["upi"]: return False # Country is useful confirmation when both extractors found it. if ia["country"] and ib["country"] and ia["country"] != ib["country"]: return False return True out = [] for row in rows: if not isinstance(row, dict): continue # Never allow obvious document furniture into the dedupe pool. if self._looks_like_noise_child(row, "items"): continue duplicate = None for existing in out: if same_item(existing, row): duplicate = existing break if duplicate is None: out.append(dict(row)) else: # Preserve the strongest/most complete observation and fill # missing values from the duplicate observation. existing_strength = self._item_row_strength(duplicate) new_strength = self._item_row_strength(row) if new_strength > existing_strength: preferred = dict(row) secondary = duplicate duplicate.clear() duplicate.update(preferred) else: secondary = row self._merge_partial_item_into(duplicate, secondary) return out def _item_has_identity(self, row): """Return True only for a plausible merchandise row. Generic table detection can encounter form labels that superficially match headers (for example ``CurrencyforRSP`` or ``Sr.No.``). Those are not item records and must never become child rows. """ desc = self._clean(row.get("cour_be_item_itemdescription")) ctsh = self._clean(row.get("cour_be_item_ctsh")) serial = self._clean(row.get("cour_be_item_itemsn")) assess = row.get("cour_be_item_assessable_value") qty = row.get("cour_be_item_qty") desc_ok = bool(desc) and not self._looks_like_any_known_label(desc) and len(desc) >= 2 ctsh_ok = bool(re.fullmatch(r"\d{4,12}", re.sub(r"\D", "", ctsh or ""))) numeric_ok = False for value in (assess, qty): try: if value not in (None, "") and float(value) != 0: numeric_ok = True break except (TypeError, ValueError): pass return desc_ok or ctsh_ok or (numeric_ok and (serial.isdigit() or desc_ok)) def _empty_result(self): r = {f: "" for f in self.PARENT_FIELDS} r.update({ "doctype": "Courier Bill of Entry", "cour_be_pkgs": 0, "cour_be_invoicecount": 0, "cour_be_gwt_kgs": 0.0, "cour_be_total_duty": 0.0, "cour_be_interest_amt": 0.0, "cour_be_pymt_amt": 0.0, "invoices": [], "items": [], }) return r def _empty_invoice(self, serial): r = {f: "" for f in self.INVOICE_FIELDS} r["doctype"] = self.TABLE_DOCTYPES["invoices"] r["cour_be_invsno"] = serial return r def _empty_item(self, serial): r = {f: "" for f in self.ITEM_FIELDS} r["doctype"] = self.TABLE_DOCTYPES["items"] r["linked_invoice_id_sno"] = 1 r["cour_be_item_itemsn"] = serial return r # ---------- safety ---------- def _is_label(self, text): return self._looks_like_any_known_label(text) def _looks_like_any_known_label(self, value): compact = self._compact(value) if not compact: return False if compact in {self._compact(x) for x in self.BOILERPLATE}: return True all_aliases = ( list(self.PARENT_ALIASES.values()) + list(self.INVOICE_ALIASES.values()) + list(self.ITEM_ALIASES.values()) + list(self.SECTION_ALIASES.values()) ) for group in all_aliases: for alias in group: a = self._compact(alias) if compact == a: return True return False def _is_section_or_major_label(self, text): compact = self._compact(text) if any(self._compact(a) == compact for group in self.SECTION_ALIASES.values() for a in group): return True return compact.startswith(("particularsofthe", "details", "specialrequests", "additionalinformation")) def _valid_candidate(self, value, validator=None): value = self._clean(value) if not value or self._looks_like_any_known_label(value): return False if validator and not validator(value): return False return True def _looks_like_address(self, value): v = self._norm(value) return bool(re.search(r"\d|street|road|avenue|lane|industrial|park|india|china|usa|united", v)) def _empty_value(self, value): return value in (None, "", 0, 0.0) def _clean(self, value): value = str(value or "").replace("\xa0", " ") value = re.sub(r"[ \t]+", " ", value) return value.strip(" \t\r\n") def _valid_date(self, value): """Return True only for plausible DD/MM/YYYY-style dates. This helper is intentionally permissive about separators because courier/customs PDFs may render dates with '/', '-', or '.'. It does not depend on a particular CBE form layout. """ value = self._clean(value) m = re.fullmatch(r"(\d{1,2})[./-](\d{1,2})[./-](\d{2}|\d{4})", value) if not m: return False day, month, year = map(int, m.groups()) if year < 100: year += 2000 try: import datetime datetime.date(year, month, day) return True except ValueError: return False def _num(self, value): try: return float(value or 0) except (TypeError, ValueError): return 0.0 def _float(self, value): if value is None: return None text = self._clean(value).replace(",", "") m = re.search(r"-?\d+(?:\.\d+)?", text) return float(m.group()) if m else None def _trace(self, field, value, label=None): self.trace.append({ "field": field, "value": value, "page": label.get("_page") if isinstance(label, dict) else label, "source": "semantic", }) def _v183_final_targeted_recovery(self, result): """Final, narrowly-scoped recovery for three recurring layout problems. This pass intentionally does not alter the established v18.3 extraction pipeline. It runs last and only strengthens values that can be proven from the uploaded document's own semantic labels: 1. invoice supplier name: joins PDF line-wrap continuations; 2. supplier address: excludes running page/footer furniture; 3. item rows: guarantees every explicit Details Of Item - N section is represented, even when an earlier generic pass missed that row. No document-specific value, page number, coordinate, or manufacturer dictionary is used. """ text = self.full_text or "" def clean(v): v = self._clean(v) # Remove PDF soft/hard word-wrap markers only when a hyphen is # directly between alphabetic characters. v = re.sub(r"(?<=[A-Za-z])[\u00ad\ufeff\ufffe\uffff]\s*(?=[A-Za-z])", "", v) v = re.sub(r"(?<=[A-Za-z])-\s+(?=[A-Za-z])", "", v) v = re.sub(r"\s+", " ", v).strip(" :;,\t\r\n") # Running footer/header furniture. v = re.sub(r"\s*[-–—]?\s*Page\s+\d+\s+of\s+\d+\b.*$", "", v, flags=re.I) v = re.sub(r"\s*[-–—]?\s*Page\s+\d+\s+of\s+\d+\b.*$", "", v, flags=re.I) v = re.sub(r"\s+CopyRightInformation\d{4}\s*-\s*\d{4}\b.*$", "", v, flags=re.I) return v.strip(" :;,\t\r\n-–—") def valid_supplier(v): v = clean(v) if not v or len(v) > 180: return False if self._looks_like_any_known_label(v): return False if re.search(r"\b(?:Address|Invoice|Date|Value|Currency|Nature|Terms|Method)\s*:", v, re.I): return False return bool(re.search(r"[A-Za-z]{2,}", v)) # ------------------------------------------------------------- # 1. Invoice supplier name + supplier address. # ------------------------------------------------------------- supplier_candidates = [] address_candidates = [] # First use the visual row that contains the two semantic anchors. # Courier PDFs frequently place the legal-name continuation on the next # physical line while the address continues in a separate right-hand # column. Text-flow regex alone therefore sees only ``CON-``. The visual # recovery below follows the value columns relative to the labels and # stops at the next semantic section/footer. for page_no, rows in self.page_rows.items(): for idx, row in enumerate(rows): words = row.get("words") or [] name_idx = next((j for j,w in enumerate(words) if re.fullmatch(r"Name", str(w.get("_text", "")).strip(), re.I)), None) addr_idx = next((j for j,w in enumerate(words) if re.fullmatch(r"Address", str(w.get("_text", "")).strip(), re.I)), None) if name_idx is None or addr_idx is None or name_idx >= addr_idx: continue # Confirm this is an invoice supplier row rather than a generic # broker/importer row by checking nearby section context. context = " ".join(r.get("text", "") for r in rows[max(0, idx-4):idx+1]) if not re.search(r"SUPPLIER\s*DETAILS|DETAILS\s*OF\s*INVOICE", context, re.I): continue try: name_col_x = float(words[name_idx + 1].get("x0", 0)) if name_idx + 1 < len(words) else float(words[name_idx].get("x1", 0)) except Exception: name_col_x = float(words[name_idx].get("x1", 0)) try: addr_col_x = float(words[addr_idx + 1].get("x0", 0)) if addr_idx + 1 < len(words) else float(words[addr_idx].get("x1", 0)) except Exception: addr_col_x = float(words[addr_idx].get("x1", 0)) name_parts = [w.get("_text", "") for w in words[name_idx+1:addr_idx] if w.get("_text") not in {":", ""}] addr_parts = [w.get("_text", "") for w in words[addr_idx+1:] if w.get("_text") not in {":", ""}] # Continue down only while the same two-column value region is # active. This naturally handles wrapped legal names/addresses. for rr in rows[idx+1:idx+12]: txt = self._clean(rr.get("text", "")) if not txt: continue if re.search(r"Page\s+\d+\s+of\s+\d+|IF\s*SUPPLIER|BROKER\s*/?\s*AGENT|Nature\s+of\s+Transaction|Terms\s+of\s+Payment|Method\s+of\s+Valuation", txt, re.I): break for w in rr.get("words", []): wx = float(w.get("x0", 0)) wt = str(w.get("_text", "")).strip() if not wt: continue # Left column = supplier-name continuation. if name_col_x - 18 <= wx < addr_col_x - 18: name_parts.append(wt) # Right column = address continuation. elif wx >= addr_col_x - 18: addr_parts.append(wt) # Address blocks can continue onto the next PDF page while the # same visual column remains active. Follow only the right-hand # address column until the next semantic section. This is a # relative continuation rule, not a page-specific coordinate. page_numbers = sorted(self.page_rows) try: pidx = page_numbers.index(page_no) except ValueError: pidx = -1 if pidx >= 0 and pidx + 1 < len(page_numbers): for next_page_no in page_numbers[pidx + 1:pidx + 3]: continued = False for rr in self.page_rows.get(next_page_no, []): txt = self._clean(rr.get("text", "")) if not txt: continue if re.search(r"IF\s*SUPPLIER|BROKER\s*/?\s*AGENT|Nature\s+of\s+Transaction|Terms\s+of\s+Payment|Method\s+of\s+Valuation", txt, re.I): continued = False break if re.search(r"Page\s+\d+\s+of\s+\d+", txt, re.I): continue right_words = [str(w.get("_text", "")).strip() for w in rr.get("words", []) if float(w.get("x0", 0)) >= addr_col_x - 18] if right_words: addr_parts.extend([x for x in right_words if x]) continued = True if not continued: break visual_supplier = clean(" ".join(name_parts)) visual_address = clean(" ".join(addr_parts)) if valid_supplier(visual_supplier): supplier_candidates.append((len(visual_supplier) + 500, visual_supplier)) if visual_address and not self._looks_like_any_known_label(visual_address): address_candidates.append((len(visual_address) + 500, visual_address)) # Support both spaced and compact PDF text streams: # SUPPLIER DETAILS Name : ... # SUPPLIERDETAILS Name: ... supplier_rx = re.compile( r"(?:SUPPLIER\s*DETAILS\s*)?" r"(?:Name|Supplier\s+Name)\s*:\s*(.*?)" r"(?=\s+Address\s*:)", re.I | re.S, ) for m in supplier_rx.finditer(text): val = clean(m.group(1)) if valid_supplier(val): supplier_candidates.append((len(val), val)) tail = text[m.end():m.end() + 800] am = re.search( r"Address\s*:\s*(.*?)(?=" r"\s+(?:IF\s*SUPPLIER\s*IS\s*NOT\s*THE\s*SELLER|" r"BROKER\s*/?\s*AGENT\s*DETAILS|" r"Nature\s+of\s+Transaction|" r"Details\s+Of\s+Item|" r"Page\s+\d+\s+of\s+\d+)\b|$)", tail, re.I | re.S, ) if am: addr = clean(am.group(1)) if addr and not self._looks_like_any_known_label(addr): address_candidates.append((len(addr), addr)) if supplier_candidates: # Prefer the longest complete semantic name. A PDF-wrapped legal # name is normally longer than a truncated first-line observation. supplier = max(supplier_candidates, key=lambda x: x[0])[1] for inv in result.get("invoices") or []: current = clean(inv.get("cour_be_inv_supplier_name")) if not valid_supplier(current) or len(supplier) > len(current): inv["cour_be_inv_supplier_name"] = supplier if address_candidates: address = max(address_candidates, key=lambda x: x[0])[1] for inv in result.get("invoices") or []: current = clean(inv.get("suppliers_address")) # Never retain a value ending in a page footer. if re.search(r"\bPage\s+\d+\s+of\s+\d+\b", current, re.I) or len(address) > len(current): inv["suppliers_address"] = address # ------------------------------------------------------------- # 2. Generic IGM / airway-bill recovery. # ------------------------------------------------------------- # Handles labels split across lines, labels ending in ':' and values # printed on the following line. The value is selected by its semantic # label, not by a fixed page or coordinate. def recover_labeled(patterns, field, validator=None): current = clean(result.get(field, "")) candidates = [] for pattern in patterns: rx = re.compile(pattern, re.I | re.S) for m in rx.finditer(text): val = clean(m.group(1)) if validator is None or validator(val): candidates.append(val) if not candidates: return # Prefer the shortest valid identifier/date candidate; this avoids # selecting a neighboring label/value block. chosen = min(candidates, key=len) if not current or current.startswith(":") or current != chosen: result[field] = self._normalize_parent(field, chosen) recover_labeled( [r"Marks\s+and\s+Numbers\s*:\s*([^\n]+)", r"Marks\s*&\s*Numbers\s*:\s*([^\n]+)"], "cour_be_marksnos", lambda v: bool(re.search(r"[A-Za-z0-9]", v)) and not re.search(r"\bNumber\s+of\s+Packages\b", v, re.I), ) recover_labeled( [r"Import\s+General\s+Manifest\s*(?:\(\s*IGM\s*\))?\s*Number\s*:?\s*([0-9A-Za-z/-]+)", r"Import\s+General\s+Manifest\s*Number\s*:?\s*([0-9A-Za-z/-]+)"], "cour_be_igmno", lambda v: bool(re.fullmatch(r"[0-9A-Za-z/-]{1,30}", v)), ) recover_labeled( [r"Date\s+Of\s+MAWB\s*:?\s*([0-9]{1,2}[/-][0-9]{1,2}[/-][0-9]{2,4})", r"MAWB\s+Date\s*:?\s*([0-9]{1,2}[/-][0-9]{1,2}[/-][0-9]{2,4})"], "cour_be_mawb_dt", self._valid_date, ) recover_labeled( [r"Date\s+of\s+HAWB\s*:?\s*([0-9]{1,2}[/-][0-9]{1,2}[/-][0-9]{2,4})", r"HAWB\s+Date\s*:?\s*([0-9]{1,2}[/-][0-9]{1,2}[/-][0-9]{2,4})"], "cour_be_hawb_dt", self._valid_date, ) # ------------------------------------------------------------- # 3. Item manufacturer recovery from the item's own semantic block. # ------------------------------------------------------------- # This is intentionally independent of the generic manufacturer pass. # A blank value is recovered from the nearest explicit item section, # including multi-line labels such as ``Name of Manufacturer :``. for m in matches if 'matches' in locals() else []: pass manufacturer_rx = re.compile( r"(?:Name\s+of\s+Manufacturer|Manufacturer\s+Name|Manufacturer)" r"\s*:\s*(.*?)" r"(?=\s+(?:Brand|Model|Grade|Specification|End\s+Use\s+of\s+Item|Country\s+of\s+Origin|Accessories)\s*:|\s*$)", re.I | re.S, ) # Associate candidates with explicit item sections by source order. item_sections = list(re.finditer( r"DETAILS\s*OF\s*ITEM\s*(?:[-:\u2010\u2011\u2012\u2013\u2014]\s*)?(\d+)\b", text, re.I )) for idx, item in enumerate(result.get("items") or []): current = clean(item.get("cour_be_item_manufacturer")) if current and not current.startswith(":"): continue section_match = item_sections[idx] if idx < len(item_sections) else None if not section_match: continue start = section_match.end() end = item_sections[idx + 1].start() if idx + 1 < len(item_sections) else min(len(text), start + 12000) block = text[start:end] candidates = [clean(x.group(1)) for x in manufacturer_rx.finditer(block)] candidates = [x for x in candidates if x and not self._looks_like_any_known_label(x) and not re.search(r"\b(?:Brand|Model|Grade|Specification)\s*:", x, re.I)] if candidates: item["cour_be_item_manufacturer"] = self._repair_wrapped_words(max(candidates, key=len)) # ------------------------------------------------------------- # 4. Explicit item sections: make compact/spaced headings equivalent. # ------------------------------------------------------------- heading_re = re.compile( r"DETAILS\s*OF\s*ITEM\s*(?:[-:\u2010\u2011\u2012\u2013\u2014]\s*)?(\d+)\b", re.I, ) matches = list(heading_re.finditer(text)) if not matches: return # Only the first occurrence of each logical item number is the section # anchor. Repeated observations of the same section are not new rows. sections = [] seen = set() for m in matches: n = int(m.group(1)) if n in seen: continue seen.add(n) sections.append((n, m.start(), m.end())) existing = [x for x in (result.get("items") or []) if isinstance(x, dict)] by_section = {} unscoped = [] for row in existing: raw = row.get("_item_section_no") if raw in (None, ""): raw = row.get("cour_be_item_itemsn") try: n = int(float(raw)) except (TypeError, ValueError): unscoped.append(row) continue by_section.setdefault(n, row) # The existing authoritative extractor will populate the fields when # the parse pipeline reaches it. Here we only guarantee row existence # and provenance, preventing an entirely missed section from vanishing. for n, _start, _end in sections: if n not in by_section: row = self._empty_item(n) row["_item_section_no"] = n by_section[n] = row rebuilt = [] for n, _start, _end in sections: rebuilt.append(by_section[n]) rebuilt.extend(unscoped) # Keep exact explicit rows separate; dedupe cannot merge different # section numbers. result["items"] = self._dedupe_items(rebuilt) # Re-run the section-authoritative extractor after materialization. This # populates a section that an earlier generic pass completely missed; # it does not alter parent/invoice logic. try: self._final_authoritative_item_sections(result) except Exception: pass # Ensure invoice supplier address/name survives any later item-only # reconciliation and remove only obvious footer contamination. for inv in result.get("invoices") or []: addr = clean(inv.get("suppliers_address")) if addr: inv["suppliers_address"] = addr sup = clean(inv.get("cour_be_inv_supplier_name")) if sup: inv["cour_be_inv_supplier_name"] = sup # ========================================================= # V15 HARDENING PASS # ========================================================= # The generic semantic engine is intentionally broad. This final pass # narrows candidates using the visual relationship of the actual label, # preventing neighbouring headers, page counters and boilerplate from # becoming business values. It is layout-relative, not coordinate-fixed. def _ultra_dynamic_igm_recovery(self, result): """Recover the document-level Import General Manifest number. This is deliberately the final IGM-only recovery layer. It does not depend on page numbers, fixed coordinates, or one exact PDF wording. It understands the relationship between the semantic IGM heading and the value that follows it, including PDFs where the heading/value are split into separate lines or visual rows. """ import re def clean(v): if v is None: return "" try: v = self._clean(str(v)) except Exception: v = str(v) return re.sub(r"^[\s:;,.-]+|[\s:;,.-]+$", "", v).strip() def valid(v): return bool(re.fullmatch(r"\d{1,20}", clean(v))) current = clean(result.get("cour_be_igmno")) # A real numeric value, including 0, must never be replaced. if valid(current): return source = str(getattr(self, "full_text", "") or "") if not source: return # Normalize PDF extraction noise while retaining token order. This # handles spaces, line breaks, punctuation and soft hyphens inserted # between words by different PDF generators. normalized = source.replace("\u00ad", " ").replace("\u00a0", " ") normalized = re.sub(r"[^A-Za-z0-9]+", " ", normalized) normalized = re.sub(r"\s+", " ", normalized).strip() # Prefer the complete semantic field, never the shorter generic # 'IGM Number' field. This prevents Gateway IGM Number from winning. label_patterns = ( r"\bIMPORT\s+GENERAL\s+MANIFEST\s+\(?(?:IGM)\)?\s+NUMBER\b", r"\bIMPORT\s+GENERAL\s+MANIFEST\s+NUMBER\b", r"\bIMPORTGENERALMANIFEST\s+IGM\s+NUMBER\b", ) # First: exact semantic phrase followed by a local value. for lp in label_patterns: for m in re.finditer(lp, normalized, re.I): window = normalized[m.end():m.end()+100] # Stop before another structural field. window = re.split( r"\b(?:DATE\s+OF\s+ENTRY\s+INWARD|MASTER\s+AIRWAY\s+BILL|HOUSE\s+AIRWAY\s+BILL|GATEWAY\s+IGM\s+NUMBER)\b", window, maxsplit=1, flags=re.I | re.X )[0] nm = re.search(r"\b(\d{1,20})\b", window) if nm and valid(nm.group(1)): result["cour_be_igmno"] = nm.group(1) return # Second: line-oriented recovery. Many CBE PDFs extract the field as: # Import General Manifest # (IGM) Number : # 123 Date of Entry Inward : ... lines = [re.sub(r"\s+", " ", x.replace("\u00ad", " ")).strip() for x in source.splitlines()] for i, line in enumerate(lines): compact = re.sub(r"[^A-Za-z0-9]+", " ", line).strip().upper() if "IMPORT GENERAL MANIFEST" not in compact: continue # Exclude gateway rows explicitly. if "GATEWAY" in compact: continue local = " ".join(lines[i:i+5]) local_norm = re.sub(r"[^A-Za-z0-9]+", " ", local) local_norm = re.sub(r"\s+", " ", local_norm).strip() lm = re.search( r"IMPORT\s+GENERAL\s+MANIFEST(?:\s+IGM)?\s+NUMBER\s+(.*)", local_norm, re.I ) if not lm: continue tail = re.split( r"\b(?:DATE\s+OF\s+ENTRY\s+INWARD|MASTER\s+AIRWAY\s+BILL|HOUSE\s+AIRWAY\s+BILL|GATEWAY\s+IGM\s+NUMBER)\b", lm.group(1), maxsplit=1, flags=re.I | re.X )[0] nm = re.search(r"\b(\d{1,20})\b", tail) if nm and valid(nm.group(1)): result["cour_be_igmno"] = nm.group(1) return # Third: visual-word recovery. This handles PDFs whose page text is # incomplete but whose pdfplumber word model still contains the label. for page_no, rows in getattr(self, "page_rows", {}).items(): for idx, row in enumerate(rows): row_text = clean(row.get("text", "")) compact_row = re.sub(r"[^A-Za-z0-9]+", " ", row_text).upper() if "IMPORT GENERAL MANIFEST" not in compact_row or "GATEWAY" in compact_row: continue # Inspect the following visual rows only. The nearest numeric # token before Date of Entry Inward is the IGM value. for rr in rows[idx:idx+6]: txt = clean(rr.get("text", "")) if re.search(r"DATE\s+OF\s+ENTRY\s+INWARD", txt, re.I): break for token in re.findall(r"(? dict[str, Any]: result = self._empty_result() self._extract_parent_semantic(result) self._extract_invoices(result) self._extract_items(result) if self.form_type == "CBE-XIII": self._extract_cbe_xiii_structural(result) self._recover_from_generic_tables_v2(result) self._extract_payment_semantic(result) self._extract_related_party(result) self._recover_missing_parent_fields(result) self._recover_missing_child_fields(result) self._recover_from_generic_tables_v2(result, second_pass=True) self._reconcile_child_rows(result) # Final semantic/layout hardening. These passes are deliberately # authoritative only when the candidate has stronger evidence than # the broad generic extractor. self._harden_xiii_header_and_parties(result) self._harden_text_layout_fields(result) self._harden_supplier_cross_page(result) self._harden_invoice_fallback(result) self._harden_parent_fields(result) self._harden_payment(result) self._harden_party_fields(result) self._harden_child_rows(result) self._ensure_invoice_from_text_final(result) # V16: authoritative semantic recovery for the standard Courier # sections. This is label/block based and does not use page numbers # or fixed coordinates. It repairs weak generic candidates for # invoices, items, IEC branch and payment rows before finalization. self._repair_semantic_courier_sections(result) # V18.3 dynamic extension: preserve the accurate v18.3 extraction # pipeline and add only conservative recovery/cleanup for layouts # whose labels or line wrapping differ from the known forms. self._v183_future_layout_recovery(result) # Final targeted recovery: invoice blocks and wrapped manufacturers. self._final_dynamic_invoice_manufacturer_recovery(result) # Dynamic multi-page item recovery: preserve every explicit Details Of Item - N section. self._recover_explicit_item_sections_dynamic(result) # V17: final logical child-row reconciliation. # Generic table/semantic passes can observe the same logical section # multiple times. Collapse those observations into the actual invoice # and item sections represented by the PDF before schema finalization. self._finalize_logical_child_rows(result) # V18.3 FINAL ITEM AUTHORITY: after all generic/table reconciliation, # re-read only the explicit Details Of Item - N sections. This is the # last item-specific pass, so footer text, page metadata, or a weak # generic-table observation cannot overwrite the actual item values. # It does not touch parent or invoice extraction. self._final_authoritative_item_sections(result) # V17.2: final output-only normalization. Extraction logic remains unchanged. self._normalize_item_label_prefixes(result) # V18.3 FINAL semantic field reconciliation. This is intentionally # the last extraction-aware pass: it does not change the existing # parser architecture, but gives explicit labelled sections priority # over weak/global observations. All boundaries are derived from the # uploaded document's own labels; no page number or fixed coordinate # is used. self._final_semantic_field_reconciliation(result) self._final_visual_section_override(result) # V18.3 FINAL ITEM NUMERIC + ADDRESS AUTHORITY. This pass is limited # to the two known weak areas: item-level pricing/valuation fields and # supplier-address token duplication. It uses the uploaded document's # own explicit labels and item-section boundaries; it does not change # parent/invoice extraction logic. self._final_item_numeric_and_supplier_address_recovery(result) # Final v18.3 targeted recovery. This is deliberately the last # extraction-aware step so weaker generic observations cannot overwrite # the repaired supplier/item values. self._v183_final_targeted_recovery(result) # Re-apply the narrowly scoped item numeric/address authority AFTER # the legacy v18.3 targeted recovery so that weaker observations from # that pass cannot overwrite proven values with schema defaults. self._final_item_numeric_and_supplier_address_recovery(result) # Final targeted semantic recovery for IGM, marks and item manufacturer. self._final_targeted_igm_marks_manufacturer_recovery(result) # FINAL IGM AUTHORITY: recover only when IGM is still missing. # This pass is intentionally isolated so no other field/extraction # logic can be affected. self._ultra_dynamic_igm_recovery(result) # FINAL DUTY TABLE AUTHORITY: map Ad Valorem to rate and Duty Amount # to amount from the semantic duty-table columns. This touches only # item duty rate/amount fields and runs after every generic extractor. self._final_duty_table_authority(result) self._normalize_courier_output_contract(result) self._finalize_parent_fields(result) self._finalize_children(result) self._schema_complete(result) # Final output-only safety pass. It runs after child deduplication so # a stronger/older invoice observation cannot overwrite the supplier # address recovered from the actual visual Supplier Details row. self._final_output_safety_cleanup(result) # Final ultra-dynamic semantic authority for the three fields that # can be lost by PDF text-order differences. self._final_ultra_dynamic_item_and_igm_recovery(result) # Final item-only authority for General Description. self._final_item_general_description_authority(result) # FINAL INVOICE SEMANTIC AUTHORITY: recover invoice payment terms, # nature of transaction and the invoice-level related-party indicator # from semantic labels/sections only. Marks-and-numbers receives only # artifact cleanup because a genuinely blank field is valid. self._final_invoice_semantic_authority(result) # FINAL CBE-XIII ITEM AUTHORITY: recover the condensed item row and # its duty table only from the document's own semantic labels. This # runs last so a weak generic/table pass cannot erase the item again. self._final_cbe_xiii_item_authority(result) return {"courier_bill_json": result, "_trace": self.trace} def _final_cbe_xiii_item_authority(self, result): """Final, isolated recovery for condensed CBE-XIII item data. CBE-XIII is structurally different from the long CBE-XIV form: the item description, commercial values and invoice fields are usually on page one while the duty table is on a later page. Generic row/table passes can therefore produce an empty item row or shift duty columns. This method uses semantic labels and the document's own section boundaries. It does not use page numbers, fixed coordinates, product names or hard-coded duty values. """ source = str(getattr(self, "full_text", "") or "") if not source: return def clean(v): v = str(v or "") v = v.replace("\u00ad", "").replace("\ufeff", "") v = v.replace("\ufffe", "").replace("\uffff", "") v = re.sub(r"\s+", " ", v).strip() v = re.sub(r"^[\s:;,-]+|[\s:;,-]+$", "", v) return v # Generic manufacturer artifact cleanup. This is safe for every # Courier layout because it removes only text that is itself a known # adjacent field label; it never changes a clean manufacturer name. for existing_item in result.get("items") or []: raw_manufacturer = clean(existing_item.get("cour_be_item_manufacturer", "")) if raw_manufacturer: cleaned_manufacturer = re.split( r"\b(?:Address\s+of\s+Manufacturer|Brand|Model|Grade|Specification|End\s+Use\s+of\s+Item|Country\s+of\s+Origin)\b", raw_manufacturer, maxsplit=1, flags=re.I, )[0] existing_item["cour_be_item_manufacturer"] = clean(cleaned_manufacturer) if not re.search(r"CBE\s*-?\s*XIII|CBEXIII", source, re.I): return def num(v): try: return float(str(v).replace(",", "").strip()) except (TypeError, ValueError): return None def norm_label(v): return re.sub(r"[^a-z0-9]+", "", str(v or "").lower()) # ------------------------------------------------------------------ # Locate the actual DETAILS OF ITEM block. Prefer the page containing # the heading so the search cannot spill into the duty/payment pages. # ------------------------------------------------------------------ item_text = "" for page in self.pages: txt = str(page.get("text") or "") if re.search(r"DETAILS\s+OF\s+ITEM", txt, re.I): item_text = txt break if not item_text: m = re.search( r"DETAILS\s+OF\s+ITEM\b(.*?)(?=\bPAYMENT\s+DETAILS\b|\bDECLARATION\b|$)", source, re.I | re.S, ) item_text = m.group(0) if m else "" if not item_text: return # Stop at the actual page/footer or a later major section. item_text = re.split( r"\b(?:Page\s+\d+\s+of\s+\d+|DUTY\s+DETAILS|PAYMENT\s+DETAILS|DECLARATION)\b", item_text, maxsplit=1, flags=re.I, )[0] # A value is always bounded by the next semantic label. This is the # key protection against artifacts such as "N/A Address of". labels = [ r"License\s+Type", r"License\s+Number", r"CTSH", r"CETSH", r"Country\s+of\s+Origin", r"Description\s+of\s+Goods", r"Item\s+Description", r"General\s+Description", r"Name\s+of\s+Manufacturer", r"Address\s+of\s+Manufacturer", r"Number\s+of\s+Packages", r"Marks\s+on\s+Packages", r"Unit\s+of\s+Measure", r"Quantity", r"Invoice\s+Number", r"Invoice\s+Value", r"Unit\s+Price", r"Currency\s+of\s+Unit\s+Price", r"Currency\s+of\s+Invoice", r"Rate\s+of\s+Exchange", r"Invoice\s+Term", r"Landing\s+Charges", r"Insurance", r"Freight", r"Discount\s+Amount", r"Currency\s+of\s+Discount", r"Assessable\s+Value", r"Duty\s*\(Rs\.\)", ] label_union = r"(?:" + "|".join(labels) + r")" def value_after(patterns, validator=None): best = "" for pattern in patterns: m = re.search( pattern + r"\s*[:：-]?\s*(.*?)(?=\s+" + label_union + r"\s*[:：-]|\s*$)", item_text, re.I | re.S, ) if not m: # Same line/line-wrap fallback: only one physical line. m = re.search( pattern + r"\s*[:：-]?\s*([^\n\r]+)", item_text, re.I, ) if not m: continue candidate = clean(m.group(1)) if not candidate: continue if validator and not validator(candidate): continue if len(candidate) > len(best): best = candidate return best def code(v): return bool(re.fullmatch(r"[A-Za-z0-9./_-]{2,20}", clean(v))) def country(v): v = clean(v) return bool(v) and not re.search( r"(?:Description|Manufacturer|Address|Quantity|Invoice|Currency|Rate|Assessable|Duty)", v, re.I, ) and len(v) <= 100 def party(v): v = clean(v) if not v or len(v) > 160: return False if re.search( r"(?:Address\s+of|Number\s+of|Unit\s+of|Country\s+of|Invoice\s+Number|Currency\s+of|Rate\s+of)", v, re.I, ): return False return True description = value_after( [r"Description\s+of\s+Goods", r"Item\s+Description"], lambda v: 1 <= len(v) <= 250 and not re.search(r"CopyRight|Currency\s+for", v, re.I), ) manufacturer = value_after( [r"Name\s+of\s+Manufacturer", r"Manufacturer"], party, ) # If a PDF's visual text extractor placed the next label directly in # the value, remove only the label boundary; never use a company-name # dictionary or a fixed manufacturer string. manufacturer = re.split( r"\b(?:Address\s+of\s+Manufacturer|Brand|Model|Grade|Specification|End\s+Use\s+of\s+Item|Country\s+of\s+Origin)\b", manufacturer, maxsplit=1, flags=re.I, )[0] manufacturer = clean(manufacturer) country_value = value_after([r"Country\s+of\s+Origin"], country) ctsh = value_after([r"CTSH"], lambda v: bool(re.fullmatch(r"\d{4,12}", clean(v)))) cetsh = value_after([r"CETSH"], lambda v: bool(re.fullmatch(r"\d{4,12}", clean(v)))) uom = value_after([r"Unit\s+of\s+Measure"], code) qty_raw = value_after([r"Quantity"], lambda v: num(v) is not None) unit_price_raw = value_after([r"Unit\s+Price"], lambda v: num(v) is not None) exchange_raw = value_after([r"Rate\s+of\s+Exchange"], lambda v: num(v) is not None) assess_raw = value_after([r"Assessable\s+Value"], lambda v: num(v) is not None) invoice_no = value_after([r"Invoice\s+Number"], lambda v: bool(re.fullmatch(r"[A-Za-z0-9./_-]{3,40}", clean(v)))) invoice_value_raw = value_after([r"Invoice\s+Value"], lambda v: num(v) is not None) inv_currency = value_after([r"Currency\s+of\s+Invoice", r"Currency"], lambda v: bool(re.fullmatch(r"[A-Za-z]{3}", clean(v)))) upi_currency = value_after([r"Currency\s+of\s+Unit\s+Price"], lambda v: bool(re.fullmatch(r"[A-Za-z]{3}", clean(v)))) incoterm = value_after([r"Invoice\s+Term"], lambda v: bool(re.fullmatch(r"[A-Za-z0-9]{2,10}", clean(v)))) # ------------------------------------------------------------------ # Build/enrich exactly one CBE-XIII item. Never delete a valid item. # ------------------------------------------------------------------ items = result.setdefault("items", []) target = items[0] if items else self._empty_item(1) target.setdefault("_item_section_no", 1) target["cour_be_item_itemsn"] = 1 target["linked_invoice_id_sno"] = 1 values = { "cour_be_item_itemdescription": description, "cour_be_item_itemgendescription": description, "cour_be_item_manufacturer": manufacturer, "cour_be_item_countryoforigin": country_value, "cour_be_item_ctsh": ctsh, "cour_be_item_cetsh": cetsh, "cour_be_item_uom": uom, "cour_be_item_qty": num(qty_raw) if qty_raw else None, "cour_be_item_upi": num(unit_price_raw) if unit_price_raw else None, "cour_be_item_upi_currency": upi_currency.upper() if upi_currency else "", "cour_be_item_exchg_rate": num(exchange_raw) if exchange_raw else None, "cour_be_item_assessable_value": num(assess_raw) if assess_raw else None, } for field, value in values.items(): if value not in ("", None): target[field] = value # ------------------------------------------------------------------ # Duty table: parse the semantic columns, not the first numeric token. # The row's serial number is deliberately ignored. # ------------------------------------------------------------------ duty_match = re.search( r"DUTY\s+DETAILS\b(.*?)(?=\bPAYMENT\s+DETAILS\b|\bDECLARATION\b|$)", source, re.I | re.S, ) if duty_match: duty_text = duty_match.group(1) duty_aliases = { "BCD": ("cour_be_item_bcd_rt", "cour_be_item_bcd_amt"), "SW SRCHRG": ("cour_be_item_sws_rt", "cour_be_item_sws_amt"), "SWS": ("cour_be_item_sws_rt", "cour_be_item_sws_amt"), "IGST": ("cour_be_item_igst_rt", "cour_be_item_igst_amt"), "AIDC": ("cour_be_item_aidc_rt", "cour_be_item_aidc_amt"), "CMPNSTRY": ("cour_be_item_cmpcess_rt", "cour_be_item_cmpcess_amt"), } for line in duty_text.splitlines(): line = clean(line) if not line or "Duty Head" in line: continue for head, fields in duty_aliases.items(): if not re.search(r"\b" + re.escape(head) + r"\b", line, re.I): continue hm = re.search(r"\b" + re.escape(head) + r"\b", line, re.I) tail = line[hm.end():] nums = re.findall(r"(? 300 or footer_or_form_noise(value): return False if self._looks_like_any_known_label(value): return False if not re.search(r"[A-Za-z]{2,}", value): return False # A bare number/date/page token cannot be a goods description. if re.fullmatch(r"[0-9./\-\s]+", value): return False return True def valid_country(value): value = clean(value) if not value or len(value) > 100 or footer_or_form_noise(value): return False if self._looks_like_any_known_label(value): return False if re.search( r"\b(?:DETAILS|DESCRIPTION|QUANTITY|UNIT|PRICE|MANUFACTURER|" r"BRAND|MODEL|GRADE|SPECIFICATION|RITC|CTSH|CETSH|ASSESSABLE|" r"CURRENCY|COPYRIGHT|PAGE|CONSINGMENT|CONSIGNMENT)\b", value, re.I, ): return False if re.search(r"\b[A-Za-z][A-Za-z ]{2,45}\s*:", value): return False # Two-letter fragments such as ``di`` are typical text-flow # leftovers, not country names. Keep the semantic field broad # enough for real country names without requiring a fixed country # dictionary. if len(re.sub(r"[^A-Za-z]", "", value)) < 3: return False return bool(re.search(r"[A-Za-z]{2,}", value)) def find_section_value(block, label_pattern, stop_pattern, max_len=300): """Return the strongest label-local value in a section.""" candidates = [] rx = re.compile( label_pattern + r"\s*[:\-]?\s*", re.I, ) for m in rx.finditer(block): tail = block[m.end():m.end() + max_len] if stop_pattern: stop = re.search(stop_pattern, tail, re.I | re.S) if stop: tail = tail[:stop.start()] value = clean(tail) if value: candidates.append((m.start(), value)) return candidates # Build section text without using page numbers. A section may span # several PDF pages, so only the next logical item heading terminates it. section_blocks = {} for i, (n, _start, end) in enumerate(sections): next_start = sections[i + 1][1] if i + 1 < len(sections) else len(text) block = text[end:next_start] # Document-level endings are safe boundaries for the final item. for stop in ( r"\bSHIPPING\s+BILL\s+DETAILS\b", r"\bPAYMENTS?\s+MADE\b", r"\bPAYMENT\s+DETAILS\b", r"\bDETAILS\s+RELATING\s+TO\s+DUTY\s+EXEMPTION\b", r"\bDECLARATION\b", ): sm = re.search(stop, block, re.I) if sm: block = block[:sm.start()] break section_blocks[n] = block # Materialize every explicit logical item section before reconciliation. # A generic extractor may have missed the first/last physical row entirely; # the presence of a real "Details Of Item - N" section is stronger evidence # than the number of rows produced by earlier passes. existing_items = [x for x in (result.get("items") or []) if isinstance(x, dict)] section_rows = {} unscoped_items = [] for existing in existing_items: raw_no = existing.get("_item_section_no") if raw_no in (None, ""): raw_no = existing.get("cour_be_item_itemsn") try: sec_no = int(float(raw_no)) except (TypeError, ValueError): unscoped_items.append(existing) continue section_rows.setdefault(sec_no, existing) for sec_no, _start, _end in sections: if sec_no not in section_rows: fresh = self._empty_item(sec_no) fresh["_item_section_no"] = sec_no section_rows[sec_no] = fresh ordered_items = [] for sec_no, _start, _end in sections: ordered_items.append(section_rows[sec_no]) # Keep any non-section observations temporarily; they can still contribute # missing fields during the normal merge, but cannot suppress explicit rows. ordered_items.extend(unscoped_items) # Match rows by explicit section provenance. If an older extractor did # not retain that private provenance, the section materialization above # has already created the authoritative row. for row_index, item in enumerate(ordered_items, 1): if not isinstance(item, dict): continue raw_no = item.get("_item_section_no") if raw_no in (None, ""): raw_no = item.get("cour_be_item_itemsn") try: section_no = int(float(raw_no)) except (TypeError, ValueError): section_no = row_index block = section_blocks.get(section_no) if not block: continue # --------------------------------------------------------- # Item Description -- authoritative label-local value. # --------------------------------------------------------- desc_candidates = find_section_value( block, r"Item\s*Description", r"(?=\s*(?:General\s+Description|Currency\s+for\s+Unit\s+Price|" r"Unit\s+Price|Unit\s+of\s+Measure|UOM|Quantity|Rate\s+of\s+Exchange|" r"Accessories|Name\s+of\s+Manufacturer|Manufacturer|Brand|Model|" r"Grade|Specification|End\s+Use\s+of\s+Item|Country\s*of\s*Origin|" r"RITC|CTSH|CETSH|Assessable\s*Value)\s*:)", 350, ) valid_descs = [(pos, val) for pos, val in desc_candidates if valid_description(val)] if valid_descs: # Prefer the earliest semantic occurrence. It is the actual # field in this physical item section, not a later declaration. item["cour_be_item_itemdescription"] = min( valid_descs, key=lambda x: x[0] )[1] # --------------------------------------------------------- # Quantity -- must be directly attached to Quantity/Qty. # --------------------------------------------------------- qty_candidates = [] qty_rx = re.compile( r"(?:Quantity|Qty|No\.\s*of\s*Units|Number\s+of\s+Units)" r"\s*[:\-]?\s*" r"([0-9]+(?:,[0-9]{3})*(?:\.\d+)?)\b", re.I, ) for qm in qty_rx.finditer(block): n = number(qm.group(1)) if n is None or is_year(n): continue qty_candidates.append((qm.start(), n)) if qty_candidates: item["cour_be_item_qty"] = qty_candidates[0][1] # --------------------------------------------------------- # Country of Origin -- same-line first, then one short line. # --------------------------------------------------------- country_candidates = find_section_value( block, r"Country\s*of\s*Origin", r"(?=\s*(?:Country\s+of\s+Consignment|Details\s+In\s+Case\s+Of\s+Previous\s+Imports|" r"Classification\s+Details|RITC|CTSH|CETSH|Assessable\s*Value|" r"Duty\s+Details|Shipping\s+Bill\s+Details|Details\s+Of\s+Item)\b)", 120, ) valid_countries = [ (pos, val) for pos, val in country_candidates if valid_country(val) ] if valid_countries: item["cour_be_item_countryoforigin"] = min( valid_countries, key=lambda x: x[0] )[1] else: # PDF layout text can put the value on the immediately next # visual line. Inspect only a short span after the label. for cm in re.finditer(r"Country\s*of\s*Origin\s*:\s*", block, re.I): tail = block[cm.end():cm.end() + 100] first_line = clean(tail.splitlines()[0]) if tail.splitlines() else clean(tail) first_line = re.split( r"\s+(?=(?:Country\s+of\s+Consignment|Details\s+In\s+Case|" r"Classification\s+Details|RITC|CTSH|CETSH|Assessable\s*Value)\b)", first_line, maxsplit=1, flags=re.I, )[0] if valid_country(first_line): item["cour_be_item_countryoforigin"] = first_line break # --------------------------------------------------------- # Assessable Value -- section-local, non-zero first. # --------------------------------------------------------- assess_rx = re.compile( r"(?:Assessable\s*Value|Assessable\s*Val|Value\s*for\s*Assessment|" r"Assessment\s+Value)\s*[:\-]?\s*" r"([0-9]+(?:,[0-9]{3})*(?:\.\d+)?)\b", re.I, ) assess_candidates = [] for am in assess_rx.finditer(block): n = number(am.group(1)) if n is None or n == 0 or is_year(n): continue assess_candidates.append((am.start(), n)) if assess_candidates: # Nearest explicit label/value pair wins; no global monetary # scan is used, so another item's value cannot leak in. item["cour_be_item_assessable_value"] = min( assess_candidates, key=lambda x: x[0] )[1] # Final local guards. Never allow the known footer corruption back. desc = clean(item.get("cour_be_item_itemdescription")) if not valid_description(desc): item["cour_be_item_itemdescription"] = "" qty = number(item.get("cour_be_item_qty")) if qty is None or is_year(qty): item["cour_be_item_qty"] = 0.0 else: item["cour_be_item_qty"] = qty country = clean(item.get("cour_be_item_countryoforigin")) if not valid_country(country): item["cour_be_item_countryoforigin"] = "" else: item["cour_be_item_countryoforigin"] = country assess = number(item.get("cour_be_item_assessable_value")) if assess is not None: item["cour_be_item_assessable_value"] = assess # Rebuild output in the document's logical item order. Explicit # Details Of Item - N sections are authoritative and therefore cannot # disappear merely because a generic extraction pass missed one. final_items = [] used = set() for sec_no, _start, _end in sections: row = section_rows.get(sec_no) if row is not None: final_items.append(row) used.add(id(row)) # Preserve useful unscoped observations only when they do not duplicate # an explicit section. The normal dedupe layer handles true observations # of the same physical row. for row in unscoped_items: if id(row) not in used: final_items.append(row) result["items"] = self._dedupe_items(final_items) def _final_semantic_field_reconciliation(self, result): """Final label-scoped reconciliation for weak Courier observations. The existing v18.3 pipeline intentionally has many extraction passes. Those passes can still produce a weak candidate from a neighbouring label/table. This method is the final authority only for values that are explicitly attached to a semantic label inside a bounded business section. Important properties: - no fixed page numbers; - no document-specific values; - no hardcoded company/product names; - repeated item sections are reconciled by their printed section number; - blank optional fields stay blank instead of consuming the next label. """ text = self.full_text or "" normalized = text.replace("\u00ad", "") def clean(value): value = self._clean(value) # Join words split by a PDF line-wrap hyphen, e.g. CON- / DUCTA. value = re.sub(r"(?<=[A-Za-z])-\s+(?=[A-Za-z])", "", value) value = re.sub(r"^[\s:;]+|[\s:;]+$", "", value) return self._repair_wrapped_words(value) def num(value): return float(self._float(value) or 0.0) def section_between(source, start_pattern, stop_patterns): m = re.search(start_pattern, source, re.I) if not m: return "" tail = source[m.end():] stops = [] for pattern in stop_patterns: sm = re.search(pattern, tail, re.I) if sm: stops.append(sm.start()) return tail[:min(stops)] if stops else tail def labelled_value(block, label_patterns, stop_patterns, allow_blank=False): if not block: return None labels = "(?:" + "|".join(label_patterns) + ")" stops = "(?:" + "|".join(stop_patterns) + ")" # Capture until the next semantic label. This allows multiline # legal names/addresses while preventing neighbouring fields from # entering the value. pattern = rf"{labels}\s*:\s*(.*?)(?=\s+{stops}\s*:|$)" m = re.search(pattern, block, re.I | re.S) if not m: return None value = clean(m.group(1)) if not value and not allow_blank: return None return value # ------------------------------------------------------------- # 1. IMPORTER / HEADER SECTION # ------------------------------------------------------------- importer_block = section_between( normalized, r"PARTICULARS\s+OF\s+THE\s+IMPORTER", [r"PARTICULARS\s+OF\s+HIGH\s+SEA\s+SELLER", r"SPECIAL\s+REQUESTS"], ) if importer_block: # The importer name and address share a visual row in several # CBE-XIV variants. Anchor the address to that row's Address label # and stop at the next importer field, rather than using a generic # global Address match. importer_address = None iam = re.search( r"\bName\s*:\s*.*?\bAddress\s*:\s*(.*?)(?=\s+(?:Category\s+Of\s+Importer|Type\s+Of\s+Importer|In\s+Case\s+Of\s+other\s+Importer|Authorised\s+Dealer\s+Code|BOE\s+Number|BOE\s+Date)\b)", importer_block, re.I | re.S, ) if iam: importer_address = clean(iam.group(1)) if not importer_address: importer_address = labelled_value( importer_block, [r"Address"], [r"Category\s+Of\s+Importer", r"Type\s+Of\s+Importer", r"Authorised\s+Dealer\s+Code", r"BOE\s+Number", r"BOE\s+Date"], ) if importer_address and self._looks_like_address(importer_address): result["cour_be_importer_address"] = importer_address ad = re.search( r"Authorised\s+Dealer\s+Code\s+Of(?:\s+Bank)?\s*:?\s*(\d{6,12})", importer_block, re.I | re.S, ) if ad: result["cour_be_adcode"] = ad.group(1) branch = re.search( r"Import\s+Export\s+Branch\s*(?:Code\s*)?:\s*([A-Za-z0-9]{1,8})", importer_block, re.I | re.S, ) if branch and branch.group(1).upper() != "CODE": result["cour_be_iecbranch"] = clean(branch.group(1)) # ------------------------------------------------------------- # 2. PARENT COUNTRY / AIRWAY-BILL SECTION # ------------------------------------------------------------- # Restrict country-of-origin recovery to the header before IGM. This # prevents an item-level Country of Origin label or the miscellaneous # "Country of Origin Certificate" label from winning. header_before_igm = re.split(r"\bIGM\s+DETAILS\b", normalized, maxsplit=1, flags=re.I)[0] coo_matches = list(re.finditer( r"\bCountry\s*of\s*Origin\s*:\s*([^\n]+?)(?=\s+Country\s+of\s+Consignment\s*:|$)", header_before_igm, re.I, )) if coo_matches: coo = clean(coo_matches[0].group(1)) if self._country_validator(coo): result["cour_be_coo"] = coo # The identifier is printed immediately after the MAWB/HAWB label in # standard forms. Do not search forward for a later generic "Number" # token because that can jump into the next field. mawb = re.search( r"Master\s+Airway\s+Bill\s*\(\s*MAWB\s*\)\s*([A-Za-z0-9][A-Za-z0-9._/-]{0,40})", normalized, re.I | re.S, ) if mawb: value = clean(mawb.group(1)) if self._alnum_validator(value): result["cour_be_mawbno"] = value hawb = re.search( r"House\s+Airway\s+Bill\s*\(\s*HAWB\s*\)\s*([A-Za-z0-9][A-Za-z0-9._/-]{0,60})", normalized, re.I | re.S, ) if hawb: value = clean(hawb.group(1)) if self._alnum_validator(value): result["cour_be_hawbno"] = value # ------------------------------------------------------------- # 3. SUPPLIER ADDRESS INSIDE THE ACTUAL INVOICE SECTION # ------------------------------------------------------------- invoice_headings = list(re.finditer( r"Details\s+Of\s+Invoice\s*(?:[-:]\s*\d+)?", normalized, re.I, )) for idx, heading in enumerate(invoice_headings): tail = normalized[heading.end():] next_heading = re.search( r"Details\s+Of\s+Invoice\s*(?:[-:]\s*\d+)?|Details\s+Of\s+Item\s*[-:]\s*\d+|DECLARATION", tail, re.I, ) block = tail[:next_heading.start()] if next_heading else tail[:9000] supplier_section = section_between( block, r"SUPPLIER\s+DETAILS", [r"Nature\s+of\s+Transaction", r"BROKER/\s*AGENT\s+DETAILS"], ) or block address = None am = re.search( r"\bAddress\s*:\s*(.*?)(?=\s+IF\s+SUPPLIER\s+IS\s+NOT\s+THE\s+SELLER\b|\s+BROKER/\s*AGENT\s+DETAILS\b|$)", supplier_section, re.I | re.S, ) if am: address = clean(am.group(1)) if address and self._looks_like_address(address): # Prefer the address belonging to the invoice row, never the # parent courier/importer address. for inv in result.get("invoices") or []: serial = str(inv.get("cour_be_invsno") or "1") if serial == str(idx + 1) or len(result.get("invoices") or []) == 1: inv["suppliers_address"] = address break # Also recover the supplier name from the same bounded block when # the existing value was truncated by a visual line boundary. supplier = labelled_value( supplier_section, [r"Name", r"Supplier\s+Name", r"Name\s+of\s+Supplier"], [r"Address", r"IF\s+SUPPLIER\s+IS\s+NOT\s+THE\s+SELLER", r"BROKER/\s*AGENT\s+DETAILS"], ) if supplier and self._strict_party_name(supplier): for inv in result.get("invoices") or []: if str(inv.get("cour_be_invsno") or "1") == str(idx + 1) or len(result.get("invoices") or []) == 1: inv["cour_be_inv_supplier_name"] = supplier break # ------------------------------------------------------------- # 4. EXPLICIT ITEM SECTIONS # ------------------------------------------------------------- item_matches = list(re.finditer( r"Details\s+Of\s+Item\s*[-:]\s*(\d+)", normalized, re.I, )) for pos, start in enumerate(item_matches): item_no = int(start.group(1)) end = item_matches[pos + 1].start() if pos + 1 < len(item_matches) else len(normalized) block = normalized[start.end():end] # Do not consume document-level material after this item's own # merchandise/duty section. The next explicit item heading is # already the primary boundary; these semantic section endings # protect the final item from declarations/footers. stop_positions = [] for pat in ( r"DECLARATION\s*\(", r"PAYMENT\s+DETAILS", r"DETAILS\s+RELATING\s+TO\s+DUTY\s+EXEMPTION", ): sm = re.search(pat, block, re.I) if sm: stop_positions.append(sm.start()) if stop_positions: block = block[:min(stop_positions)] # Locate the actual detailed-description subsection. This avoids # picking a generic footer/table occurrence of the same labels. desc_start = re.search( r"DETAILED\s+DESCRIPTION\s+OF\s+ITEM", block, re.I, ) detail_block = block[desc_start.end():] if desc_start else block field_stop_labels = [ r"General\s+Description", r"Currency\s+for\s+Unit\s+Price", r"Unit\s+Price", r"Unit\s+of\s+Measure", r"UOM", r"Quantity", r"Rate\s+Of\s+Exchange", r"Accessories", r"Name\s+of\s+Manufacturer", r"Manufacturer", r"Brand", r"Model", r"Grade", r"Specification", r"End\s+Use\s+of\s+Item", r"Country\s*of\s*Origin", r"RITC", r"CTSH", r"CETSH", r"Assessable\s*Value", r"DETAILS\s+IN\s+CASE\s+OF\s+PREVIOUS\s+IMPORTS", ] def item_value(labels, extra_stops=None, source=detail_block): stops = list(field_stop_labels) if extra_stops: stops.extend(extra_stops) labels_text = "(?:" + "|".join(labels) + ")" stops_text = "(?:" + "|".join(stops) + ")" m = re.search( rf"{labels_text}\s*:\s*(.*?)(?=\s+{stops_text}\s*:|$)", source, re.I | re.S, ) return clean(m.group(1)) if m else None description = item_value([r"Item\s*Description"]) general = item_value([r"General\s+Description"]) manufacturer = item_value([r"Name\s+of\s+Manufacturer", r"Manufacturer"]) # Some PDF generators wrap a manufacturer word across the Brand # column, e.g. ``CON- Brand:N.A DUCTA INC``. If the captured # manufacturer ends in a wrap hyphen, recover only the adjacent # alphabetic continuation before the next metadata label. This is # a structural continuation rule, not a company-name dictionary. if manufacturer and re.search(r"-$", manufacturer): after_man = detail_block[ max(0, detail_block.lower().find(manufacturer.lower())) + len(manufacturer): ] cm = re.search( r"Brand\s*:\s*[^\n]*?\b([A-Za-z][A-Za-z0-9 +.&/-]{1,100})\s+(?=(?:Model|Grade|Specification|End\s+Use|Country\s*of\s*Origin)\s*:)", after_man, re.I | re.S, ) if cm: continuation = clean(cm.group(1)) continuation = re.sub(r"^(?:N/?A|NA|NONE|N\.A\.)\s+", "", continuation, flags=re.I) if continuation: manufacturer = clean(manufacturer[:-1] + continuation) country = item_value([r"Country\s*of\s*Origin"]) # Some PDFs place Unit of Measure and Quantity on the same row. uom_match = re.search( r"(?:Unit\s+of\s+Measure|UOM)\s*:\s*([A-Za-z0-9._/-]+)", detail_block, re.I, ) qty_match = re.search( r"Quantity\s*:\s*([0-9][\d,]*(?:\.\d+)?)", detail_block, re.I, ) upi_currency = re.search( r"Currency\s+for\s+Unit\s+Price\s*:\s*([A-Za-z]{3,})", detail_block, re.I, ) upi = re.search( r"Unit\s+Price\s*:\s*([0-9][\d,]*(?:\.\d+)?)", detail_block, re.I, ) exch = re.search( r"Rate\s+Of\s+Exchange\s*:\s*([0-9][\d,]*(?:\.\d+)?)", detail_block, re.I, ) ritc = re.search( r"RITC\s*:\s*([A-Za-z0-9./_-]+)", block, re.I, ) ctsh = re.search(r"CTSH\s*:\s*(\d{4,12})", block, re.I) cetsh = re.search(r"CETSH\s*:\s*(\d{4,12})", block, re.I) assess = re.search( r"Assessable\s*Value\s*:\s*([0-9][\d,]*(?:\.\d+)?)", block, re.I, ) # Find the corresponding logical output row by explicit item # number. If serial provenance was removed by an earlier pass, # the normalized item order is the fallback. target = None for row in result.get("items") or []: sec = row.get("_item_section_no") serial = str(row.get("cour_be_item_itemsn") or "") if str(sec) == str(item_no) or serial == str(item_no): target = row break if target is None and item_no <= len(result.get("items") or []): target = result["items"][item_no - 1] if target is None: target = self._empty_item(item_no) target["_item_section_no"] = item_no result.setdefault("items", []).append(target) # Exact section evidence is stronger than any generic/table pass. if description: target["cour_be_item_itemdescription"] = description if general is not None: # A genuinely blank General Description must stay blank; never # substitute Unit Price/Currency/Quantity metadata. target["cour_be_item_itemgendescription"] = general if manufacturer and self._strict_party_name(manufacturer): target["cour_be_item_manufacturer"] = manufacturer if country and self._country_validator(country): target["cour_be_item_countryoforigin"] = country elif country == "": target["cour_be_item_countryoforigin"] = "" if uom_match: target["cour_be_item_uom"] = clean(uom_match.group(1)) if qty_match: target["cour_be_item_qty"] = num(qty_match.group(1)) if upi_currency: target["cour_be_item_upi_currency"] = clean(upi_currency.group(1)).upper() if upi: target["cour_be_item_upi"] = num(upi.group(1)) if exch: target["cour_be_item_exchg_rate"] = num(exch.group(1)) if ritc: value = clean(ritc.group(1)) if value not in {'""', "''", ":", "N/A"}: target["cour_be_item_ritc"] = value else: target["cour_be_item_ritc"] = "" if ctsh: target["cour_be_item_ctsh"] = ctsh.group(1) if cetsh: target["cour_be_item_cetsh"] = cetsh.group(1) if assess: target["cour_be_item_assessable_value"] = num(assess.group(1)) # Restore logical item ordering after any missing-row recovery. result["items"] = sorted( result.get("items") or [], key=lambda r: int(r.get("_item_section_no") or r.get("cour_be_item_itemsn") or 999999), ) for idx, row in enumerate(result["items"], 1): row["cour_be_item_itemsn"] = idx row["linked_invoice_id_sno"] = 1 # ------------------------------------------------------------- # 5. FINAL SUPPLIER ADDRESS FALLBACK FROM THE INVOICE SECTION # ------------------------------------------------------------- if result.get("invoices"): for inv in result["invoices"]: if not self._looks_like_address(str(inv.get("suppliers_address") or "")): # Re-scan the first invoice block with a very strict # boundary. This is still semantic and document-relative. m = re.search( r"SUPPLIER\s+DETAILS.*?\bAddress\s*:\s*(.+?)(?=\s+IF\s+SUPPLIER\s+IS\s+NOT\s+THE\s+SELLER|\s+BROKER/\s*AGENT\s+DETAILS)", normalized, re.I | re.S, ) if m: addr = clean(m.group(1)) if self._looks_like_address(addr): inv["suppliers_address"] = addr def _final_visual_section_override(self, result): """Ultra-conservative final override using the PDF's visual rows. This pass exists specifically for PDFs whose text-flow order differs from the visual order. It uses semantic labels to find the relevant row, then uses relative x/y relationships from that same uploaded PDF. No page number, absolute coordinate, or document value is hardcoded. """ def clean(v): v = self._clean(v) v = re.sub(r"(?<=[A-Za-z])-\s+(?=[A-Za-z])", "", v) return re.sub(r"^[\s:;]+|[\s:;]+$", "", v) def row_words_after(row, x0, x1=None): out=[] for w in row.get("words",[]): if w["x0"] < x0 - 2: continue if x1 is not None and w["x0"] >= x1: break out.append(w["_text"]) return out def visual_column_value(page_no, anchor_row_index, anchor_word_x, stop_tokens, max_rows=20): rows=self.page_rows.get(page_no,[]) collected=[] started=False for row in rows[anchor_row_index:anchor_row_index+max_rows]: txt=self._clean(row.get("text")) if not txt: continue if started and any(re.search(p, txt, re.I) for p in stop_tokens): break selected=[] for w in row.get("words",[]): if w["x0"] >= anchor_word_x - 4: selected.append(w["_text"]) if selected: collected.extend(selected) started=True return clean(" ".join(collected)) # ------------------------------------------------------------- # IMPORTER ADDRESS: locate the visual Address label within the # PARTICULARS OF THE IMPORTER section and follow its column downward. # ------------------------------------------------------------- importer_page=None importer_y=None for page_no, rows in self.page_rows.items(): for i,row in enumerate(rows): if re.search(r"PARTICULARS\s+OF\s+THE\s+IMPORTER", row["text"], re.I): importer_page=page_no; importer_y=row["_y"]; break if importer_page is not None: break if importer_page is not None: rows=self.page_rows[importer_page] for i,row in enumerate(rows): if row["_y"] < importer_y: continue if "Address:" not in row["text"] or "Name:" not in row["text"]: continue address_word=None for w in row.get("words",[]): if w["_text"].lower().startswith("address"): address_word=w; break if not address_word: continue address_start = address_word["x1"] + 4 for cw in row.get("words",[]): if cw["_text"] == ":" and cw["x0"] >= address_word["x1"]: address_start = cw["x1"] + 4 break address=visual_column_value( importer_page, i, address_start, [r"Category\s+Of\s+Importer",r"In\s+Case\s+Of\s+other\s+Importer",r"Authorised\s+Dealer",r"BOE\s+Number"], max_rows=18, ) # The first visual row contains the value after Address. The # following rows are the wrapped address column only. if address and self._looks_like_address(address): result["cour_be_importer_address"]=address break # ------------------------------------------------------------- # SUPPLIER ADDRESS: find the first Name+Address row after a # SUPPLIER DETAILS heading, including when that heading is at the # bottom of the preceding page and the values begin on the next page. # ------------------------------------------------------------- for inv in result.get("invoices") or []: best=None supplier_heading_seen=False for page_no in sorted(self.page_rows): rows=self.page_rows[page_no] for i,row in enumerate(rows): if "SUPPLIER DETAILS" in row["text"].upper(): supplier_heading_seen=True continue if not supplier_heading_seen: continue if not re.search(r"Name\s*:", row["text"]) or not re.search(r"Address\s*:", row["text"]): continue aw=None for w in row.get("words",[]): if w["_text"].lower().startswith("address"): aw=w; break if not aw: continue address_start=aw["x1"]+4 for cw in row.get("words",[]): if cw["_text"]==":" and cw["x0"]>=aw["x1"]: address_start=cw["x1"]+4 break addr=visual_column_value( page_no,i,address_start, [r"IF\s+SUPPLIER\s+IS\s+NOT\s+THE\s+SELLER",r"BROKER/\s*AGENT\s+DETAILS",r"Nature\s+of\s+Transaction"], max_rows=18, ) if addr and self._looks_like_address(addr): best=addr break if best: break if best: inv["suppliers_address"]=best # ------------------------------------------------------------- # Explicit item sections. The section heading is the business-row # identity. Visual rows provide the safest reconstruction for wrapped # manufacturer names; label regexes handle the remaining scalar fields. # ------------------------------------------------------------- text=self.full_text.replace("\u00ad","") starts=list(re.finditer(r"Details\s+Of\s+Item\s*[-:]\s*(\d+)",text,re.I)) for pos,st in enumerate(starts): item_no=int(st.group(1)) end=starts[pos+1].start() if pos+1=label_end] if not cands: continue value_start=cands[0]["x1"]+4 value_end=bw["x0"]-4 if bw else None parts=[] # Same row parts.extend(row_words_after(row,value_start,value_end)) # Following visual rows in the same left value column. if parts: for rr in rows[i+1:i+4]: if any(re.search(p,rr["text"],re.I) for p in [r"Model\s*:",r"Specification\s*:",r"Country\s*of\s*Origin\s*:"]): break vals=[] for w in rr.get("words",[]): if w["x0"]>=value_start and (value_end is None or w["x0"]=aw["x1"]: start=cw["x1"]+4; break parts=[] for rr in rows[i:i+20]: txt=self._clean(rr.get("text")) if parts and re.search(r"IF\s+SUPPLIER\s+IS\s+NOT\s+THE\s+SELLER|BROKER/\s*AGENT\s+DETAILS|Nature\s+of\s+Transaction",txt,re.I): break vals=[w["_text"] for w in rr.get("words",[]) if w["x0"]>=start-2] if vals: parts.extend(vals) visual_address=clean(" ".join(parts)) # If the address line visibly continues at a page boundary # (typically a trailing hyphen/comma), follow the same right- # hand value column on the next page until the next semantic # invoice section. This avoids retaining ``Page N of M`` while # still supporting multi-page supplier addresses. if visual_address and (re.search(r"[-,]$", visual_address) or re.search(r"[-–—]$", " ".join(parts))): page_numbers=sorted(self.page_rows) try: pidx=page_numbers.index(page_no) except ValueError: pidx=-1 if pidx >= 0: for next_page_no in page_numbers[pidx+1:pidx+3]: continued=[] stop=False for rr in self.page_rows.get(next_page_no,[]): txt=self._clean(rr.get("text")) if not txt or re.search(r"Page\s+\d+\s+of\s+\d+",txt,re.I): continue if re.search(r"IF\s+SUPPLIER|BROKER/\s*AGENT\s+DETAILS|Nature\s+of\s+Transaction|Terms\s+of\s+Payment|Method\s+of\s+Valuation",txt,re.I): stop=True break vals=[w["_text"] for w in rr.get("words",[]) if w["x0"]>=start-2] if vals: continued.extend(vals) # A postal code is a strong natural end of # an address continuation. Do not consume # repeated country/city fragments after it. if re.search(r"\b\d{5,6}\b", " ".join(continued)): break if continued: visual_address=clean(visual_address+" "+" ".join(continued)) if stop: break break if visual_address: break if visual_address and self._looks_like_address(visual_address): for inv in result.get("invoices") or []: current = clean(inv.get("suppliers_address")) # Never replace a stronger cross-page supplier address with the # shorter single-page observation. The visual value is used only # when it is actually stronger or the current value is empty. if not current or len(visual_address) >= len(current): inv["suppliers_address"] = visual_address def _final_dynamic_invoice_manufacturer_recovery(self, result): """Last conservative recovery pass for invoice/manufacturer fields. This pass is deliberately layout-relative. It does not use page numbers, fixed coordinates, or document-specific manufacturer names. It repairs an empty/partial invoice row by following semantic labels and reconstructs wrapped manufacturer text from the visual rows surrounding the Manufacturer label. """ def empty(v): return v in ("", None, 0, 0.0) def clean(v): v = self._clean(v) v = re.sub(r"^[\s:;\-]+|[\s:;\-]+$", "", v) return self._repair_wrapped_words(v) def value_from_match(pattern, block, flags=re.I | re.S): m = re.search(pattern, block, flags) return clean(m.group(1)) if m else "" # -------------------------------------------------------------- # 1. Find the strongest invoice block dynamically. # -------------------------------------------------------------- invoice_blocks = [] headings = re.finditer( r"(?:Details\s+Of\s+Invoice(?:\s*[-:]\s*\d+)?|" r"Invoice\s+Details|Invoice\s+Information)", self.full_text, re.I, ) for h in headings: tail = self.full_text[h.end():] stop = re.search( r"(?:Details\s+Of\s+Item|Details\s+Of\s+Items|" r"Item\s+Details|Duty\s+Details|Payment\s+Details|" r"Declaration|End\s+Of\s+Invoice)", tail, re.I, ) invoice_blocks.append(tail[:stop.start()] if stop else tail[:7000]) # If no heading survives text extraction, build a local semantic # block around Invoice Number. if not invoice_blocks: for m in re.finditer( r"Invoice\s*(?:Number|No\.?)", self.full_text, re.I ): left = max(0, m.start() - 500) tail = self.full_text[m.start():] stop = re.search( r"(?:Details\s+Of\s+Item|Item\s+Details|Duty\s+Details|" r"Payment\s+Details|Declaration)", tail, re.I, ) invoice_blocks.append( self.full_text[left:m.start()] + tail[:stop.start() if stop else 7000] ) # Score blocks by the number of meaningful invoice labels they contain. def block_score(b): labels = ( r"Invoice\s*(?:Number|No\.?)", r"Date\s+of\s+Invoice|Invoice\s+Date", r"Invoice\s+Value|Value\s+of\s+Invoice", r"Currency", r"Incoterm|Terms\s+of\s+Invoice", r"Nature\s+of\s+Transaction", r"Terms\s+of\s+Payment", r"Method\s+of\s+Valuation", r"Provisional\s*/?\s*Final", ) return sum(bool(re.search(x, b, re.I)) for x in labels) invoice_blocks.sort(key=block_score, reverse=True) # Use an existing row if it is merely an empty schema row. invoices = result.setdefault("invoices", []) target = None for row in invoices: if ( empty(row.get("cour_be_invoice_no")) and empty(row.get("cour_be_inv_value")) ): target = row break if target is None and invoice_blocks: target = self._empty_invoice(1) invoices.append(target) if target is not None and invoice_blocks: block = invoice_blocks[0] patterns = { "cour_be_invoice_no": ( r"(?:Invoice\s*(?:Number|No\.?)|Inv\s*(?:Number|No\.?))" r"\s*[:#\-]?\s*([A-Z0-9][A-Z0-9./_-]{2,})" ), "cour_be_invoice_date": ( r"(?:Date\s+of\s+Invoice|Invoice\s+Date|Inv\s+Date)" r"\s*[:\-]?\s*(\d{1,2}[/-]\d{1,2}[/-]\d{2,4})" ), "cour_be_inv_value": ( r"(?:Invoice\s+Value|Value\s+of\s+Invoice|Invoice\s+Amount)" r"\s*[:\-]?\s*([0-9][\d,]*(?:\.\d+)?)" ), "cour_be_inv_currency": ( r"(?:Currency\s+of\s+Invoice|Invoice\s+Currency|Currency)" r"\s*[:\-]?\s*([A-Z]{3})\b" ), "cour_be_incoterm": ( r"(?:Incoterm|Incoterms|Terms\s+of\s+Invoice|" r"Terms\s+of\s+Delivery)" r"\s*[:\-]?\s*([A-Z]{3})\b" ), "cour_be_inv_nature_of_txn": ( r"(?:Nature\s+of\s+Transaction|Nature\s+of\s+Txn)" r"\s*[:\-]?\s*([A-Za-z][A-Za-z /&-]{1,40}?)(?=\s+" r"(?:Terms|Valuation|Provisional|Related)\b|$)" ), "cour_be_inv_pymt_terms": ( r"(?:Terms\s+of\s+Payment|Payment\s+Terms)" r"\s*[:\-]?\s*([A-Za-z][A-Za-z /&-]{1,40}?)(?=\s+" r"(?:Method|Valuation|Provisional|Related)\b|$)" ), "cour_be_valuationmethod": ( r"(?:Method\s+of\s+Valuation|Valuation\s+Method)" r"\s*[:\-]?\s*(Rule\s*\d+|[A-Za-z][A-Za-z0-9 /&-]{1,40}?)" r"(?=\s+(?:Provisional|Final|Related|Terms)\b|$)" ), "cour_be_prov_final": ( r"(?:Provisional\s*/?\s*Final|Assessment\s+Type)" r"\s*[:\-]?\s*(FINAL|PROVISIONAL)\b" ), "cour_be_inv_related": ( r"(?:Are\s+the\s+Buyer\s+and\s+Seller\s+Related|" r"Buyer\s+and\s+Seller\s+Related)" r"\s*[:\-]?\s*([YN])\b" ), } for field, pattern in patterns.items(): if not empty(target.get(field)): continue val = value_from_match(pattern, block) if not val: # Retry against the full document only for uniquely # shaped values. This avoids borrowing values from # unrelated pages. if field in { "cour_be_invoice_no", "cour_be_invoice_date", "cour_be_inv_currency", "cour_be_incoterm", "cour_be_prov_final", }: val = value_from_match(pattern, self.full_text) if not val: continue if field == "cour_be_inv_value": target[field] = float(self._float(val) or 0.0) elif field == "cour_be_inv_related": target[field] = val.upper() if val.upper() in {"Y", "N"} else "" elif field == "cour_be_invoice_date": target[field] = self._format_date(val) elif field == "cour_be_inv_pymt_terms": # Never accept a monetary token as payment terms. if self._float(val) is None: target[field] = val else: target[field] = val # Supplier name/address: use semantic labels inside the invoice # block and collect only the value region before the next label. if empty(target.get("cour_be_inv_supplier_name")): m = re.search( r"(?:Name\s+of\s+Consignor|Supplier\s+Name|" r"Name\s+of\s+Supplier|Consignor)" r"\s*[:\-]?\s*(.+?)(?=\s+(?:Address|Invoice|Date|" r"Value|Currency|Incoterm|Terms)\s*(?:of)?\b|$)", block, re.I | re.S, ) if m: val = clean(m.group(1)) if ( val and not self._looks_like_any_known_label(val) and not re.search( r"(?i)\b(?:Address|Invoice|Date|Value|Currency)\s*(?:of)?\s*:", val, ) ): target["cour_be_inv_supplier_name"] = val if empty(target.get("suppliers_address")): m = re.search( r"(?:Address\s+of\s+Consignor|Supplier\s+Address|" r"Address\s+of\s+Supplier|Exporter\s+Address|Seller\s+Address)" r"\s*[:\-]?\s*(.+?)(?=\s+(?:Invoice\s+Number|Invoice\s+Date|" r"Invoice\s+Value|Currency|Incoterm|Nature\s+of|" r"Terms\s+of|Method\s+of\s+Valuation|Provisional)\b|$)", block, re.I | re.S, ) if m: val = clean(m.group(1)) if val and not self._looks_like_any_known_label(val): target["suppliers_address"] = val # -------------------------------------------------------------- # 2. Manufacturer recovery from visual label context. # -------------------------------------------------------------- # Work on every item row. A row may already contain a truncated # manufacturer; in that case recover its continuation rather than # replacing it with a document-specific constant. for item in result.get("items") or []: current = clean(item.get("cour_be_item_manufacturer")) if current: current = re.sub( r"(?i)^(?:manufacturer(?:\s+name)?|name\s+of\s+manufacturer)" r"\s*:\s*", "", current, ) current = re.sub( r"(?i)\s+(?:Brand|Model|Grade|Specification|End\s+Use\s+of\s+Item)" r"\s*:\s*.*$", "", current, ).strip() candidates = [] for label in self._find_labels(self.ITEM_ALIASES["cour_be_item_manufacturer"]): page = label["_page"] rows = self.page_rows.get(page, []) try: idx = next(i for i, r in enumerate(rows) if r is label["row"]) except StopIteration: idx = min( range(len(rows)), key=lambda i: abs(rows[i]["_y"] - label["_y"]) ) if rows else -1 if idx < 0: continue pieces = [] # Same row: only words after the label. for w in label["row"]["words"]: if w["x0"] > label["x1"] + 1: pieces.append(w["_text"]) # Following visual rows are continuation candidates. Stop # at metadata labels/section headings, not at an arbitrary # number of characters. for r in rows[idx + 1:idx + 5]: rt = self._clean(r["text"]) compact = self._compact(rt) if not rt: continue if self._is_section_or_major_label(rt): break if re.match( r"(?i)^(?:Brand|Model|Grade|Specification|End\s+Use\s+of\s+Item|" r"Quantity|UOM|Country\s*of\s*Origin)\b", rt, ): break pieces.extend(w["_text"] for w in r["words"]) candidate = clean(" ".join(pieces)) candidate = re.sub( r"(?i)\s+(?:Brand|Model|Grade|Specification|End\s+Use\s+of\s+Item)" r"\s*:.*$", "", candidate, ).strip() if candidate and not self._looks_like_any_known_label(candidate): candidates.append(candidate) # Prefer a visual candidate that contains more information than # the current truncated value. if candidates: best = max(candidates, key=len) if not current or len(best) > len(current): current = best # Generic word-continuation recovery: if the extracted name ends # at a short alphabetic fragment, inspect adjacent visual rows for # a token that naturally continues it. No manufacturer dictionary # is required. if current: current = re.sub( r"(?i)\s+(?:Brand|Model|Grade|Specification|End\s+Use\s+of\s+Item)" r"\s*:.*$", "", current, ).strip() # Join explicit PDF word-wraps, e.g. "CON-" + "DUCTA INC". current = re.sub( r"(?i)([A-Za-z]{2,})-\s+([A-Za-z]{2,})", r"\1\2", current, ) item["cour_be_item_manufacturer"] = current # -------------------------------------------------------------- # 3. Reconcile invoice count from actual rows. # -------------------------------------------------------------- valid_invoices = [] for inv in result.get("invoices") or []: if any( not empty(inv.get(f)) for f in ( "cour_be_invoice_no", "cour_be_invoice_date", "cour_be_inv_value", "cour_be_inv_supplier_name", ) ): valid_invoices.append(inv) if valid_invoices: result["invoices"] = valid_invoices result["cour_be_invoicecount"] = len(valid_invoices) def _v183_future_layout_recovery(self, result): """Conservative future-layout recovery layered on top of v18.3. v18.3 remains the primary extractor. This method only: 1. recovers an invoice when v18.3 could not create one; 2. replaces obviously contaminated parent values when a stronger semantic candidate is available; 3. removes layout labels from manufacturer values and reconstructs wrapped manufacturer names from nearby visual text. It intentionally avoids page numbers and fixed coordinates. """ text = self.full_text.replace("\u00ad", "") text = re.sub(r"[ \t]+", " ", text) def clean(value): return self._clean(str(value or "").strip(" \t\r\n:;,")) def is_empty(value): return value in ("", None, 0, 0.0) def bad_parent_candidate(field, value): v = clean(value) if not v: return True c = re.sub(r"[^a-z0-9]+", "", v.lower()) # Explicit form/layout phrases which are never business values. bad_phrases = ( "incaseofotherimporter", "othertypeof", "dateofarrival", "totalamountchallandate", "countryofconsignment", "nameof", "typeofboe", ) if any(p in c for p in bad_phrases): return True if field == "cour_be_arrivalairport": return not bool(re.fullmatch(r"[A-Za-z0-9]{2,8}", v)) if field == "cour_be_coo": return len(v.split()) > 8 if field in {"cour_be_iecbranch", "cour_be_igmno"}: return not bool(re.fullmatch(r"[A-Za-z0-9]{1,20}", v)) if field == "cour_be_tr6_no": return not bool(re.fullmatch(r"[A-Za-z0-9/-]{6,30}", v)) return False # -------------------------------------------------------------- # 1. Recover an invoice from any semantic invoice heading. # -------------------------------------------------------------- if not result.get("invoices"): starts = list(re.finditer( r"(?:Details\s+Of\s+Invoice(?:\s*-\s*\d+)?|" r"Invoice\s+Details|Invoice\s+Information)", text, re.I )) # A PDF may have no heading in the extracted text. In that case # use the first occurrence of the semantic "Invoice Number". if not starts: m = re.search(r"Invoice\s*(?:Number|No\.?)\s*[:\-]", text, re.I) if m: starts = [m] if starts: start = starts[0].start() remainder = text[start + 1:] # Bound the block at the next logical section, not a page. stop = re.search( r"(?:Details\s+Of\s+Item|Item\s+Details|" r"Payment\s+Details|Duty\s+Details|Declaration|" r"Out\s+Of\s+Charge|Shipping\s+Bill\s+Details)", remainder, re.I ) block = remainder[:stop.start()] if stop else remainder[:6000] inv = self._empty_invoice(1) patterns = { "cour_be_invoice_no": ( r"(?:Invoice\s*(?:Number|No\.?)|Inv\s*(?:Number|No\.?))" r"\s*[:\-]?\s*([A-Z0-9][A-Z0-9./_-]{2,})" ), "cour_be_invoice_date": ( r"(?:Date\s+of\s+Invoice|Invoice\s+Date)" r"\s*[:\-]?\s*(\d{1,2}[/-]\d{1,2}[/-]\d{2,4})" ), "cour_be_inv_value": ( r"(?:Invoice\s+Value|Value\s+of\s+Invoice|Invoice\s+Amount)" r"\s*[:\-]?\s*([0-9][\d,]*(?:\.\d+)?)" ), "cour_be_inv_currency": ( r"(?:Invoice\s+Currency|Currency)" r"\s*[:\-]?\s*([A-Z]{3})\b" ), "cour_be_incoterm": ( r"(?:Incoterm|Terms\s+of\s+Invoice|Terms\s+of\s+Delivery)" r"\s*[:\-]?\s*([A-Z]{3})\b" ), "cour_be_inv_nature_of_txn": ( r"(?:Nature\s+of\s+Transaction|Nature\s+of\s+Txn)" r"\s*[:\-]?\s*([A-Za-z][A-Za-z ]{1,35})" r"(?=\s+(?:If\s+Others|Terms\s+of\s+Payment|" r"Method\s+of\s+Valuation|Conditions)\b|$)" ), "cour_be_inv_pymt_terms": ( r"(?:Terms\s+of\s+Payment|Payment\s+Terms)" r"\s*[:\-]?\s*([A-Za-z][A-Za-z ]{1,35})" r"(?=\s+(?:If\s+Others|Conditions|Method\s+of\s+Valuation)\b|$)" ), "cour_be_valuationmethod": ( r"(?:Method\s+of\s+Valuation|Valuation\s+Method)" r"\s*[:\-]?\s*(Rule\s*\d+|[A-Za-z][A-Za-z0-9 ]{1,30})" r"(?=\s+(?:Terms\s+of\s+Invoice|Invoice\s+Value)\b|$)" ), "cour_be_prov_final": ( r"(?:Indication\s+for\s+Provisional\s*/\s*Final|" r"Provisional\s*/?\s*Final|Assessment\s+Type)" r"\s*[:\-]?\s*(FINAL|PROVISIONAL)\b" ), "cour_be_inv_related": ( r"(?:Are\s+the\s+Buyer\s+and\s+Seller\s+Related|" r"Buyer\s+and\s+Seller\s+Related)" r".{0,80}?\b([YN])\b" ), } for field, pattern in patterns.items(): m = re.search(pattern, block, re.I | re.S) if not m: continue value = clean(m.group(1)) if field == "cour_be_invoice_date": value = self._format_date(value) elif field == "cour_be_inv_value": value = float(self._float(value) or 0.0) if value not in ("", None): inv[field] = value # Supplier name/address are intentionally bounded by the # next semantic label, so legal names may wrap freely. supplier = re.search( r"(?:SUPPLIER\s+DETAILS\s+)?" r"(?:Name\s+of\s+Supplier|Supplier\s+Name|Supplier|" r"Consignor|Seller)\s*[:\-]?\s*(.+?)" r"(?=\s+(?:Address|Supplier\s+Address|Invoice\s+Number|" r"Date\s+of\s+Invoice|Invoice\s+Value)\s*:)", block, re.I | re.S ) if supplier: inv["cour_be_inv_supplier_name"] = clean(supplier.group(1)) address = re.search( r"(?:Supplier\s+Address|Address\s+of\s+Supplier|" r"Consignor\s+Address|Seller\s+Address)\s*[:\-]?\s*(.+?)" r"(?=\s+(?:IF\s+SUPPLIER|Invoice\s+Number|" r"Date\s+of\s+Invoice|Nature\s+of\s+Transaction|" r"Terms\s+of\s+Payment|Method\s+of\s+Valuation)\b)", block, re.I | re.S ) if address: inv["suppliers_address"] = clean(address.group(1)) if ( inv.get("cour_be_invoice_no") or inv.get("cour_be_invoice_date") or float(inv.get("cour_be_inv_value") or 0) > 0 ): result["invoices"] = [inv] # -------------------------------------------------------------- # 2. Recover contaminated parent values only. Strong v18.3 values # are left untouched. # -------------------------------------------------------------- parent_aliases = getattr(self, "PARENT_ALIASES", {}) for field in ( "cour_be_importername", "cour_be_arrivalairport", "cour_be_airline", "cour_be_flightno", "cour_be_coo", "cour_be_countryofconsignment", "cour_be_iecbranch", "cour_be_tr6_no", ): current = result.get(field) if not bad_parent_candidate(field, current): continue aliases = parent_aliases.get(field, []) if not aliases: continue # First use the same semantic engine that made v18.3 accurate. validator = self._parent_validator(field) value, label = self._semantic_value( aliases, validator, max_dx=300, max_dy=150 ) # If semantic search does not find a candidate, use the existing # regex-label fallback. Never accept a known label as the value. if not value: value = self._regex_label_value(aliases, validator) if value and not bad_parent_candidate(field, value): result[field] = self._normalize_parent(field, value) self._trace(field, result[field], "v18.3-future-recovery") # -------------------------------------------------------------- # 3. Manufacturer cleanup/recovery. # # Important: no manufacturer-name dictionary is used. We remove # layout metadata and then, if the value is visibly truncated, # collect the continuation from the same item's visual neighborhood. # -------------------------------------------------------------- manufacturer_tail = re.compile( r"\s+(?:Brand|Model|Grade|Specification|End\s+Use\s+of\s+Item|" r"End\s+Use|Country\s*of\s*Origin|RITC|CTSH|CETSH)\s*:", re.I ) manufacturer_prefix = re.compile( r"^(?:Name\s+of\s+Manufacturer|Manufacturer\s+Name|" r"Manufacturer|Brand)\s*:\s*", re.I ) for item in result.get("items", []): raw = clean(item.get("cour_be_item_manufacturer")) if not raw: continue # Remove leading labels and everything belonging to the next # labelled field. value = manufacturer_prefix.sub("", raw).strip() value = manufacturer_tail.split(value, maxsplit=1)[0].strip() value = clean(value) # If the current value is short/truncated, find a manufacturer # labelled visual block and collect up to four continuation rows. if value: prefix_compact = re.sub(r"[^a-z0-9]+", "", value.lower()) if len(prefix_compact) >= 8 and len(value.split()) <= 6: candidates = [] for page_no, rows in self.page_rows.items(): for i, row in enumerate(rows): row_text = self._clean( row.get("_text") or row.get("text") or "" ) if not re.search( r"\b(?:Name\s+of\s+)?Manufacturer\b|\bBrand\b", row_text, re.I ): continue parts = [] for offset in range(4): pos = i + offset if pos >= len(rows): break part = self._clean( rows[pos].get("_text") or rows[pos].get("text") or "" ) if not part: continue if offset == 0: mm = re.search( r"(?:Name\s+of\s+)?Manufacturer|Brand" r"\s*:?\s*(.*)$", part, re.I ) if mm: part = self._clean(mm.group(1)) else: continue if re.search( r"^(?:Brand|Model|Grade|Specification|" r"End\s+Use|Country\s*of\s*Origin|" r"RITC|CTSH|CETSH)\s*:", part, re.I ): break parts.append(part) candidate = clean(" ".join(parts)) candidate = manufacturer_tail.split( candidate, maxsplit=1 )[0].strip() if candidate: cc = re.sub( r"[^a-z0-9]+", "", candidate.lower() ) if cc.startswith(prefix_compact[:min(12, len(prefix_compact))]): candidates.append(candidate) if candidates: longest = max(candidates, key=lambda x: len( re.sub(r"[^a-z0-9]+", "", x) )) if len(longest) > len(value): value = longest item["cour_be_item_manufacturer"] = value # -------------------------------------------------------------- # 4. Remove label/prefix leakage from the common item fields. # This is deliberately narrow so existing v18.3 values are not # reformatted unnecessarily. # -------------------------------------------------------------- item_prefixes = { "cour_be_item_itemdescription": ( r"^(?:Item\s+Description|Description\s+of\s+Item|" r"Description)\s*:\s*" ), "cour_be_item_itemgendescription": ( r"^(?:General\s+Description|Item\s+General\s+Description)\s*:\s*" ), "cour_be_item_uom": ( r"^(?:Unit\s+of\s+Measure|UOM|Unit)\s*:\s*" ), } for item in result.get("items", []): for field, pattern in item_prefixes.items(): value = clean(item.get(field)) if value: item[field] = clean(re.sub(pattern, "", value, flags=re.I)) # Invoice count must reflect actual logical invoice rows. result["cour_be_invoicecount"] = len(result.get("invoices") or []) def _repair_semantic_courier_sections(self, result): """Final semantic recovery for Courier XIV/XIII style sections. The method intentionally relies on labels and bounded text blocks, never absolute PDF coordinates. It is therefore useful when a new Courier form moves the same labelled sections to another page or changes line wrapping. """ text = self.full_text.replace("\u00ad", "") text = re.sub(r"[ \t]+", " ", text) # ------------------------------------------------------------- # IEC branch: the value may be on the following line. Accept 0/43 # and reject the literal label word "Code". # ------------------------------------------------------------- branch_patterns = [ r"Import\s+Export\s+Branch\s*Code\s*:\s*([A-Za-z0-9]{1,8})", r"IEC\s+Branch\s+Code\s*:\s*([A-Za-z0-9]{1,8})", r"Import\s+Export\s+Branch\s*\n?\s*Code\s*:\s*([A-Za-z0-9]{1,8})", ] for pat in branch_patterns: m = re.search(pat, text, re.I | re.S) if m: value = self._clean(m.group(1)) if value.upper() != "CODE" and not re.search(r"\d{5,}", value): result["cour_be_iecbranch"] = value break # ------------------------------------------------------------- # Payment table: identify the complete row by its structure rather # than searching for arbitrary numbers near "Total Amount". # ------------------------------------------------------------- payment_patterns = [ r"PAYMENT\s+DETAILS.*?TR\s*-?\s*6\s+Challan\s+Number\s+Total\s+Amount\s+Challan\s+Date\s+(?:1\s+)?(\d{7,20})\s+([0-9][\d,]*(?:\.\d+)?)\s+(\d{1,2}[/-]\d{1,2}[/-]\d{2,4})", r"PAYMENT\s+DETAILS.*?\b(\d{7,20})\s+([0-9][\d,]*(?:\.\d+)?)\s+(\d{1,2}[/-]\d{1,2}[/-]\d{2,4})", ] payment_candidates = [] for pat in payment_patterns: for m in re.finditer(pat, text, re.I | re.S): challan, amount, date = m.group(1), m.group(2), m.group(3) if challan == amount: continue payment_candidates.append((len(m.group(0)), challan, amount, date)) if payment_candidates: _, challan, amount, date = max(payment_candidates, key=lambda x: x[0]) result["cour_be_tr6_no"] = challan result["cour_be_pymt_amt"] = float(self._float(amount) or 0.0) result["cour_be_total_duty"] = float(self._float(amount) or 0.0) result["cour_be_challan_date"] = self._format_date(date) # ------------------------------------------------------------- # Invoice blocks. A block starts at "DETAILS OF INVOICE(S)" or # "Details Of Invoice - N" and ends at the next invoice/item section. # This handles line wrapping without assuming a page number. # ------------------------------------------------------------- invoice_starts = list(re.finditer( r"Details\s+Of\s+Invoice\s*-\s*(\d+)", text, re.I )) invoice_rows = [] for idx, start in enumerate(invoice_starts): end = invoice_starts[idx + 1].start() if idx + 1 < len(invoice_starts) else len(text) block = text[start.start():end] item_pos = re.search(r"DETAILS\s+OF\s+ITEM(?:\(S\))?\s+IN\s+INVOICE", block, re.I) if item_pos: block = block[:item_pos.start()] inv = self._empty_invoice(int(start.group(1))) self._set_invoice_regex(inv, "cour_be_invoice_no", r"Invoice\s+Number\s*:\s*(.+?)(?=\s+Date\s+of\s+Invoice\s*:)", block) self._set_invoice_regex(inv, "cour_be_invoice_date", r"Date\s+of\s+Invoice\s*:\s*(\d{1,2}[/-]\d{1,2}[/-]\d{2,4})", block, date=True) self._set_invoice_regex(inv, "cour_be_inv_supplier_name", r"SUPPLIER\s+DETAILS\s+Name\s*:\s*(.+?)(?=\s+Address\s*:)", block) self._set_invoice_regex(inv, "suppliers_address", r"SUPPLIER\s+DETAILS\s+Name\s*:.*?Address\s*:\s*(.+?)(?=\s+IF\s+SUPPLIER\s+IS\s+NOT\s+THE\s+SELLER)", block) self._set_invoice_regex(inv, "cour_be_inv_nature_of_txn", r"Nature\s+of\s+Transaction\s*:\s*([^\n]+?)(?=\s+If\s+Others\s*:|\s+Terms\s+of\s+Payment)", block) self._set_invoice_regex(inv, "cour_be_inv_pymt_terms", r"Terms\s+of\s+Payment\s*:\s*([^\n]+?)(?=\s+If\s+Others\s*:|\s+Conditions\s+or\s+Restrictions)", block) self._set_invoice_regex(inv, "cour_be_valuationmethod", r"Method\s+of\s+Valuation\s*:\s*([^\n]+?)(?=\s+Terms\s+of\s+Invoice)", block) self._set_invoice_regex(inv, "cour_be_incoterm", r"Terms\s+of\s+Invoice\s*:\s*([A-Za-z0-9 ._-]+?)(?=\s+Invoice\s+Value)", block) self._set_invoice_regex(inv, "cour_be_inv_value", r"Invoice\s+Value\s*:\s*([0-9][\d,]*(?:\.\d+)?)", block, number=True) self._set_invoice_regex(inv, "cour_be_inv_currency", r"Invoice\s+Value\s*:\s*[0-9][\d,]*(?:\.\d+)?\s+Currency\s*:\s*([A-Za-z]{3,})", block) self._set_invoice_regex(inv, "cour_be_inv_related", r"Are\s+the\s+Buyer\s+and\s+Seller\s+Related\s*\??\s*:\s*([YN])", block) self._set_invoice_regex(inv, "cour_be_prov_final", r"Indication\s+for\s+Provisional\s*/\s*Final\s*:\s*([A-Za-z]+)", block) if inv.get("cour_be_invoice_no") or inv.get("cour_be_inv_value"): invoice_rows.append(inv) if invoice_rows: merged = self._merge_invoice_rows((result.get("invoices") or []) + invoice_rows) result["invoices"] = merged # If the generic pass produced no invoice, recover the first explicit # invoice from the declaration/standard invoice section as a fallback. if not result.get("invoices"): m = re.search( r"DETAILS\s+OF\s+INVOICE\(S\).*?Details\s+Of\s+Invoice\s*-\s*1.*?Invoice\s+Number\s*:\s*(.+?)\s+Date\s+of\s+Invoice\s*:\s*(\d{1,2}[/-]\d{1,2}[/-]\d{2,4})", text, re.I | re.S, ) if m: inv = self._empty_invoice(1) inv["cour_be_invoice_no"] = self._clean(m.group(1)) inv["cour_be_invoice_date"] = self._format_date(m.group(2)) result["invoices"] = [inv] # ------------------------------------------------------------- # Item blocks. Locate every "Details Of Item - N" semantically. # ------------------------------------------------------------- item_starts = list(re.finditer(r"Details\s+Of\s+Item\s*-\s*(\d+)", text, re.I)) repaired_items = [] for idx, start in enumerate(item_starts): end = item_starts[idx + 1].start() if idx + 1 < len(item_starts) else len(text) block = text[start.start():end] # Do not let declarations/footer text bleed into an item. for stop_pat in (r"SHIPPING\s+BILL\s+DETAILS", r"PAYMENTS\s+MADE", r"DETAILS\s+RELATING\s+TO\s+DUTY\s+EXEMPTION", r"PAYMENT\s+DETAILS"): sm = re.search(stop_pat, block, re.I) if sm: block = block[:sm.start()] section_no = int(start.group(1)) item = self._empty_item(section_no) # Preserve the explicit physical section identity through all # reconciliation/deduplication passes. This prevents distinct # item sections that share description/CTSH/manufacturer from # being mistaken for duplicate observations. item["_item_section_no"] = section_no self._set_item_regex(item, "cour_be_item_itemdescription", r"Item\s+Description\s*:\s*(.+?)(?=\s+General\s+Description\s*:)", block) self._set_item_regex( item, "cour_be_item_itemgendescription", r"General\s+Description\s*:\s*(.*?)(?=\s*(?:Currency\s+for\s+Unit\s+Price|Unit\s+Price|Quantity\s*:|Unit\s+of\s+Measure|UOM\s*:|Rate\s+of\s+Exchange|Name\s+of\s+Manufacturer|Country\s*of\s*Origin|RITC\s*:|CTSH\s*:|CETSH\s*:|Assessable\s*Value\s*:|Brand\s*:|Model\s*:|Grade\s*:|Specification\s*:|End\s+Use\s+of\s+Item\s*:|$))", block, ) self._set_item_regex(item, "cour_be_item_upi_currency", r"Currency\s+for\s+Unit\s+Price\s*:\s*([A-Za-z]{3,})", block) self._set_item_regex(item, "cour_be_item_upi", r"Unit\s+Price\s*:\s*([0-9][\d,]*(?:\.\d+)?)", block, number=True) self._set_item_regex(item, "cour_be_item_uom", r"Unit\s+of\s+Measure\s*:\s*([A-Za-z0-9._/-]+)", block) self._set_item_regex(item, "cour_be_item_qty", r"Quantity\s*:\s*([0-9][\d,]*(?:\.\d+)?)", block, number=True) self._set_item_regex(item, "cour_be_item_exchg_rate", r"Rate\s+Of\s+Exchange\s*:\s*([0-9][\d,]*(?:\.\d+)?)", block, number=True) self._set_item_regex(item, "cour_be_item_manufacturer", r"Name\s+of\s+Manufacturer\s*:\s*(.+?)(?=\s+Brand\s*:)", block) self._set_item_regex(item, "cour_be_item_countryoforigin", r"Country\s*of\s*Origin\s*:\s*(.+?)(?=\s+DETAILS\s+IN\s+CASE\s+OF\s+PREVIOUS\s+IMPORTS)", block) self._set_item_regex(item, "cour_be_item_ritc", r"RITC\s*:\s*([^\s]+)", block) self._set_item_regex(item, "cour_be_item_ctsh", r"CTSH\s*:\s*(\d{4,12})", block) self._set_item_regex(item, "cour_be_item_cetsh", r"CETSH\s*:\s*(\d{4,12})", block) self._set_item_regex(item, "cour_be_item_assessable_value", r"Assessable\s*Value\s*:\s*([0-9][\d,]*(?:\.\d+)?)", block, number=True) duty = self._parse_duty_text(block) for head, rate_field, amount_field in [ ("bcd", "cour_be_item_bcd_rt", "cour_be_item_bcd_amt"), ("aidc", "cour_be_item_aidc_rt", "cour_be_item_aidc_amt"), ("sws", "cour_be_item_sws_rt", "cour_be_item_sws_amt"), ("igst", "cour_be_item_igst_rt", "cour_be_item_igst_amt"), ("cmpcess", "cour_be_item_cmpcess_rt", "cour_be_item_cmpcess_amt"), ]: if head in duty: item[rate_field] = float(duty[head][0]) item[amount_field] = float(duty[head][1]) if item.get("cour_be_item_itemdescription") or item.get("cour_be_item_ctsh"): repaired_items.append(item) if repaired_items: result["items"] = self._dedupe_items((result.get("items") or []) + repaired_items) # ------------------------------------------------------------- # Invoice-level SVB recovery. # # SVB information is frequently rendered as a visually separate # block and can be split across lines/columns. Generic field # extraction may therefore miss it or accidentally read the # item-level SVB block (which is commonly N/A/N). Recover it only # from the invoice-level SVB context and, as a bounded fallback, # from the corresponding Details Of Invoice block. # ------------------------------------------------------------- self._recover_invoice_svb_dynamic(result, text) # If the payment row exists, its amount is the authoritative parent # duty/payment total for Courier forms. if result.get("cour_be_pymt_amt") not in (None, "", 0, 0.0): result["cour_be_total_duty"] = float(result["cour_be_pymt_amt"]) result["cour_be_invoicecount"] = len(result.get("invoices") or []) def _recover_invoice_svb_dynamic(self, result, text): """Recover invoice-level SVB fields using semantic section boundaries. This is deliberately independent of page numbers and fixed coordinates. It distinguishes the invoice-level SVB block from repeated item-level SVB blocks, which may contain different Y/N or N/A values. """ invoices = result.get("invoices") or [] if not invoices: return def empty(v): return v in (None, "", 0, 0.0) def clean_value(v): v = self._clean(v or "") v = re.sub(r"\s+", " ", v).strip(" :;-\t\r\n") return self._repair_wrapped_words(v) def valid_ref(v): v = clean_value(v) if not v or v.upper() in {"N/A", "NA", "N", "NONE", "-"}: return False # SVB references are identifiers, not monetary values. Require # at least one alphabetic character or a slash/dash structure. return bool(re.search(r"[A-Za-z]", v) and re.search(r"[A-Za-z0-9/._-]", v)) def valid_date(v): v = clean_value(v) if v.upper() in {"N/A", "NA", "N", "NONE", "-"}: return False return bool(re.fullmatch(r"\d{1,2}[/-]\d{1,2}[/-]\d{2,4}", v)) def value(pattern, block, flags=re.I | re.S): m = re.search(pattern, block, flags) return clean_value(m.group(1)) if m else "" # First preference: explicit invoice-level SVB sections. The next # item-level section is a hard semantic boundary, so item SVB values # cannot contaminate invoice fields. invoice_svb_blocks = [] svb_anchor = re.compile( r"DETAILS\s+OF\s+SPECIAL\s+VALUATION\s+BRANCH\s*" r"\(SVB\)\s+LOADING\s+WHEREVER\s+APPLICABLE\s*" r"\(\s*AT\s+INVOICE\s+LEVEL\s*\)", re.I, ) anchors = list(svb_anchor.finditer(text)) item_svb_anchor = re.compile( r"DETAILS\s+OF\s+SPECIAL\s+VALUATION\s+BRANCH\s*" r"\(SVB\)\s+LOADING\s+WHEREVER\s+APPLICABLE\s*" r"\(\s*AT\s+ITEM\s+LEVEL\s*\)", re.I, ) item_anchors = list(item_svb_anchor.finditer(text)) for i, m in enumerate(anchors): end_candidates = [] if i + 1 < len(anchors): end_candidates.append(anchors[i + 1].start()) # Never cross into the first item-level SVB section after this # invoice-level section. for im in item_anchors: if im.start() > m.end(): end_candidates.append(im.start()) break # Item details are also a natural boundary. nxt_item = re.search(r"DETAILS\s+OF\s+ITEM(?:\(S\))?\s+IN\s+INVOICE", text[m.end():], re.I) if nxt_item: end_candidates.append(m.end() + nxt_item.start()) end = min(end_candidates) if end_candidates else len(text) invoice_svb_blocks.append(text[m.start():end]) # Fallback: derive the SVB portion from each explicit invoice block. if not invoice_svb_blocks: invoice_starts = list(re.finditer(r"Details\s+Of\s+Invoice\s*-\s*(\d+)", text, re.I)) for i, m in enumerate(invoice_starts): end = invoice_starts[i + 1].start() if i + 1 < len(invoice_starts) else len(text) block = text[m.start():end] # Restrict to invoice-level content, before item details. item_pos = re.search(r"DETAILS\s+OF\s+ITEM(?:\(S\))?\s+IN\s+INVOICE", block, re.I) if item_pos: block = block[:item_pos.start()] if re.search(r"SPECIAL\s+VALUATION\s+BRANCH.*?SVB", block, re.I | re.S): invoice_svb_blocks.append(block) if not invoice_svb_blocks: return # Use the strongest invoice-level block for a single invoice. For # multiple invoices, consume blocks in document order. for idx, inv in enumerate(invoices): block = invoice_svb_blocks[idx] if idx < len(invoice_svb_blocks) else invoice_svb_blocks[0] # SVB Reference Number: allow slash, dash, dot and alphanumeric # identifiers; do not accept N/A/N as a real reference. if empty(inv.get("cour_svb_refno")): ref = value( r"SVB\s+Reference\s+(?:Number|No\.?|#)\s*[:\-]?\s*" r"([A-Z0-9][A-Z0-9/._-]{2,})", block, ) if valid_ref(ref): inv["cour_svb_refno"] = ref # SVB Date may appear immediately after the reference number or # on the next visual line. if empty(inv.get("cour_be_inv_svbdate")): svb_date = value( r"SVB\s+Date\s*[:\-]?\s*(\d{1,2}[/-]\d{1,2}[/-]\d{2,4})", block, ) if valid_date(svb_date): inv["cour_be_inv_svbdate"] = self._format_date(svb_date) # Buyer/Seller relationship: require the complete semantic # question before accepting Y/N. This prevents nearby N/A or # declaration tokens from being assigned to the field. if empty(inv.get("cour_be_inv_related")): related = value( r"Are\s+the\s+Buyer\s+and\s+Seller\s+Related\s*\??\s*" r"[:\-]?\s*([YN])\b", block, ) if related.upper() in {"Y", "N"}: inv["cour_be_inv_related"] = related.upper() # Provisional/final indicator is also tied to the SVB block. if empty(inv.get("cour_be_prov_final")): status = value( r"Indication\s+for\s+Provisional\s*/?\s*Final\s*" r"[:\-]?\s*(FINAL|PROVISIONAL)\b", block, ) if status.upper() in {"FINAL", "PROVISIONAL"}: inv["cour_be_prov_final"] = status.upper() # Last bounded fallback for a document where the invoice-level heading # was damaged by PDF text extraction: only search the complete # invoice declaration region, never the entire document indiscriminately. for inv in invoices: if not any(empty(inv.get(f)) for f in ( "cour_svb_refno", "cour_be_inv_svbdate", "cour_be_inv_related", "cour_be_prov_final", )): continue m = re.search( r"(?:Details\s+Of\s+Invoice\s*-\s*\d+|DETAILS\s+OF\s+INVOICE\(S\))" r".*?(?=DETAILS\s+OF\s+ITEM(?:\(S\))?\s+INVOICE|$)", text, re.I | re.S, ) if not m: continue block = m.group(0) if empty(inv.get("cour_svb_refno")): ref = value( r"SVB\s+Reference\s+(?:Number|No\.?|#)\s*[:\-]?\s*" r"([A-Z0-9][A-Z0-9/._-]{2,})", block, ) if valid_ref(ref): inv["cour_svb_refno"] = ref if empty(inv.get("cour_be_inv_svbdate")): d = value(r"SVB\s+Date\s*[:\-]?\s*(\d{1,2}[/-]\d{1,2}[/-]\d{2,4})", block) if valid_date(d): inv["cour_be_inv_svbdate"] = self._format_date(d) if empty(inv.get("cour_be_inv_related")): rel = value( r"Are\s+the\s+Buyer\s+and\s+Seller\s+Related\s*\??\s*[:\-]?\s*([YN])\b", block, ) if rel.upper() in {"Y", "N"}: inv["cour_be_inv_related"] = rel.upper() if empty(inv.get("cour_be_prov_final")): status = value( r"Indication\s+for\s+Provisional\s*/?\s*Final\s*[:\-]?\s*(FINAL|PROVISIONAL)\b", block, ) if status.upper() in {"FINAL", "PROVISIONAL"}: inv["cour_be_prov_final"] = status.upper() def _harden_sws_amounts_from_duty_rows(self, result): """Keep SWS amount tied to the explicit Duty Amount column. Some forms show SWS Rate=10 while Duty Amount=0. The parser must not invent an amount by multiplying the rate. Conversely, if a generic table pass put a rate into the amount field, recover the explicit amount from that item's own duty section. """ items = result.get("items") or [] for item in items: section_no = item.get("_item_section_no") or item.get("_section_no") if not section_no: continue # Reconstruct the explicit item block from the full text using the # semantic Details Of Item - N boundary. m = re.search( rf"Details\s+Of\s+Item\s*-\s*{re.escape(str(section_no))}\b(.*?)(?=Details\s+Of\s+Item\s*-\s*\d+\b|$)", self.full_text, re.I | re.S, ) if not m: continue block = m.group(0) duty = self._parse_duty_text(block) sws = duty.get("sws") if sws and len(sws) >= 2: try: item["cour_be_item_sws_rt"] = float(sws[0]) item["cour_be_item_sws_amt"] = float(sws[1]) except (TypeError, ValueError): pass def _set_invoice_regex(self, row, field, pattern, block, date=False, number=False): m = re.search(pattern, block, re.I | re.S) if not m: return value = self._clean(m.group(1)) if number: value = float(self._float(value) or 0.0) elif date: value = self._format_date(value) else: value = self._repair_wrapped_words(value) if field == "cour_be_inv_related": value = value.upper() if value not in {"Y", "N"}: return if field == "cour_be_inv_pymt_terms" and self._float(value) is not None: return if value not in ("", None): row[field] = value def _set_item_regex(self, row, field, pattern, block, number=False): m = re.search(pattern, block, re.I | re.S) if not m: return value = self._clean(m.group(1)) if number: value = float(self._float(value) or 0.0) else: value = self._repair_wrapped_words(value) if field == "cour_be_item_manufacturer": value = self._resolve_manufacturer_continuation(value, block) value = self._force_complete_manufacturer_name(value) if field in {"cour_be_item_itemdescription", "cour_be_item_itemgendescription", "cour_be_item_manufacturer", "cour_be_item_countryoforigin"}: if self._looks_like_any_known_label(value): return if value not in ("", None): row[field] = value def _strict_party_name(self, value): v = self._clean(value) if not v or len(v) > 180: return False u = v.upper() if self._looks_like_any_known_label(v): return False bad = ( "ADDRESS", "IMPORTER", "CONSIGNEE", "CONSIGNOR", "SUPPLIER", "COURIER", "AIRPORT", "AIRLINE", "FLIGHT", "NUMBER", "DETAILS", "PARTICULARS", "IF SUPPLIER", "OTHER TYPE", "TYPE OF BOE", "EXPORT CODE", "IMPORT EXPORT BRANCH", "NOTE:", "PAGE ", "COPY", "DECLARATION", "TOTAL DUTY", ) if any(x in u for x in bad): return False # Party names may contain a few digits in legal names, but long # numeric runs are almost always PDF metadata/pincodes/amounts. if re.search(r"\d{5,}", v): return False return bool(re.search(r"[A-Za-z]{2,}", v)) def _strict_airline(self, value): v = self._clean(value) if not self._strict_party_name(v): return False u = v.upper() if any(x in u for x in ("AIRPORT OF", "FLIGHT NO", "NUMBER", "DATE OF", "PAGE")): return False return len(v) <= 100 def _strict_courier(self, value): v = self._clean(value) if not self._strict_party_name(v): return False u = v.upper() if any(x in u for x in ("NAME OF THE", "AUTHORIZED COURIER", "AUTHORISED COURIER")): return False return len(v) <= 100 def _strict_category(self, value): v = self._clean(value) if not v or self._looks_like_any_known_label(v): return False u = re.sub(r"[^A-Z ]", "", v.upper()).strip() return u in { "HOME CONSUMPTION", "WAREHOUSE", "RE EXPORT", "RE-EXPORT", "OTHER", "NORMAL", } def _strict_boe_type(self, value): v = self._clean(value) if not v or re.search(r"CBE\s*-?\s*(XIII|XIV)", v, re.I): return False return v.lower() in {"normal", "home consumption", "warehouse", "re-export", "re export", "other"} def _label_value_candidates(self, aliases, validator, right=500, below=180): """Return scored candidates tied to a concrete label occurrence.""" candidates=[] for label in self._find_labels(aliases): row=label["row"] # Same-row candidate. Stop at the first next label. This is the # critical fix for values such as 'FEDERAL EXPRESS Airport of...'. vals=[] for w in row["words"]: if w["x0"] <= label["x1"] + 1: continue if w["x0"] - label["x1"] > right: break t=self._clean(w["_text"]) if self._is_label(t): break vals.append(t) joined=self._clean(" ".join(vals)) if validator(joined): # Prefer the shortest valid prefix on the same visual row. candidates.append((1050 - len(joined)*0.15, joined, label)) # Below-label candidate. Stop as soon as a new major label starts. center=(label["x0"]+label["x1"])/2 for r in self.page_rows.get(label["_page"],[]): if r["_y"] <= label["y1"]+1: continue dy=r["_y"]-label["y1"] if dy>below: break if self._is_section_or_major_label(r["text"]): break selected=[] for w in r["words"]: wc=(w["x0"]+w["x1"])/2 if abs(wc-center)>220: continue t=self._clean(w["_text"]) if self._is_label(t): continue selected.append(t) val=self._clean(" ".join(selected)) if val and validator(val): candidates.append((980-dy*1.5-len(val)*0.15,val,label)) return sorted(candidates,key=lambda x:x[0],reverse=True) def _best_strict_label_value(self, aliases, validator): c=self._label_value_candidates(aliases,validator) return c[0][1] if c else "" def _harden_xiii_header_and_parties(self, result): """Semantic recovery for condensed CBE-XIII rows.""" for page in self.pages: text=str(page.get("text") or "") lines=text.splitlines() upper=text.upper() if "CBE-XIII" not in upper and "CBEXIII" not in upper: continue # Authorized courier value is printed directly after the label in # many condensed forms and may be followed by Address on the same row. cm=re.search(r"Name\s+of\s+the\s+Authorized\s+(.+?)(?=\s+Address\s+of|\s+Address\s*:|\n|$)",text,re.I) if cm: cv=self._clean(cm.group(1)) cv=re.sub(r"^Courier\s*:\s*", "", cv, flags=re.I) if self._strict_courier(cv): result["cour_be_couriername"]=cv # Header row: Airlines / Flight / Airport / First Port / Date / Time. for i,l in enumerate(lines): if "AIRLINES" in l.upper() and "FLIGHT NO" in l.upper() and "AIRPORT OF ARRIVAL" in l.upper(): data="" for nxt in lines[i+1:i+3]: if self._clean(nxt): data=self._clean(nxt) break # The airline can wrap to the next visual line with a # hyphen. Capture the flight first, then everything before it. m=re.search(r"^(.*?)\s+(?:FIRST\s+PORT\s+OF\s+ARRIVAL\s+)?([A-Z]{1,4}\s*\d{2,6})\s+([A-Z]{3})\s+[^ ]+\s+(\d{1,2}[/-]\d{1,2}[/-]\d{2,4})",data,re.I) if m: raw_airline=self._clean(m.group(1)) airline=raw_airline # If the first line ended with a PDF hyphen, join the # next visual line without inserting a space. if airline.endswith("-") and i+2=4 and self._strict_airline(parts[0]) and self._flight_validator(parts[1]): result["cour_be_airline"]=parts[0] result["cour_be_flightno"]=parts[1].upper() if self._code_validator(parts[2]): result["cour_be_arrivalairport"]=parts[2].upper() if self._date_validator(parts[3]): result["cour_be_arrivaldate"]=self._format_date(parts[3]) break break # ------------------------------------------------------------- # Importer section: Name: Address:
# ------------------------------------------------------------- msec=re.search(r"PARTICULARS\s+OF\s+THE\s+IMPORTER", upper) if msec: start=msec.end() nextsec=re.search(r"PARTICULARS\s+OF\s+THE\s+CUSTOMS\s+HOUSE\s+AGENT|DETAILS\s+OF\s+AUTHORIZED\s+COURIER", upper[start:]) block=text[start:start+(nextsec.start() if nextsec else len(text)-start)] blines=block.splitlines() name_parts=[] started=False for j,l in enumerate(blines): mm=re.search(r"\bName\s*:\s*(.*?)\s+Address\s*:",l,re.I) if mm: val=self._clean(mm.group(1)) if self._strict_party_name(val): name_parts=[val] started=True continue if started: if re.search(r"Category\s+Of\s+Importer|Authorised\s+Dealer|BOE\s+Number",l,re.I): break v=self._clean(l) if v and not self._looks_like_any_known_label(v) and not re.search(r"Address\s*:",v,re.I): # Only accept continuation fragments belonging to # the legal name; address lines contain addresses. # The first continuation row commonly contains # the legal suffix followed by the first address # fragment in a second visual column. Split on the # large whitespace gap and retain only the left # fragment for the party name. pieces=re.split(r"\s{2,}", str(l).strip()) if len(pieces)>1 and len(name_parts)==1: left=self._clean(pieces[0]) if left and not self._looks_like_any_known_label(left): name_parts.append(left) started=False continue if not self._looks_like_address(v): name_parts.append(v) if name_parts: candidate=self._repair_wrapped_words(self._clean(" ".join(name_parts))) if self._strict_party_name(candidate): result["cour_be_importername"]=candidate # IEC / branch live on the row immediately before the Name row. mm=re.search(r"Import\s+export\s+Code\s*:\s*([0-9]{8,12}).*?Import\s+Export\s+Branch\s+([A-Za-z0-9]{1,8})",block,re.I|re.S) if mm: result["cour_be_iec"]=mm.group(1) result["cour_be_iecbranch"]=mm.group(2) # Category is a controlled value, so extract only the text # following its label and stop at the next label. cm=re.search(r"Category\s+Of\s+BOE\s*:\s*([^\n]+)",block,re.I) if cm: val=re.split(r"Type\s+Of\s+BOE\s*:",cm.group(1),flags=re.I)[0] val=self._clean(val) if self._strict_category(val): result["cour_be_category"]=val tm=re.search(r"Type\s+Of\s+BOE\s*:\s*([^\n]+)",block,re.I) if tm: val=self._clean(tm.group(1)) if self._strict_boe_type(val): result["cour_be_betype_description"]=val # ------------------------------------------------------------- # Authorized courier section. The value may be split across # 'M/s UPS Express Private' + next-line 'Courier: Limited'. # ------------------------------------------------------------- csec=re.search(r"DETAILS\s+OF\s+AUTHORIZED\s+COURIER",upper) if csec: cblock=text[csec.end():csec.end()+1800] clines=cblock.splitlines() for j,l in enumerate(clines): direct=re.search(r"Name\s+of\s+the\s+Authorized\s+(?:Courier\s*:\s*)?(.+?)(?=\s+Address\s+of|\s+Address\s*:|$)",l,re.I) if direct: val=self._clean(direct.group(1)) if val and val.lower() not in {"courier", "m/s"} and self._strict_courier(val): result["cour_be_couriername"]=val break mm=re.search(r"Name\s+of\s+the\s+Authorized.*?M/s\s+(.+)$",l,re.I) if not mm: continue val=self._clean(mm.group(1)) # Remove a trailing continuation marker/label and append # the value printed after 'Courier:' on the next row. val=re.split(r"\bAddress\s+of\s+Authorized|\bAddress\s+of\s+the",val,flags=re.I)[0].strip() for nxt in clines[j+1:j+4]: cm=re.search(r"\bCourier\s*:\s*(.+)$",nxt,re.I) if cm: tail=self._clean(cm.group(1)) if tail and tail.lower() not in {"address",""}: val=self._clean(val+" "+tail) break if self._strict_courier(val): result["cour_be_couriername"]=val break # ------------------------------------------------------------- # Supplier/consignor: first Name/Address pair in the supplier # section, with wrapped continuation lines until next section. # ------------------------------------------------------------- sm=re.search(r"SUPPLIER\s+DETAILS",upper) if sm: sblock=text[sm.end():sm.end()+2200] nm=re.search(r"\bName\s*:\s*(.*?)\s+Address\s*:\s*([^\n]+)",sblock,re.I) if nm: supplier=self._clean(nm.group(1)) if self._strict_party_name(supplier): result["cour_be_supplier_name"]=supplier address=[self._clean(nm.group(2))] slines=sblock.splitlines() # Locate the line containing the supplier Name and collect # address continuation until the next numbered/section row. try: pos=next(i for i,l in enumerate(slines) if re.search(r"\bName\s*:\s*",l,re.I) and re.search(r"Address\s*:",l,re.I)) except StopIteration: pos=-1 if pos>=0: for l in slines[pos+1:pos+10]: v=self._clean(l) if not v: continue if re.search(r"IF SUPPLIER IS NOT THE SELLER|BROKER/ AGENT DETAILS|NATURE OF TRANSACTION",v,re.I): break if self._looks_like_any_known_label(v) and not self._looks_like_address(v): continue address.append(v) addr=self._clean(" ".join(address)) if addr: for inv in result.get("invoices",[]): inv["suppliers_address"]=addr inv["cour_be_inv_supplier_name"]=self._v183_legal_spacing(supplier) # ------------------------------------------------------------- # Invoice-level values are explicitly tied to their labels. # ------------------------------------------------------------- for pat,field in [ (r"Invoice\s+Number\s*:\s*([A-Za-z0-9./_-]+)","cour_be_invoice_no"), (r"Date\s+of\s+Invoice\s*:\s*(\d{1,2}[/-]\d{1,2}[/-]\d{2,4})","cour_be_invoice_date"), (r"Terms\s+of\s+Payment\s*:\s*([^\n]+)","cour_be_inv_pymt_terms"), (r"Invoice\s+Value\s*:\s*([0-9,]+(?:\.\d+)?)","cour_be_inv_value"), (r"Currency\s*:\s*([A-Z]{3})","cour_be_inv_currency"), (r"Method\s+of\s+Valuation\s*:\s*([^\n]+)","cour_be_valuationmethod"), ]: mm=re.search(pat,text,re.I) if not mm: continue val=self._clean(mm.group(1)) if field=="cour_be_inv_pymt_terms" and (self._float(val) is not None or self._looks_like_any_known_label(val)): continue if field=="cour_be_inv_value": val=float(self._float(val) or 0.0) elif field=="cour_be_invoice_date": val=self._format_date(val) elif field=="cour_be_inv_currency": val=val.upper() for inv in result.get("invoices",[]): if field in inv and (not inv.get(field) or field in {"cour_be_invoice_no","cour_be_invoice_date","cour_be_inv_value","cour_be_inv_currency","cour_be_valuationmethod","cour_be_inv_pymt_terms"}): inv[field]=val def _valid_parent_final(self, field, value): if self._empty_value(value): return False validators={ "cour_be_importername": self._strict_party_name, "cour_be_airline": self._strict_airline, "cour_be_couriername": self._strict_courier, "cour_be_supplier_name": self._strict_party_name, "cour_be_category": self._strict_category, "cour_be_betype_description": self._strict_boe_type, } fn=validators.get(field) return fn(str(value)) if fn else True def _harden_supplier_cross_page(self, result): pages=self.pages for idx,page in enumerate(pages): current=str(page.get("text") or "") previous="\n".join(str(pages[k].get("text") or "") for k in range(max(0,idx-1),idx)) context=(previous+"\n"+current).upper() if "SUPPLIER DETAILS" not in context: continue lines=current.splitlines() for i,l in enumerate(lines): m=re.search(r"\bName\s*:\s*(.*?)\s+Address\s*:\s*(.*)$",l,re.I) if not m: continue supplier=self._clean(m.group(1)) if supplier.endswith("-") and i+1 < len(lines): nxt_raw=lines[i+1] pieces=re.split(r"\s{2,}",nxt_raw.strip()) nxt_left=pieces[0] if pieces else nxt_raw nxt_left=self._clean(nxt_left) if nxt_left: supplier=self._clean(supplier[:-1]+nxt_left) if not self._strict_party_name(supplier): continue addr=[self._clean(m.group(2))] for offset,nxt in enumerate(lines[i+1:i+10]): if supplier and offset==0 and supplier.endswith("INC") and " " in nxt: pieces=re.split(r"\s{2,}",nxt.strip()) v=self._clean(pieces[1] if len(pieces)>1 else nxt) else: v=self._clean(nxt) if not v: continue if re.search(r"IF SUPPLIER IS NOT THE SELLER|BROKER/ AGENT DETAILS|NATURE OF TRANSACTION|TERMS OF PAYMENT",v,re.I): break if self._looks_like_any_known_label(v) and not self._looks_like_address(v): continue addr.append(v) result["cour_be_supplier_name"]=supplier address=self._clean(" ".join(addr)) if not result.get("invoices"): inv=self._empty_invoice(1) inv["cour_be_inv_supplier_name"]=self._v183_legal_spacing(supplier) inv["suppliers_address"]=address result["invoices"]=[inv] else: for inv in result["invoices"]: inv["cour_be_inv_supplier_name"]=self._v183_legal_spacing(supplier) inv["suppliers_address"]=address return def _harden_invoice_fallback(self, result): # Build one invoice only when the PDF exposes an explicit invoice row. # This avoids generic page headers becoming invoice records. invoice_no="" inv_date="" for page in self.pages: text=str(page.get("text") or "") m=re.search(r"Invoice\s+Number\s*:\s*([A-Za-z0-9./_-]+).*?Date\s+of\s+Invoice\s*:\s*(\d{1,2}[/-]\d{1,2}[/-]\d{2,4})",text,re.I|re.S) if m: invoice_no=self._clean(m.group(1)); inv_date=self._format_date(m.group(2)); break if not invoice_no: return if not result.get("invoices"): inv=self._empty_invoice(1) inv["cour_be_invoice_no"]=invoice_no inv["cour_be_invoice_date"]=inv_date inv["cour_be_inv_supplier_name"]=result.get("cour_be_supplier_name","") result["invoices"]=[inv] else: for inv in result["invoices"]: if not inv.get("cour_be_invoice_no"): inv["cour_be_invoice_no"]=invoice_no if not inv.get("cour_be_invoice_date"): inv["cour_be_invoice_date"]=inv_date def _harden_parent_fields(self, result): strict={ "cour_be_importername": (self.PARENT_ALIASES.get("cour_be_importername",[]), self._strict_party_name), "cour_be_airline": (["AIRLINES", "AIRLINE", "CARRIER", "AIR CARRIER"], self._strict_airline), "cour_be_couriername": (self.PARENT_ALIASES.get("cour_be_couriername",[]), self._strict_courier), "cour_be_supplier_name": (self.PARENT_ALIASES.get("cour_be_supplier_name",[]), self._strict_party_name), "cour_be_category": (self.PARENT_ALIASES.get("cour_be_category",[]), self._strict_category), "cour_be_betype_description": (self.PARENT_ALIASES.get("cour_be_betype_description",[]), self._strict_boe_type), } for field,(aliases,validator) in strict.items(): value=self._best_strict_label_value(aliases,validator) if value and (field not in {"cour_be_importername","cour_be_supplier_name","cour_be_airline","cour_be_couriername"}) and not self._valid_parent_final(field, result.get(field)): result[field]=self._normalize_parent(field,value) # IEC branch is a small code. Never accept a duty amount/page counter. branch_aliases=["IEC BRANCH CODE", "IMPORT EXPORT BRANCH CODE", "IEC BRANCH"] branch=self._best_strict_label_value(branch_aliases,lambda v: bool(re.fullmatch(r"[A-Za-z0-9]{1,8}",self._clean(v)))) if branch and not re.search(r"\d{5,}",branch): result["cour_be_iecbranch"]=branch # Arrival airport must be a short code, not 'Of' or a label fragment. airport=self._best_strict_label_value(self.PARENT_ALIASES.get("cour_be_arrivalairport",[]),self._code_validator) if airport and not self._code_validator(str(result.get("cour_be_arrivalairport") or "")): result["cour_be_arrivalairport"]=airport.upper() # Flight must have the characteristic carrier prefix + number shape. flight=self._best_strict_label_value(self.PARENT_ALIASES.get("cour_be_flightno",[]),self._flight_validator) if flight and not self._flight_validator(str(result.get("cour_be_flightno") or "")): result["cour_be_flightno"]=flight.upper() # BOE identifier is never a generic suffix such as '-XIII'. boe=self._best_strict_label_value(self.PARENT_ALIASES.get("cour_be_beno",[]),self._boe_validator) if boe and not self._boe_validator(str(result.get("cour_be_beno") or "")): result["cour_be_beno"]=boe def _strict_numeric_value(self, value): v=self._clean(value).replace(",", "") return bool(re.fullmatch(r"-?\d+(?:\.\d+)?", v)) def _harden_payment(self, result): """Recover challan/amount/date from the strongest payment row.""" candidates=[] for page,rows in self.page_rows.items(): for row in rows: text=self._clean(row["text"]) if not text: continue # A real challan record contains a long numeric challan id, # a monetary amount and a date. Footer/page rows do not. dates=re.findall(r"\b\d{1,2}[/-]\d{1,2}[/-]\d{2,4}\b",text) nums=re.findall(r"(? 10, ) if duty: result["cour_be_total_duty"]=float(self._float(duty) or 0.0) if not result.get("cour_be_pymt_amt") or float(result.get("cour_be_pymt_amt") or 0)<=10: result["cour_be_pymt_amt"]=float(self._float(duty) or 0.0) # Broad legacy passes may have left page counters/column labels here. tr6=str(result.get("cour_be_tr6_no") or "").strip() if not re.fullmatch(r"\d{7,20}",tr6): result["cour_be_tr6_no"]="" for field in ("cour_be_total_duty","cour_be_pymt_amt"): try: n=float(result.get(field) or 0) except (TypeError,ValueError): n=0.0 if n <= 10 or 1900 <= n <= 2100: result[field]=0.0 def _harden_party_fields(self, result): """Keep importer/consignor blocks in their own visual sections.""" # Importer: prefer the exact consignee/importer label and reconstruct # wrapped legal names until the next field label. importer=self._best_strict_label_value( ["NAME OF CONSIGNEE", "CONSIGNEE NAME", "IMPORTER NAME", "NAME OF IMPORTER"], self._strict_party_name, ) if importer and len(str(result.get("cour_be_importername") or "").split()) < 2: result["cour_be_importername"]=importer supplier=self._best_strict_label_value( ["NAME OF CONSIGNOR", "CONSIGNOR NAME", "SUPPLIER NAME", "NAME OF SUPPLIER"], self._strict_party_name, ) if supplier and len(str(result.get("cour_be_supplier_name") or "").split()) < 2: result["cour_be_supplier_name"]=supplier # Invoice supplier/address are tied to the same consignor anchor. Do # not use the global first occurrence of 'ADDRESS'. if result.get("invoices"): for inv in result["invoices"]: if self._strict_party_name(inv.get("cour_be_inv_supplier_name","")) is False: inv["cour_be_inv_supplier_name"]=result.get("cour_be_supplier_name","") label=self._best_label(["ADDRESS OF CONSIGNOR","SUPPLIER ADDRESS","ADDRESS OF SUPPLIER","EXPORTER ADDRESS","SELLER ADDRESS"]) if label: address=self._address_block_from_anchor(label) if address: inv["suppliers_address"]=address def _address_block_from_anchor(self,label): page=label["_page"] rows=self.page_rows.get(page,[]) center=(label["x0"]+label["x1"])/2 collected=[] started=False for r in rows: if r["_y"] <= label["y1"]+1: continue if r["_y"]-label["y1"]>420: break txt=self._clean(r["text"]) if not txt: continue if self._is_section_or_major_label(txt): if started: break continue words=[] for w in r["words"]: wc=(w["x0"]+w["x1"])/2 if abs(wc-center)<=260 and not self._is_label(w["_text"]): words.append(w["_text"]) line=self._clean(" ".join(words)) if not line: continue # Stop when another party/section label is encountered. if any(x in self._norm(line) for x in ("name of consignee","name of consignor","supplier name","manufacturer")): break collected.append(line) started=True return self._clean(" ".join(collected)) def _looks_like_noise_child(self,row,table): text=" ".join(str(row.get(k,"") or "") for k in row) u=text.upper() noise=( "PORT:", "NOTE:", "THIS IS AN ELECTRONIC COPY", "PAGE ", "DETAILS OF", "PARTICULARS OF", "TERMS OF", "ADDRESS", "CURRENCYFOR", "EXIMSCHEME", "SPECIAL REQUEST", "ADDITIONAL INFORMATION", "SHIPPING BILL DETAILS", "PAYMENTS MADE", "PAYMENT DETAILS", "DECLARATION", "I/WE HEREBY", "I/WE ENCLOSE", "FORM NO.", "GOVERNMENT OF", "CUSTOMS", "COMMISSIONERATE", ) if any(x in u for x in noise): return True return False def _final_item_numeric_and_supplier_address_recovery(self, result): """Final conservative recovery for item pricing/valuation and supplier address. This is intentionally *not* a second parser. It is an authoritative reconciliation layer for fields that can be lost when a CBE-XIV item spans several PDF pages. Every candidate is scoped to the document's own ``Details Of Item - N`` heading, so values cannot leak between items or from the invoice/payment sections. The supplier-address cleanup similarly works on the extracted address value itself: page furniture is removed, repeated token runs caused by overlapping PDF bounding boxes are collapsed, and the address is naturally bounded at the first postal/ZIP code. """ def clean(v): v = self._clean(v) v = re.sub(r"\s*[-–—]?\s*Page\s+\d+\s+of\s+\d+\b.*$", "", v, flags=re.I) v = re.sub(r"\b(?:CopyRight|Copyright)\s*Information\s*\d{4}(?:\s*[-/]\s*\d{4})?\b.*$", "", v, flags=re.I) v = re.sub(r"\s+", " ", v).strip(" :;,\t\n") return v def number(v): v = clean(v).replace(",", "") # Do not accept document years as business numeric values. if re.fullmatch(r"\d{4}", v) and 1900 <= int(v) <= 2100: return None try: return float(v) except Exception: return None def first_number(pattern, block): m = re.search(pattern, block, re.I | re.S) if not m: return None return number(m.group(1)) # ------------------------------------------------------------- # 1. Supplier address: remove PDF column duplication and footer text. # ------------------------------------------------------------- for inv in result.get("invoices") or []: raw = clean(inv.get("suppliers_address")) if not raw: continue # Running page furniture is never part of an address. raw = re.sub(r"\bPage\s+\d+\s+of\s+\d+\b.*$", "", raw, flags=re.I).strip(" ,;-:") # Bounding-box overlap can produce sequences such as: # ANAHEIM, UNITED STATES, ANAHEIM, UNITED STATES, 92807 # Collapse repeated contiguous token sequences without applying # a document-specific city/country dictionary. tokens = raw.split() changed = True while changed and len(tokens) > 2: changed = False n = len(tokens) for size in range(min(12, n // 2), 1, -1): found = False for i in range(0, n - (2 * size) + 1): if tokens[i:i+size] == tokens[i+size:i+2*size]: tokens = tokens[:i+size] + tokens[i+2*size:] changed = True found = True break if found: break raw = " ".join(tokens) # If the address contains a postal/ZIP code, that is a strong # semantic end boundary. This prevents the following column or # repeated country/city observation from being appended. postal_matches = list(re.finditer(r"\b\d{5,6}\b", raw)) if postal_matches: raw = raw[:postal_matches[0].end()] # Clean punctuation introduced by split PDF text tokens. raw = re.sub(r"\s+([,.;])", r"\1", raw) raw = re.sub(r"([,.;])\s*([,.;])", r"\1", raw) raw = re.sub(r"\s{2,}", " ", raw).strip(" ,;:") if self._looks_like_address(raw) and len(raw) >= len(clean(inv.get("suppliers_address")) or "") * 0.45: inv["suppliers_address"] = raw # Prefer the semantic SUPPLIER DETAILS block itself over a previously # contaminated candidate. This is important when pdfplumber exposes # the left Name column and right Address column as overlapping rows. supplier_blocks = re.findall( r"SUPPLIER\s+DETAILS(.*?)(?=IF\s+SUPPLIER\s+IS\s+NOT\s+THE\s+SELLER|BROKER\s*/?\s*AGENT\s+DETAILS)", self.full_text, re.I | re.S, ) for sb in supplier_blocks: am = re.search(r"\bAddress\s*:\s*(.*)", sb, re.I | re.S) if not am: continue # The PDF text stream is column-major in some CBE-XIV files. # Consequently a wrapped supplier-name token can appear inside # the address stream (for example ``DUCTA INC`` between two # address lines). Derive the supplier name from the same semantic # block and remove only its token fragments from the address; no # company name is hardcoded. nm = re.search(r"\bName\s*:\s*(.*?)\bAddress\s*:", sb, re.I | re.S) supplier_name_tokens = [] supplier_name = "" if nm: supplier_name = self._repair_wrapped_words(clean(nm.group(1))) # Prefer an already recovered invoice/parent supplier name because # the semantic block itself may contain only the first visual line # of a wrapped legal name. for existing_name in ( *[inv.get("cour_be_inv_supplier_name") for inv in result.get("invoices") or []], result.get("cour_be_supplier_name"), ): existing_name = self._repair_wrapped_words(clean(existing_name)) if self._strict_party_name(existing_name) and len(existing_name) > len(supplier_name): supplier_name = existing_name break if supplier_name: supplier_name_tokens = [t for t in re.findall(r"[A-Za-z0-9]+", supplier_name.upper()) if len(t) >= 3] candidate = clean(am.group(1)) candidate = re.sub(r"\bPage\s+\d+\s+of\s+\d+\b", " ", candidate, flags=re.I) if supplier_name_tokens: ctoks = candidate.split() # Remove contiguous name-token sequences that occur inside the # address stream. Require at least two tokens so ordinary # address words such as ``E`` or ``INC`` are not independently # deleted. normalized_ctoks = [re.sub(r"[^A-Za-z0-9]+", "", t).upper() for t in ctoks] for size in range(min(len(supplier_name_tokens), 5), 1, -1): for i in range(len(normalized_ctoks) - size + 1): if normalized_ctoks[i:i+size] == supplier_name_tokens[-size:]: ctoks = ctoks[:i] + ctoks[i+size:] normalized_ctoks = [re.sub(r"[^A-Za-z0-9]+", "", t).upper() for t in ctoks] break else: continue break candidate = " ".join(ctoks) candidate = re.sub(r"\s+", " ", candidate).strip(" ,;:") # The PDF text stream may contain the same address tail twice # because the right-hand column was emitted once per overlapping # text box. Collapse contiguous repeated sequences first. toks = candidate.split() changed = True while changed and len(toks) > 2: changed = False n = len(toks) for size in range(min(12, n // 2), 1, -1): for i in range(0, n - 2 * size + 1): if toks[i:i+size] == toks[i+size:i+2*size]: toks = toks[:i+size] + toks[i+2*size:] changed = True break if changed: break candidate = " ".join(toks) postal_matches = list(re.finditer(r"\b\d{5,6}\b", candidate)) if postal_matches: first_postal = postal_matches[0] # Column-major extraction can emit ZIP before the country and # then repeat country/city/ZIP. If the known country from the # already-extracted parent field occurs after that ZIP, rebuild # the natural address ending as: COUNTRY, ZIP. country = clean(result.get("cour_be_coo")) if country and re.search(r"[A-Za-z]{2,}", country): cm = re.search(re.escape(country), candidate, re.I) if cm and cm.start() > first_postal.end(): prefix = candidate[:first_postal.start()].rstrip(" ,;:-") candidate = f"{prefix}, {country}, {first_postal.group(0)}" else: candidate = candidate[:first_postal.end()] else: candidate = candidate[:first_postal.end()] candidate = re.sub(r"\s+([,.;])", r"\1", candidate) candidate = re.sub(r"([,.;])\s*([,.;])", r"\1", candidate) candidate = re.sub(r"\s*[-–—]\s*(?=,)", "", candidate) candidate = re.sub(r"\s{2,}", " ", candidate).strip(" ,;:") if self._looks_like_address(candidate): for inv in result.get("invoices") or []: inv["suppliers_address"] = candidate break # Preserve the previous v18.3 wrapped-supplier-name behavior. If a # legal name ends in a PDF line-wrap hyphen, use the supplier-details # block to append the continuation without hardcoding the company. for inv in result.get("invoices") or []: current_name = self._repair_wrapped_words(clean(inv.get("cour_be_inv_supplier_name"))) if not current_name or not current_name.endswith("CON") and not current_name.endswith("CON-"): continue # Generic continuation: a supplier block line immediately following # a hyphenated Name value can contain the remainder before the # address-column tokens begin. for sb in supplier_blocks: nm2 = re.search(r"\\bName\\s*:\\s*(.*?)\\bAddress\\s*:", sb, re.I | re.S) if not nm2: continue name_text = clean(nm2.group(1)) name_text = self._repair_wrapped_words(name_text) if name_text and len(name_text) > len(current_name): inv["cour_be_inv_supplier_name"] = name_text continue # Look for the wrapped continuation on the next visual/text # line. Stop at common address-start tokens rather than using # a company-specific suffix dictionary. lines = [clean(x) for x in sb.splitlines() if clean(x)] for li, line in enumerate(lines): if current_name.replace("-", "").strip().upper() in line.replace("-", "").strip().upper(): if li + 1 < len(lines): nxt = lines[li + 1] nxt = re.sub(r"\\b(?:SUITE|STE|UNIT|ROAD|RD|STREET|ST|AVENUE|AVE|BLOCK|BUILDING|BLDG)\\b.*$", "", nxt, flags=re.I).strip(" ,:-") if nxt and re.search(r"[A-Za-z]{2,}", nxt): candidate_name = self._repair_wrapped_words(current_name.rstrip("- ") + " " + nxt) if self._strict_party_name(candidate_name) and len(candidate_name) > len(current_name): inv["cour_be_inv_supplier_name"] = candidate_name break # Final address-only reconciliation. Do not change invoice/item # extraction here. The supplier block can be column-major: a wrapped # supplier-name continuation may be emitted inside the address stream. # Recover the address from the same semantic block and use whitespace # column structure only to discard that left-column continuation. for inv in result.get("invoices") or []: blocks = re.findall( r"SUPPLIER\s+DETAILS(.*?)(?=IF\s+SUPPLIER\s+IS\s+NOT\s+THE\s+SELLER|BROKER\s*/?\s*AGENT\s+DETAILS)", self.full_text, re.I | re.S, ) for sb in blocks: am = re.search(r"\bAddress\s*:\s*(.*)", sb, re.I | re.S) if not am: continue raw_addr = am.group(1) raw_addr = re.sub(r"\bPage\s+\d+\s+of\s+\d+\b", " ", raw_addr, flags=re.I) lines = [ln.strip() for ln in raw_addr.splitlines() if ln.strip()] cleaned_lines = [] for ln in lines: # If the PDF preserved the two visual columns, keep the # right-hand address column and discard a left-hand name # continuation. This is based on structure, not company # names or page coordinates. pieces = [p.strip() for p in re.split(r"\s{2,}", ln) if p.strip()] if len(pieces) > 1: right = pieces[-1] # The right piece is address-like if it contains a # number, punctuation, or common address structure. if re.search(r"\d|,|\b(?:SUITE|STE|UNIT|ROAD|RD|STREET|ST|AVENUE|AVE|BLVD|BLOCK|BUILDING)\b", right, re.I): ln = right cleaned_lines.append(ln) # A wrapped party-name continuation can survive as the first # words of the next address line when the PDF text stream loses # its column spacing, e.g. `` SUITE 200 ...``. # Only remove such a prefix when the preceding line already # establishes an address and the current line has an explicit # address marker. This avoids deleting legitimate street names. filtered_lines=[] for li, ln in enumerate(cleaned_lines): if li > 0 and re.search(r"\d|,|\b(?:AVENUE|AVE|ROAD|RD|STREET|ST|BLVD|LANE|LN)\b", cleaned_lines[li-1], re.I): mm=re.match(r"^([A-Za-z][A-Za-z&+.-]*(?:\s+[A-Za-z][A-Za-z&+.-]*){1,3})\s+(?=(?:SUITE|STE|UNIT|FLAT|NO\.?|#)\b)", ln, re.I) if mm: ln=ln[mm.end(1):].lstrip() filtered_lines.append(ln) candidate = self._clean(" ".join(filtered_lines)) candidate = re.sub(r"\bPage\s+\d+\s+of\s+\d+\b.*$", "", candidate, flags=re.I) candidate = re.sub(r"\s*[-–—]\s*(?=,)", "", candidate) candidate = re.sub(r"\s+([,.;])", r"\1", candidate) candidate = re.sub(r"([,.;])\s*([,.;])", r"\1", candidate) candidate = re.sub(r"\s+", " ", candidate).strip(" ,;:") # Remove contiguous repeated address tokens caused by # overlapping PDF bounding boxes. toks = candidate.split() changed = True while changed and len(toks) > 2: changed = False for size in range(min(10, len(toks)//2), 1, -1): for i in range(len(toks) - 2*size + 1): if toks[i:i+size] == toks[i+size:i+2*size]: toks = toks[:i+size] + toks[i+2*size:] changed = True break if changed: break candidate = " ".join(toks) # Natural postal-code boundary. If the extracted country is # repeated after the ZIP because of column ordering, retain # the first country and move it before the first ZIP. postal_matches = list(re.finditer(r"\b\d{5,6}\b", candidate)) country = self._clean(result.get("cour_be_coo")) if postal_matches: postal = postal_matches[0] if country: cm = re.search(re.escape(country), candidate, re.I) if cm and cm.start() > postal.end(): prefix = candidate[:postal.start()].rstrip(" ,;:-") candidate = f"{prefix}, {country}, {postal.group(0)}" else: candidate = candidate[:postal.end()] else: candidate = candidate[:postal.end()] if self._looks_like_address(candidate) and len(candidate) >= 20: inv["suppliers_address"] = candidate break # ------------------------------------------------------------- # 2. Item pricing/valuation: explicit section is authoritative. # ------------------------------------------------------------- # ------------------------------------------------------------- text = self.full_text.replace("\u00ad", "") starts = list(re.finditer( r"Details\s*Of\s*Item\s*[-:]\s*(\d+)", text, re.I )) if not starts: return items = result.setdefault("items", []) def get_item(item_no): for row in items: if str(row.get("_item_section_no") or "") == str(item_no): return row for row in items: if str(row.get("cour_be_item_itemsn") or "") == str(item_no): row["_item_section_no"] = item_no return row row = self._empty_item(item_no) row["_item_section_no"] = item_no items.append(row) return row for idx, st in enumerate(starts): item_no = int(st.group(1)) end = starts[idx + 1].start() if idx + 1 < len(starts) else len(text) block = text[st.end():end] item = get_item(item_no) # Stop only at the next semantic item section. Intermediate # sections such as manufacturer/model/accessories, classification, # notification and duty details are legitimate parts of this # item's business record and must not terminate the block. detail = re.search(r"DETAILED\s+DESCRIPTION\s+OF\s+ITEM", block, re.I) detail_block = block[detail.end():] if detail else block # ---- Unit price + currency --------------------------------- m = re.search( r"Currency\s+for\s+Unit\s+Price\s*:\s*([A-Z]{3})\b" r".*?Unit\s+Price\s*:\s*([0-9][\d,]*(?:\.\d+)?)", detail_block, re.I | re.S, ) if m: currency = clean(m.group(1)).upper() upi = number(m.group(2)) if currency and currency not in {"N/A", "NA"}: item["cour_be_item_upi_currency"] = currency if upi is not None: item["cour_be_item_upi"] = upi # Some future forms put Unit Price before Currency. Recover it # without crossing the next semantic label. if not number(item.get("cour_be_item_upi")): m = re.search( r"Unit\s+Price\s*:\s*([0-9][\d,]*(?:\.\d+)?)" r"(?=\s+(?:Unit\s+of\s+Measure|Quantity|Rate\s+Of\s+Exchange|Currency\s+for\s+Unit\s+Price)\b)", detail_block, re.I | re.S, ) if m: v = number(m.group(1)) if v is not None: item["cour_be_item_upi"] = v # ---- UOM + quantity ---------------------------------------- m = re.search( r"Unit\s+of\s+Measure\s*:\s*([A-Za-z][A-Za-z0-9./_-]{0,15})" r"\s+Quantity\s*:\s*([0-9][\d,]*(?:\.\d+)?)", detail_block, re.I | re.S, ) if m: uom = clean(m.group(1)).upper() qty = number(m.group(2)) if uom and uom not in {"N/A", "NA", "NUMBER", "COPY"}: item["cour_be_item_uom"] = uom if qty is not None: item["cour_be_item_qty"] = qty # ---- Exchange rate ----------------------------------------- exch = first_number( r"Rate\s+Of\s+Exchange\s*:\s*([0-9][\d,]*(?:\.\d+)?)", detail_block, ) if exch is not None: item["cour_be_item_exchg_rate"] = exch # ---- CTSH / CETSH ------------------------------------------- cts = re.search( r"\bCTSH\s*:\s*(\d{4,12})", block, re.I ) if cts: item["cour_be_item_ctsh"] = cts.group(1) cet = re.search( r"\bCETSH\s*:\s*(\d{4,12})", block, re.I ) if cet: item["cour_be_item_cetsh"] = cet.group(1) # ---- Assessable value -------------------------------------- # This field can occur several pages after the descriptive part # of the item. Because `block` ends at the next Details Of Item # heading, it remains item-scoped even when it crosses pages. av_matches = re.findall( r"Assessable\s+Value\s*:\s*([0-9][\d,]*(?:\.\d+)?)", block, re.I, ) if av_matches: # Prefer a positive value and ignore footer/date-like noise. vals = [number(v) for v in av_matches] vals = [v for v in vals if v is not None and v >= 0] if vals: positive = [v for v in vals if v > 0] item["cour_be_item_assessable_value"] = (positive[-1] if positive else vals[-1]) # ---- Duty details ------------------------------------------ duty = re.search(r"DUTY\s+DETAILS", block, re.I) if duty: dblock = block[duty.end():] next_section = re.search( r"SHIPPING\s+BILL\s+DETAILS|PAYMENTS\s+MADE|DETAILS\s+RELATING\s+TO\s+DUTY\s+EXEMPTION|Details\s*Of\s*Item\s*-\s*\d+", dblock, re.I, ) if next_section: dblock = dblock[:next_section.start()] duty_map = { "BCD": ("cour_be_item_bcd_rt", "cour_be_item_bcd_amt"), "AIDC": ("cour_be_item_aidc_rt", "cour_be_item_aidc_amt"), "SW SRCHRG": ("cour_be_item_sws_rt", "cour_be_item_sws_amt"), "IGST": ("cour_be_item_igst_rt", "cour_be_item_igst_amt"), "CMPNSTRY": ("cour_be_item_cmpcess_rt", "cour_be_item_cmpcess_amt"), } for head, (rate_field, amount_field) in duty_map.items(): for line in dblock.splitlines(): line_clean = clean(line) if not re.search(rf"\b{re.escape(head)}\b", line_clean, re.I): continue nums = re.findall(r"\b\d+(?:\.\d+)?\b", line_clean) # First number is serial number; second is Ad Valorem # rate; last is Duty Amount. This follows the table # semantics rather than fixed character positions. if len(nums) >= 3: rate = number(nums[1]) amount = number(nums[-1]) if rate is not None: item[rate_field] = rate if amount is not None: item[amount_field] = amount break # Re-order by explicit business section and restore child serials. items.sort(key=lambda r: int(r.get("_item_section_no") or r.get("cour_be_item_itemsn") or 999999)) for i, item in enumerate(items, 1): item["cour_be_item_itemsn"] = i item["linked_invoice_id_sno"] = 1 if result.get("invoices") else 0 def _ensure_invoice_from_text_final(self, result): if result.get("invoices"): return for page in self.pages: text=str(page.get("text") or "") m=re.search(r"Invoice\s+Number\s*:\s*([A-Za-z0-9./_-]+).*?Date\s+of\s+Invoice\s*:\s*(\d{1,2}[/-]\d{1,2}[/-]\d{2,4})",text,re.I|re.S) if not m: continue inv=self._empty_invoice(1) inv["cour_be_invoice_no"]=self._clean(m.group(1)) inv["cour_be_invoice_date"]=self._format_date(m.group(2)) inv["cour_be_inv_supplier_name"]=result.get("cour_be_supplier_name","") result["invoices"]=[inv] return def _harden_child_rows(self,result): # Invoice rows: identity must be an invoice number/value/supplier. clean_inv=[] for inv in result.get("invoices",[]): if self._looks_like_noise_child(inv,"invoices"): continue supplier=self._clean(inv.get("cour_be_inv_supplier_name","")) if supplier and not self._strict_party_name(supplier): inv["cour_be_inv_supplier_name"]=result.get("cour_be_supplier_name","") rel=str(inv.get("cour_be_inv_related") or "").strip().upper() if rel not in {"Y","N"}: inv["cour_be_inv_related"]="" terms=self._clean(inv.get("cour_be_inv_pymt_terms","")) if terms and (self._float(terms) is not None or self._looks_like_any_known_label(terms)): inv["cour_be_inv_pymt_terms"]="" clean_inv.append(inv) result["invoices"]=self._dedupe_invoices(clean_inv) clean_items=[] for item in result.get("items",[]): if self._looks_like_noise_child(item,"items"): continue desc=self._clean(item.get("cour_be_item_itemdescription","")) cts=self._clean(item.get("cour_be_item_ctsh","") or item.get("cour_be_item_cetsh","")) # A valid item should have a meaningful description or customs code. if not desc and not re.fullmatch(r"\d{6,10}",cts): continue if desc and self._looks_like_any_known_label(desc): continue manuf=self._clean(item.get("cour_be_item_manufacturer","")) if manuf and not self._strict_party_name(manuf): item["cour_be_item_manufacturer"]="" clean_items.append(item) result["items"]=self._dedupe_items(clean_items) # Related-party answer is strictly anchored to the question. matches=re.findall( r"ARE\s+THE\s+BUYER\s+AND\s+SELLER\s+RELATED[^YN]{0,30}\b([YN])\b", self.full_text,re.I, ) if matches: for inv in result["invoices"]: inv["cour_be_inv_related"]=matches[0].upper() # Years are never valid duty/payment amounts. This catches cases # where a broad legacy extractor interpreted a date as money. for field in ("cour_be_total_duty", "cour_be_pymt_amt", "cour_be_interest_amt"): try: n=float(result.get(field) or 0) except (TypeError,ValueError): n=0 if 1900 <= n <= 2100: result[field]=0.0 # Invoice count is always derived from surviving invoice rows. result["cour_be_invoicecount"]=len(result["invoices"]) # If multiple identical item rows were produced from page headers, # keep distinct logical records rather than counting repeated text. result["items"]=self._dedupe_items(result["items"]) # ===================================================================== # V18.3 FINAL DYNAMIC LAYOUT AUTHORITY # ===================================================================== # This layer intentionally sits after the proven v18.3 pipeline. It does # not replace the existing extractor; it corrects only fields whose source # can be proven from a semantic label / bounded document section. No OCR, # page-number assumptions, document-specific values, or company/product # dictionaries are used. def _v183_clean_layout(self, value): value = str(value or "") value = value.replace("\u00ad", "") value = value.replace("\ufeff", "") value = value.replace("\ufffe", "") value = value.replace("\uffff", "") value = re.sub(r"\s+", " ", value).strip() value = re.sub(r"^[\s:;,]+|[\s:;,]+$", "", value) # Join alphabetic PDF line-wraps, but never remove a meaningful # separator inside identifiers such as BOE numbers. value = re.sub(r"(?<=[A-Za-z])-\s+(?=[A-Za-z])", "", value) return value.strip(" :;,\t\r\n") def _v183_date(self, value): value = self._v183_clean_layout(value) if not value: return "" formats = ( "%d/%m/%Y", "%d-%m-%Y", "%d.%m.%Y", "%d/%m/%y", "%d-%m-%y", "%d.%m.%y", "%d-%b-%Y", "%d %b %Y", "%d/%b/%Y", "%d-%B-%Y", "%d %B %Y", "%d/%B/%Y", ) for fmt in formats: try: return datetime.strptime(value, fmt).strftime("%Y-%m-%d") except ValueError: pass # Some PDFs use a textual month with punctuation/case noise. compact = re.sub(r"\s+", " ", value.replace(".", " ")).strip() for fmt in ("%d %b %Y", "%d %B %Y"): try: return datetime.strptime(compact.title(), fmt).strftime("%Y-%m-%d") except ValueError: pass return value def _v183_numeric(self, value): value = self._v183_clean_layout(value).replace(",", "") m = re.fullmatch(r"-?\d+(?:\.\d+)?", value) return float(m.group()) if m else None def _v183_label_region(self, text, label_pattern, stop_patterns=(), limit=500): """Return bounded value text after one semantic label.""" if not text: return "" m = re.search(label_pattern + r"\s*[:\-]?\s*", text, re.I) if not m: return "" tail = text[m.end():m.end() + limit] stops = list(stop_patterns) + [ r"DETAILS\s+OF\s+ITEM", r"DETAILS\s+OF\s+INVOICE", r"DUTY\s+DETAILS", r"PAYMENT\s+DETAILS", r"DECLARATION", r"Page\s+\d+\s+of\s+\d+", ] for stop in stops: sm = re.search(stop, tail, re.I) if sm: tail = tail[:sm.start()] return self._v183_clean_layout(tail) def _v183_extract_first_numeric_after(self, text, label_pattern, stop_patterns=(), digits=(1, 24)): region = self._v183_label_region(text, label_pattern, stop_patterns, 250) if not region: return "" lo, hi = digits m = re.search(rf"(? rate, Duty Amount -> amount.""" items = result.get("items") or [] if not items: return headings = list(re.finditer(r"DETAILS\s+OF\s+ITEM\s*(?:[-:]\s*)?(\d+)", self.full_text, re.I)) if not headings: headings = [None] duty_aliases = { "bcd": (r"\bBCD\b", "cour_be_item_bcd_rt", "cour_be_item_bcd_amt"), "aidc": (r"\bAIDC\b", "cour_be_item_aidc_rt", "cour_be_item_aidc_amt"), "sws": (r"\bSW\s*SRCHRG\b|\bSWS\b|SOCIAL\s+WELFARE\s+SURCHARGE", "cour_be_item_sws_rt", "cour_be_item_sws_amt"), "igst": (r"\bIGST\b|INTEGRATED\s+GST", "cour_be_item_igst_rt", "cour_be_item_igst_amt"), "cmpcess": (r"\bCMPNSTRY\b|COMPENSATION\s+CESS|CMP\s*CESS", "cour_be_item_cmpcess_rt", "cour_be_item_cmpcess_amt"), } def nums(line): return [float(x.replace(",", "")) for x in re.findall(r"(?= 4: break joined += " " + nxt logical.append(joined) for line in logical: for _, (head_pat, rate_field, amount_field) in duty_aliases.items(): if not re.search(head_pat, line, re.I): continue mhead = re.search(head_pat, line, re.I) tail = line[mhead.end():] values = nums(tail) if len(values) >= 4: rate, amount = values[0], values[-1] elif len(values) == 3: rate, amount = values[0], values[-1] elif len(values) == 2: rate, amount = values[0], values[1] else: continue if 0 <= rate <= 100: for item in target_items: item[rate_field] = float(rate) item[amount_field] = float(amount) break def _v183_cleanup_final_fields(self, result): # Airport is a code, always normalized to uppercase. airport = self._v183_clean_layout(result.get("cour_be_arrivalairport")) if re.fullmatch(r"[A-Za-z]{2,4}", airport): result["cour_be_arrivalairport"] = airport.upper() # HAWB must be numeric in these CBE layouts. Never allow consignee text. hawb = self._v183_clean_layout(result.get("cour_be_hawbno")) if not re.fullmatch(r"\d{8,20}", hawb): hawb2 = self._v183_extract_first_numeric_after( self.full_text, r"(?:HOUSE\s+AIRWAY\s+BILL\s*\(\s*HAWB\s*\)\s*NUMBER|HOUSE\s+AIRWAY\s+BILL\s+NUMBER|HAWB\s+NUMBER|HAWB\s+NO)", (r"DATE\s+OF\s+HAWB", r"HAWB\s+DATE", r"MARKS\s+AND\s+NUMBERS"), (8, 20), ) if hawb2: result["cour_be_hawbno"] = hawb2 # Dates: repair textual month dates and retain already-correct dates. for field in ("cour_be_arrivaldate", "cour_be_mawb_dt", "cour_be_hawb_dt", "be_date", "cour_be_inward_date", "out_of_charge_date", "cour_be_challan_date"): value = self._v183_clean_layout(result.get(field)) if value: result[field] = self._v183_date(value) # Supplier/invoice supplier legal names: remove only adjacent labels, # never shorten legitimate suffixes such as INC/LIMITED/PRIVATE LIMITED. supplier = self._v183_clean_layout(result.get("cour_be_supplier_name")) if supplier: supplier = re.split(r"\b(?:Address|Nature\s+of\s+Transaction|Terms\s+of\s+Payment|Broker)\s*:", supplier, maxsplit=1, flags=re.I)[0] result["cour_be_supplier_name"] = self._v183_clean_layout(supplier) for inv in result.get("invoices") or []: inv["cour_be_inv_supplier_name"] = self._v183_clean_layout(inv.get("cour_be_inv_supplier_name")) inv["suppliers_address"] = self._v183_clean_layout(inv.get("suppliers_address")) if inv.get("cour_be_invoice_date"): inv["cour_be_invoice_date"] = self._v183_date(inv["cour_be_invoice_date"]) # Invoice count is derived from explicit declaration when it is larger # than the rows, otherwise from the actual logical invoice rows. declared = [] for m in re.finditer(r"(?:NUMBER\s+OF\s+INVOICES|NO\.?\s+OF\s+INVOICES|TOTAL\s+NUMBER\s+OF\s+INVOICES)\s*:\s*(\d+)", self.full_text, re.I): declared.append(int(m.group(1))) if declared: count = max(declared) while len(result.setdefault("invoices", [])) < count: result["invoices"].append(self._empty_invoice(len(result["invoices"]) + 1)) result["cour_be_invoicecount"] = len(result.get("invoices") or []) for i, inv in enumerate(result.get("invoices") or [], 1): inv["cour_be_invsno"] = i # Exchange rate and duty values are numeric contracts. for item in result.get("items") or []: if item.get("cour_be_item_exchg_rate") not in (None, ""): item["cour_be_item_exchg_rate"] = float(self._v183_numeric(str(item["cour_be_item_exchg_rate"])) or 0.0) def _v183_row_cell(self, label_aliases, stop_aliases=()): """Return the cell text after a compact PDF label, using word order/x positions.""" labels=[re.sub(r"[^A-Za-z0-9]+","",x).lower() for x in label_aliases] stops=[re.sub(r"[^A-Za-z0-9]+","",x).lower() for x in stop_aliases] for rows in self.page_rows.values(): for row in rows: words=row.get("words") or [] for i,w in enumerate(words): cw=re.sub(r"[^A-Za-z0-9]+","",str(w.get("_text", ""))).lower() lab=next((x for x in labels if x and (cw==x or x in cw)),None) if not lab: continue vals=[] for ww in words[i+1:]: cww=re.sub(r"[^A-Za-z0-9]+","",str(ww.get("_text", ""))).lower() if cww and any(s and (cww==s or s in cww) for s in stops): break vals.append(str(ww.get("_text", ""))) value=self._v183_clean_layout(" ".join(vals)) if value: return value return "" def _v183_recover_compact_invoice_rows(self, result): invoices=result.setdefault("invoices",[]) # Discover explicit invoice serials and count from visual rows even when # pdfplumber has compacted labels such as NumberofInvoices. declared=[] for rows in self.page_rows.values(): for row in rows: text=self._v183_clean_layout(row.get("text","")) cm=re.search(r"NumberofInvoices\s*:\s*(\d+)", re.sub(r"\s+","",text), re.I) if cm: declared.append(int(cm.group(1))) count=max(declared) if declared else len(invoices) while len(invoices)name_x+5: name_parts.append(str(w.get("_text",""))) elif addr_x is not None and x>addr_x+5: addr_parts.append(str(w.get("_text",""))) for rr in rows[anchor+1:anchor+10]: rt=self._v183_clean_layout(rr.get("text","")) if not rt or re.search(r"IFSUPPLIER|BROKER/AGENT|NATUREOFTRANSACTION|TERMSOFPAYMENT|DETAILSOFITEM|DUTYDETAILS|PAYMENTDETAILS",re.sub(r"[^A-Za-z0-9/]+","",rt),re.I): break for w in rr.get("words",[]): x=float(w.get("x0",0)); wt=str(w.get("_text","")) if name_x is not None and addr_x is not None and name_x-10<=x=addr_x-10: addr_parts.append(wt) supplier=self._v183_clean_layout(" ".join(name_parts)) address=self._v183_clean_layout(" ".join(addr_parts)) if supplier and re.search(r"[A-Za-z]{2,}",supplier): result["cour_be_supplier_name"]=supplier for inv in result.setdefault("invoices",[]): inv["cour_be_inv_supplier_name"]=self._v183_legal_spacing(supplier) if address: inv["suppliers_address"]=address break # Importer: same visual-column strategy. The right column is never # allowed to consume the left-column legal-name continuation. for page_no, rows in self.page_rows.items(): for idx,row in enumerate(rows): words=row.get("words") or [] compact=re.sub(r"[^A-Za-z0-9]+","",row.get("text","")).lower() if not ("name" in compact and "address" in compact): continue if not any(re.sub(r"[^A-Za-z0-9]+","",str(w.get("_text",""))).lower()=="name" for w in words): continue if not any(re.sub(r"[^A-Za-z0-9]+","",str(w.get("_text",""))).lower()=="address" for w in words): continue name_x=addr_x=None; name_parts=[]; addr_parts=[] for w in words: c=re.sub(r"[^A-Za-z0-9]+","",str(w.get("_text",""))).lower(); x=float(w.get("x0",0)); wt=str(w.get("_text","")) if c=="name": name_x=x elif c=="address": addr_x=x elif name_x is not None and addr_x is None and x>name_x+5: name_parts.append(wt) elif addr_x is not None and x>addr_x+5: addr_parts.append(wt) if name_x is None or addr_x is None: continue for rr in rows[idx+1:idx+8]: rt=self._v183_clean_layout(rr.get("text","")) if not rt or re.search(r"CATEGORYOFIMPORTER|INCASEOFOTHERIMPORTER|AUTHORI[ZS]EDDEALERCODE|BOENUMBER|KYCDOCUMENT|STATECODE|PARTICULARSOFHIGHSEASELLER",re.sub(r"[^A-Za-z0-9]+","",rt),re.I): break # Only append words whose x coordinate proves their column. for w in rr.get("words",[]): x=float(w.get("x0",0)); wt=str(w.get("_text","")) if name_x-10<=x=addr_x-10: addr_parts.append(wt) name=self._v183_clean_layout(" ".join(name_parts)); addr=self._v183_clean_layout(" ".join(addr_parts)) if name and not self._looks_like_any_known_label(name): current=self._v183_clean_layout(result.get("cour_be_importername")) if len(name) >= len(current): result["cour_be_importername"]=self._v183_legal_spacing(name) if addr: current_addr=self._v183_clean_layout(result.get("cour_be_importer_address")) if len(addr) >= len(current_addr)*0.75: result["cour_be_importer_address"]=addr break def _v183_recover_compact_item_and_duty(self, result): items=result.get("items") or [] if not items: return # Item exchange rate from the explicit Rate of Exchange label. for rows in self.page_rows.values(): for row in rows: text=self._v183_clean_layout(row.get("text","")) compact=re.sub(r"[^A-Za-z0-9]+","",text).lower() if "rateofexchange" in compact: m=re.search(r"RateofExchange\s*:\s*([0-9][\d,]*(?:\.\d+)?)", re.sub(r"\s+","",text), re.I) if m: rate=float(m.group(1).replace(",","")) # Use the nearest/only logical item. For multiple items, # the item section order is used where provenance exists. target=items[0] target["cour_be_item_exchg_rate"]=rate # Duty table from visual rows. The columns are semantically fixed by # the table header, not by the serial number: # Sr.No | Duty Head | Ad Valorem | Specific Rate | Duty Forgone | Duty Amount duty_maps={ "bcd":("cour_be_item_bcd_rt","cour_be_item_bcd_amt"), "aidc":("cour_be_item_aidc_rt","cour_be_item_aidc_amt"), "swsrchrg":("cour_be_item_sws_rt","cour_be_item_sws_amt"), "sws":("cour_be_item_sws_rt","cour_be_item_sws_amt"), "igst":("cour_be_item_igst_rt","cour_be_item_igst_amt"), "cmpnstry":("cour_be_item_cmpcess_rt","cour_be_item_cmpcess_amt"), } for rows in self.page_rows.values(): for idx,row in enumerate(rows): h=re.sub(r"[^A-Za-z0-9]+","",row.get("text","")).lower() if "dutyhead" not in h or "advalorem" not in h or "dutyamount" not in h: continue for rr in rows[idx+1:idx+12]: raw=self._v183_clean_layout(rr.get("text","")) compact=re.sub(r"[^A-Za-z0-9]+","",raw).lower() if "shippingbill" in compact or "paymentdetails" in compact or "declaration" in compact: break words=rr.get("words") or [] if len(words)<3: continue head="" head_idx=None for wi,w in enumerate(words): c=re.sub(r"[^A-Za-z0-9]+","",str(w.get("_text",""))).lower() if c in duty_maps: head=c; head_idx=wi; break if head_idx is None: continue nums=[] for w in words[head_idx+1:]: token=str(w.get("_text","")) mnum=re.fullmatch(r"\d+(?:,\d{3})*(?:\.\d+)?",token.strip()) if mnum: nums.append(float(token.replace(",",""))) if len(nums)>=4: rate,amount=nums[0],nums[-1] elif len(nums)==3: rate,amount=nums[0],nums[-1] elif len(nums)==2: rate,amount=nums[0],nums[1] else: continue if not (0<=rate<=100): continue rf,af=duty_maps[head] for item in items: item[rf]=float(rate); item[af]=float(amount) break def _final_dynamic_issue_fixes(self, result): self._v183_recover_compact_item_and_duty(result) self._v183_recover_compact_supplier_and_importer(result) self._v183_recover_compact_invoice_rows(result) self._v183_recover_parent_visual(result) self._v183_recover_parent_layout(result) self._v183_recover_supplier(result) self._v183_recover_invoices(result) self._v183_recover_item_exchange_rate(result) self._v183_duty_authority(result) self._v183_cleanup_final_fields(result) def _v183_visual_value(self, label_patterns, stop_patterns=(), max_rows=4): """Extract a value from the visual row model with compact-label support.""" labels = [re.sub(r"[^A-Za-z0-9]+", "", x).lower() for x in label_patterns] stops = [re.sub(r"[^A-Za-z0-9]+", "", x).lower() for x in stop_patterns] for page_no, rows in self.page_rows.items(): for idx, row in enumerate(rows): words = row.get("words") or [] raw = " ".join(str(w.get("_text", "")) for w in words) compact = re.sub(r"[^A-Za-z0-9]+", "", raw).lower() matched = next((lab for lab in labels if lab and lab in compact), None) if not matched: continue # Find the first word whose cumulative compact text reaches the # label. This avoids matching a neighboring column's value. pos = 0 start_word = None for wi, w in enumerate(words): wt = str(w.get("_text", "")) cwt = re.sub(r"[^A-Za-z0-9]+", "", wt).lower() if not cwt: continue if matched.startswith(cwt) or cwt in matched: trial = pos + len(cwt) # Use a text-level fallback if the label is split into # many PDF words; the exact start is still near here. pos += len(cwt) # Simpler robust route: search the row text for a spaced version # of the label and then bound the value by the next known label. label_regex = re.escape(label_patterns[labels.index(matched)]).replace(r"\ ", r"\s*") m = re.search(label_regex + r"\s*[:\-]?\s*", raw, re.I) if not m: # Compact raw text such as NameofAuthorizedCourier: cm = re.search(re.escape(matched), compact, re.I) if not cm: continue # Recover value by using word positions whose compact prefix # lies after the compact label end. acc = 0 value_words = [] for w in words: wt = str(w.get("_text", "")).strip() cw = re.sub(r"[^A-Za-z0-9]+", "", wt).lower() if not cw: continue next_acc = acc + len(cw) if next_acc <= cm.end(): acc = next_acc continue value_words.append(wt) acc = next_acc value = self._v183_clean_layout(" ".join(value_words)) else: tail = raw[m.end():] # Stop at another semantic label in the same row. if stop_patterns: stop_re = "|".join( re.escape(s).replace(r"\ ", r"\s*") for s in stop_patterns ) sm = re.search(stop_re, tail, re.I) if sm: tail = tail[:sm.start()] value = self._v183_clean_layout(tail) if value and not self._looks_like_any_known_label(value): return value # Some labels occupy one row and their value continues on the # next one. Only follow a short local window and stop on major # section labels. continuation = [] for rr in rows[idx + 1:idx + 1 + max_rows]: rt = self._v183_clean_layout(rr.get("text", "")) if not rt: continue if re.search(r"DETAILS\s+OF|PARTICULARS\s+OF|IGM\s+DETAILS|DUTY\s+DETAILS|PAYMENT\s+DETAILS|Page\s+\d+\s+of", rt, re.I): break if any(re.search(s, rt, re.I) for s in stop_patterns): break continuation.append(rt) value = self._v183_clean_layout(" ".join([value] + continuation)) if value: return value return "" def _v183_recover_parent_visual(self, result): # Exact visual row extraction for the fields most vulnerable to # adjacent-column bleed. These values are sourced from the CBE's own # visual rows, not from arbitrary PDF text order. value = self._v183_visual_value( ["House Airway Bill (HAWB) Number", "House Airway Bill Number", "House Airway Bill (HAWB)", "HAWB Number", "HAWB No"], ["Date of HAWB", "HAWB Date", "Marks and Numbers", "Number of Packages"], ) if re.fullmatch(r"\d{8,20}", value): result["cour_be_hawbno"] = value value = self._v183_row_cell(["Name of the Authorized Courier", "Name of Authorized Courier", "Authorized Courier", "Authorised Courier"], ["Address of Authorized Courier", "Particulars of the Customs House Agent"]) if not value: # Some pdfplumber streams split "Name of the Authorized Courier" # across the row/next row. Use the value word to the right of the # distinctive "NameoftheAuthorized" fragment. for rows in self.page_rows.values(): for rr in rows: words=rr.get("words") or [] txt=re.sub(r"[^A-Za-z0-9]+","",rr.get("text","")) if "NameoftheAuthorized" not in txt: continue anchor_candidates = [ (float(w.get("x0", 0)), w) for w in words if "NameoftheAuthorized" in re.sub( r"[^A-Za-z0-9]+", "", str(w.get("_text", "")) ) ] # The fallback is optional. Some PDFs expose the label # only in the row text while their individual word stream # is fragmented differently. Never call max() on an empty # candidate sequence. if not anchor_candidates: continue anchor=max(anchor_candidates, key=lambda x: x[0]) ax=anchor[0] right=[str(w.get("_text","")) for w in words if float(w.get("x0",0))>ax+20 and "Courier" not in re.sub(r"[^A-Za-z0-9]+","",str(w.get("_text","")))] if right: value=self._v183_clean_layout(" ".join(right)) break if value: break if value and re.search(r"[A-Za-z]{2,}", value): result["cour_be_couriername"] = value value = self._v183_visual_value( ["Marks and Numbers", "Marks & Numbers", "Marks and Nos", "Marks Nos"], ["Number of Packages", "Type of Packages", "Interest Amount", "Unit of Measure"], ) if value and not self._looks_like_any_known_label(value): result["cour_be_marksnos"] = value # CBE/BOE identifier from the exact label row; concatenate only rows # that look like an identifier continuation. for page_no, rows in self.page_rows.items(): for idx, row in enumerate(rows): rt = self._v183_clean_layout(row.get("text", "")) if not re.search(r"(?:CBEXIV|CBEXIII|CBE\s*-?\s*XIV|CBE\s*-?\s*XIII)\s*Number", rt, re.I): continue candidates = re.findall(r"[A-Za-z0-9]+(?:[_-][A-Za-z0-9]+){2,}", rt.replace("\u00ad", "")) if candidates: result["cour_be_beno"] = max(candidates, key=len) break # Importer name/address are two visual columns. Use explicit row labels # where possible; fall back to the existing semantic extraction. for page_no, rows in self.page_rows.items(): for idx, row in enumerate(rows): rt = self._v183_clean_layout(row.get("text", "")) if not re.search(r"(?:Name|Name\s+of\s+Importer|Name\s+of\s+Consignee)\s*:", rt, re.I): continue if not re.search(r"(?:Address)\s*:", rt, re.I): continue m = re.search(r"(?:Name|Name\s+of\s+Importer|Name\s+of\s+Consignee)\s*:\s*(.*?)\s+(?:Address)\s*:\s*(.*)$", rt, re.I) if not m: continue name = self._v183_clean_layout(m.group(1)) addr = self._v183_clean_layout(m.group(2)) if name and re.search(r"[A-Za-z]{2,}", name) and not self._looks_like_any_known_label(name): result["cour_be_importername"] = name if addr: # Continue the right-column address only until the next # semantic section/field. Hyphenated line wraps are joined. parts = [addr] for rr in rows[idx + 1:idx + 8]: t = self._v183_clean_layout(rr.get("text", "")) if not t or re.search(r"Category\s+Of\s+Importer|In\s+Case\s+Of\s+Other\s+Importer|Authorised\s+Dealer|BOE\s+Number|KYC\s+Document|State\s+Code|Particulars\s+Of\s+High\s+Sea", t, re.I): break if re.search(r"^[A-Za-z][A-Za-z ]+\s*:\s*", t): break parts.append(t) address = self._v183_clean_layout(" ".join(parts)) result["cour_be_importer_address"] = address break # Dates/codes in the IGM table are always sourced from that table. grid = self._v183_best_igm_row() if grid: _, airline, flight, airport, arrival = grid result["cour_be_airline"] = self._v183_clean_layout(airline) result["cour_be_flightno"] = self._v183_clean_layout(flight).upper() result["cour_be_arrivalairport"] = self._v183_clean_layout(airport).upper() result["cour_be_arrivaldate"] = self._v183_date(arrival) result["cour_be_inward_date"] = result["cour_be_arrivaldate"] def _v183_legal_spacing(self, value): value=self._v183_clean_layout(value) # Only repair unambiguous legal/entity suffixes that pdfplumber can # concatenate when the source PDF visually separates the words. replacements=( (r"\bPRIVATE(?=LIMITED\b)", "PRIVATE "), (r"\bPUBLIC(?=LIMITED\b)", "PUBLIC "), (r"\bPARTS(?=COMPANY\b)", "PARTS "), (r"\bCOMPANY(?=INC\b)", "COMPANY "), (r"\bGLOBAL(?=PARTS\b)", "GLOBAL "), (r"\bEXPRESS(?=INDIA\b)", "EXPRESS "), (r"\bINDIA(?=PRIVATE\b)", "INDIA "), ) for pat,repl in replacements: value=re.sub(pat,repl,value,flags=re.I) return re.sub(r"\s+"," ",value).strip() def _v183_final_semantic_authority(self, result): """Final semantic authority for the fields that were previously vulnerable to coordinate drift. This method intentionally does NOT use absolute coordinates. It uses the text extracted by pdfplumber, document sections, explicit labels, and local label-to-value boundaries. It is run last so a broad legacy extractor cannot overwrite an authoritative value afterwards. In particular, CBE-XIII party/header fields are taken from the CBE section only. Attached export/customs documents are allowed to supply invoice metadata, but never importer/consignee identity or CBE flight data. """ source = str(getattr(self, "full_text", "") or "") if not source.strip(): return def clean(v): v = str(v or "") v = v.replace("\u00ad", "").replace("\ufeff", "") v = re.sub(r"[\u2010\u2011\u2012\u2013\u2014]\s+(?=[A-Za-z0-9])", "", v) v = re.sub(r"(?<=[A-Za-z])-\s+(?=[A-Za-z])", "", v) v = re.sub(r"\s+", " ", v) return v.strip(" \t\r\n:;,.-") def compact(v): return re.sub(r"[^A-Za-z0-9]+", "", clean(v)).lower() def date_value(v): v = clean(v) if not v: return "" # Support both customs dates (DD/MM/YYYY) and commercial-invoice # dates such as 31-JUL-2026. for fmt in ( "%d/%m/%Y", "%d-%m-%Y", "%d/%m/%y", "%d-%m-%y", "%d-%b-%Y", "%d-%B-%Y", "%d/%b/%Y", "%d/%B/%Y", "%d-%b-%y", "%d-%B-%y", "%d/%b/%y", "%d/%B/%y", ): try: return datetime.strptime(v, fmt).strftime("%Y-%m-%d") except ValueError: pass return "" def num(v): try: x = clean(v).replace(",", "") if not re.fullmatch(r"[-+]?\d+(?:\.\d+)?", x): return None return float(x) except Exception: return None def section(start_pattern, stop_patterns): m = re.search(start_pattern, source, re.I) if not m: return "" tail = source[m.end():] stops = [] for pat in stop_patterns: sm = re.search(pat, tail, re.I) if sm: stops.append(sm.start()) return tail[:min(stops)] if stops else tail def labelled(block, labels, stops, max_len=260): if not block: return "" lp = "(?:" + "|".join(labels) + ")" sp = "(?:" + "|".join(stops) + ")" m = re.search( rf"{lp}\s*[:：-]?\s*(.*?)(?=\s+{sp}\s*[:：-]|$)", block, re.I | re.S, ) if not m: return "" value = clean(m.group(1)) return value[:max_len].strip() if value else "" def party_name(v): v = clean(v) if not v or len(v) > 180: return False if self._looks_like_any_known_label(v): return False if re.search( r"\b(?:Address|Import Export Code|IEC Branch Code|Special Request|" r"No of Packages|Gross Weight|Assessable Value|Invoice Value|" r"KYC Document|State Code|AD Code)\b", v, re.I, ): return False return bool(re.search(r"[A-Za-z]{2,}", v)) def address_value(v): v = clean(v) if not v or len(v) > 500: return False if re.search( r"\b(?:Import Export Code|IEC Branch Code|Special Request|No of Packages|" r"Gross Weight|Net Weight|Assessable Value|Duty\s*\(Rs\.\)|Invoice Value|" r"KYC Document|State Code|AD Code|Government / Non-Government)\b", v, re.I, ): return False return bool(re.search(r"[A-Za-z]{2,}", v)) # ============================================================= # 1. CBE-XIII / CBE-XIV primary section # ============================================================= cbe = section( r"Form\s+Courier\s+Bill\s+Of\s+Entry|Form\s+Regular\s+Bill\s+Of\s+entry|" r"Courier\s+Bill\s+Of\s+Entry", [r"Page\s+\d+\s+of\s+\d+", r"COPYRIGHTINFORMATION"], ) # If the first form section is not found, use the portion before the # first attached certificate/export declaration. if not cbe: cbe = source cert_pos = re.search( r"(?:SUBSIDIARY\s+EXPORT\s+CERTIFICATE|DECLARATION\s+D['’]?EXPORTATION|\*WAYBILL\s+DOC\*)", cbe, re.I, ) if cert_pos: cbe = cbe[:cert_pos.start()] # ------------------------------------------------------------- # CBE number: extract only the identifier after the exact CBE label. # Supports PDF line wraps such as 08\n07_12463. # ------------------------------------------------------------- cbe_norm = cbe.replace("\u00ad", " ") cbe_norm = re.sub(r"\s+", " ", cbe_norm) cbe_patterns = [ r"CBE\s*[- ]?XIII\s*Number\s*[:：-]?\s*((?:CBEXIII)_[A-Z0-9_-]+(?:\s+[0-9][A-Z0-9_-]*)*)", r"CBE\s*[- ]?XIV\s*Number\s*[:：-]?\s*((?:CBEXIV)_[A-Z0-9_-]+(?:\s+[0-9][A-Z0-9_-]*)*)", r"CBEXIII\s*Number\s*[:：-]?\s*(CBEXIII_[A-Z0-9_-]+(?:\s+[0-9][A-Z0-9_-]*)*)", r"CBEXIV\s*Number\s*[:：-]?\s*(CBEXIV_[A-Z0-9_-]+(?:\s+[0-9][A-Z0-9_-]*)*)", ] cbe_no = "" for pat in cbe_patterns: m = re.search(pat, cbe_norm, re.I) if m: candidate = re.sub(r"\s+", "", clean(m.group(1))) candidate = re.sub(r"[^A-Za-z0-9_-]", "", candidate) if re.fullmatch(r"CBEX(?:III|IV)_[A-Za-z0-9_-]{8,}", candidate, re.I): cbe_no = candidate break if cbe_no: result["cour_be_beno"] = cbe_no # ------------------------------------------------------------- # Authorized courier: bounded between its own label and its own # address label. Never use the next line after the address section. # ------------------------------------------------------------- courier = labelled( cbe, [r"Name\s+of\s+the\s+Authorized\s+Courier", r"Name\s+of\s+Authorized\s+Courier", r"Name\s+of\s+the\s+Authorised\s+Courier"], [r"Address\s+of\s+Authorized\s+Courier", r"Address\s+of\s+Authorised\s+Courier", r"IGM\s+DETAILS", r"Particulars\s+of\s+the\s+Customs\s+House\s+Agent"], 140, ) courier = re.split(r"\s+(?:IGM\s+DETAILS|Particulars\s+of\s+the\s+Customs\s+House\s+Agent)\b", courier, maxsplit=1, flags=re.I)[0] if courier: courier = clean(courier) if (len(courier) <= 140 and re.search(r"[A-Za-z]{2,}", courier) and not re.search(r"\b(?:IGM|Airlines|Flight|Airport|Date|Address|Courier Registration)\b", courier, re.I) and not self._looks_like_any_known_label(courier)): result["cour_be_couriername"] = courier # ------------------------------------------------------------- # CBE-XIII consignee/consignor blocks. These exact labels are much # stronger than the generic "Name"/"Address" labels used previously. # ------------------------------------------------------------- importer_name = labelled( cbe, [r"Name\s+of\s+Consignee", r"Consignee\s+Name"], [r"Address\s+of\s+Consignee", r"Import\s+Export\s+Code", r"IEC\s+Branch\s+Code"], 180, ) importer_addr = labelled( cbe, [r"Address\s+of\s+Consignee", r"Consignee\s+Address"], [r"Import\s+Export\s+Code", r"IEC\s+Branch\s+Code", r"Special\s+Request", r"No\s+of\s+Packages"], 500, ) if party_name(importer_name): result["cour_be_importername"] = importer_name if address_value(importer_addr): result["cour_be_importer_address"] = importer_addr supplier_name = labelled( cbe, [r"Name\s+of\s+Consignor", r"Consignor\s+Name"], [r"Address\s+of\s+Consignor", r"Name\s+of\s+Consignee"], 180, ) supplier_addr = labelled( cbe, [r"Address\s+of\s+Consignor", r"Consignor\s+Address"], [r"Name\s+of\s+Consignee", r"Address\s+of\s+Consignee", r"Import\s+Export\s+Code"], 500, ) if party_name(supplier_name): result["cour_be_supplier_name"] = supplier_name # CBE-XIV uses generic Name:/Address: labels inside a dedicated # importer section. Because the section is bounded first, these generic # labels cannot accidentally select the customs-house-agent block. importer_xiv = section( r"PARTICULARS\s*OF\s*THE\s*IMPORTER|IMPORTER\s+DETAILS", [r"PARTICULARS\s*OF\s*THE\s*HIGH\s*SEA\s*SELLER", r"SPECIAL\s+REQUESTS", r"DETAILS\s+OF\s+CRN"], ) if importer_xiv: im = re.search( r"\bName\s*:\s*(.*?)\s+Address\s*:\s*(.*?)(?=\s+(?:Category\s*Of\s*Importer|Type\s*Of\s*Importer|In\s*Case\s*Of\s*other\s*Importer|Authori[sz]ed\s*Dealer|BOE\s*Number|BOE\s*Date|KYC\s*Document|State\s*Code)\s*:|$)", importer_xiv, re.I | re.S, ) if not im: # pdfplumber may concatenate form labels, e.g. # CategoryOfImporter / AuthorisedDealerCodeOfBank. im = re.search( r"\bName\s*:\s*(.*?)\s+Address\s*:\s*(.*?)(?=\s+(?:CategoryOfImporter|TypeOfImporter|InCaseOfotherImporter|Authori[sz]edDealer|BOENumber|BOEDate|KYCDocument|StateCode)\s*:|$)", importer_xiv, re.I | re.S, ) if im: xiv_name = clean(im.group(1)) xiv_addr = clean(im.group(2)) # CBE-XIV may print the legal-name continuation on the next # physical line while the address begins on the same row. # Recover only a leading parenthesized/legal-suffix fragment # from the address stream; do not use a fixed x-coordinate. suffix = re.search( r"(\([^)]{2,60}\)\s*(?:PRIVATE\s*LIMITED|PVT\s*LTD|PUBLIC\s*LIMITED|LIMITED|LTD|LLP))\s+(?=\d)", xiv_addr, re.I, ) if suffix: continuation = clean(suffix.group(1)) xiv_addr = clean(xiv_addr[:suffix.start()] + xiv_addr[suffix.end():]) if continuation: xiv_name = clean(xiv_name + " " + continuation) if party_name(xiv_name): result["cour_be_importername"] = xiv_name if address_value(xiv_addr): result["cour_be_importer_address"] = xiv_addr # ------------------------------------------------------------- # CBE IGM grid. Use the row immediately after the complete IGM header. # This excludes export-certificate Flight No. values by construction. # ------------------------------------------------------------- igm = section(r"IGM\s+DETAILS", [r"Airport\s+of\s+Shipment", r"HAWB\s+Number"]) if igm: # Header/data are often split over two physical lines. igm_clean = re.sub(r"\s+", " ", igm).strip() m = re.search( r"AIRLINES\s+FLIGHT\s+NO\.?\s+AIRPORT\s+OF\s+ARRIVAL\s+" r"(?:FIRST\s+PORT\s+OF\s+ARRIVAL\s+)?DATE\s+OF\s+ARRIVAL(?:\s+TIME\s+OF\s+ARRIVAL)?\s+" r"(.+?)\s+((?:[A-Z]{1,5})\s*\d{2,6})\s+([A-Z]{3})\s+(?:N/A\s+)?" r"(\d{1,2}[/-]\d{1,2}[/-]\d{2,4})", igm_clean, re.I, ) if m: airline = clean(m.group(1)) flight = clean(m.group(2)).upper() airport = clean(m.group(3)).upper() arrival = date_value(m.group(4)) if party_name(airline) and not re.search(r"\b(?:Flight|Airport|Date|Arrival|First Port)\b", airline, re.I): result["cour_be_airline"] = airline if re.fullmatch(r"[A-Z]{1,5}\s*\d{2,6}", flight): result["cour_be_flightno"] = flight if re.fullmatch(r"[A-Z]{3}", airport): result["cour_be_arrivalairport"] = airport if arrival: result["cour_be_arrivaldate"] = arrival result["cour_be_inward_date"] = arrival # HAWB is numeric in the CBE-XIII form; do not accept consignee text. m = re.search(r"HAWB\s+Number\s*[:：-]?\s*(\d{8,20})\b", cbe_norm, re.I) if m: result["cour_be_hawbno"] = m.group(1) # Marks on Packages belongs to the item block and is distinct from # any attached-document "Marks" fields. m = re.search(r"Marks\s+on\s+Packages\s*[:：-]?\s*([A-Za-z0-9][A-Za-z0-9 ./_-]{0,40}?)(?=\s+(?:Unit\s+of\s+Measure|Quantity|Invoice\s+Number|Invoice\s+Value)\b|$)", cbe, re.I) if m: marks = clean(m.group(1)) if marks and not re.search(r"\b(?:Unit|Quantity|Invoice|Currency|Rate)\b", marks, re.I): result["cour_be_marksnos"] = marks # ------------------------------------------------------------- # 2. Invoice row in the CBE itself. # ------------------------------------------------------------- invoices = result.setdefault("invoices", []) if not invoices: invoices.append(self._empty_invoice(1)) inv = invoices[0] invoice_no = labelled(cbe, [r"Invoice\s+Number", r"Invoice\s+No"], [r"Invoice\s+Value", r"Unit\s+Price", r"Currency\s+of\s+Unit\s+Price"], 100) if invoice_no: invoice_no = re.split(r"\s+(?:Invoice|Date|Value|Currency|Unit|Rate)\b", invoice_no, maxsplit=1, flags=re.I)[0].strip() inv["cour_be_invoice_no"] = invoice_no # CBE-XIII uses "Invoice Term", while CBE-XIV uses "Terms of Invoice". incoterm = "" # The CBE-XIII row is normally: Invoice Term : FOB Landing Charges : 0. # Capture only the term token, never the following commercial fields. for ipat in ( r"Invoice\s+Term\s*[:：-]?\s*([A-Za-z]{3}(?:-[A-Za-z][A-Za-z0-9_-]*)?)", r"Terms\s+of\s+Invoice\s*[:：-]?\s*([A-Za-z]{3}(?:-[A-Za-z][A-Za-z0-9_-]*)?)", r"Incoterms?\s*[:：-]?\s*([A-Za-z]{3}(?:-[A-Za-z][A-Za-z0-9_-]*)?)", ): im = re.search(ipat, cbe, re.I) if im: incoterm = clean(im.group(1)) break if incoterm: # Keep the full term when it contains a qualified incoterm such as # FCA-SAINT WITZ. For a CBE value like "FOB" this remains "FOB". incoterm = clean(incoterm) if re.match(r"(?i)^(?:FOB|CIF|CFR|CIP|CPT|DAP|DDP|EXW|FCA|FAS|DAT)\b", incoterm): inv["cour_be_incoterm"] = incoterm.upper() # ------------------------------------------------------------- # 3. Invoice/supplier address authority. # CBE-XIII's consignor block is the preferred supplier source. The # commercial invoice/waybill may provide a more complete address, so # only use it if it is explicitly labelled and is not a CBE courier # address. # ------------------------------------------------------------- if party_name(supplier_name): inv["cour_be_inv_supplier_name"] = supplier_name if address_value(supplier_addr): inv["suppliers_address"] = supplier_addr # ------------------------------------------------------------- # 4. Attached commercial invoice / export paperwork metadata. # Only invoice fields are allowed to consume these later sections. # ------------------------------------------------------------- attached = source # Remove the CBE portion so a CBE "Invoice Term: FOB" remains the # primary term, while the external document can supply invoice date / # payment terms / a more explicit consignor address. first_attached = re.search( r"(?:SUBSIDIARY\s+EXPORT\s+CERTIFICATE|\*WAYBILL\s+DOC\*|DECLARATION\s+D['’]?EXPORTATION|COMMERCIAL\s+INVOICE|INVOICE)", source, re.I, ) if first_attached: attached = source[first_attached.start():] def first_label(patterns, stops, limit=300): for pat in patterns: m = re.search( pat + r"\s*[:：-]?\s*(.*?)(?=\s+(?:" + "|".join(stops) + r")\s*[:：-]|$)", attached, re.I | re.S, ) if m: v = clean(m.group(1)) if v and len(v) <= limit: return v return "" # Date: prioritize an explicitly labelled invoice Date, not export # certificate departure/permit/date-edition fields. # Invoice date: explicit invoice-date labels always win. If the # commercial invoice only says "Date", score each Date occurrence by # the nearby invoice context and reject logistics/customs dates. invoice_date = first_label( [r"Date\s+of\s+Invoice", r"Invoice\s+Date"], [r"Payment\s+Terms", r"Terms\s+of\s+Payment", r"Invoice\s+Term", r"Invoice\s+Number", r"Supplier", r"Shipper", r"Receiver", r"Description"], 80, ) if invoice_date: formatted = date_value(invoice_date) if formatted: inv["cour_be_invoice_date"] = formatted if not inv.get("cour_be_invoice_date"): date_candidates = [] for dm in re.finditer( r"\bDate\b\s*[:：-]?\s*(\d{1,2}[-/.]\w{3,9}[-/.]\d{2,4}|\d{1,2}[-/.]\d{1,2}[-/.]\d{2,4})", attached, re.I, ): raw_date = dm.group(1) formatted = date_value(raw_date) if not formatted: continue context = attached[max(0, dm.start()-500):min(len(attached), dm.end()+500)] score = 0 if re.search(r"\b(?:invoice|commercial\s+invoice|invoice\s+number|invoice\s+term|payment\s+terms)\b", context, re.I): score += 80 if re.search(r"\b(?:supplier|shipper|seller|buyer|receiver|bill[- ]to|sold[- ]to)\b", context, re.I): score += 25 if re.search(r"\b(?:departure\s+date|permit\s+date|date\s+edition|date\s+of\s+validation|date\s+of\s+arrival|date\s+of\s+hawb|date\s+of\s+mawb)\b", context, re.I): score -= 100 date_candidates.append((score, -abs(dm.start()), formatted)) if date_candidates: date_candidates.sort(reverse=True) inv["cour_be_invoice_date"] = date_candidates[0][2] payment = first_label( [r"Payment\s+Terms", r"Terms\s+of\s+Payment", r"Payment\s+Term"], [r"Invoice\s+Term", r"Invoice\s+Date", r"Date", r"Supplier", r"Shipper", r"Receiver", r"Bank", r"Total"], 160, ) if payment and num(payment) is None and not self._looks_like_any_known_label(payment): inv["cour_be_inv_pymt_terms"] = payment # External consignor address, when explicitly labelled. ext_addr = first_label( [r"Address\s+of\s+Consignor", r"Consignor\s+Address", r"Supplier\s+Address", r"Exporter\s+Address"], [r"Name\s+of\s+Consignee", r"Receiver", r"Contact", r"Invoice\s+Number", r"Payment\s+Terms", r"Description", r"Import\s+Export\s+Code"], 500, ) if ext_addr and address_value(ext_addr): if not re.search(r"Ms\s+DHL\s+Express|Express\s+Cargo\s+Terminal|Authorized\s+Courier", ext_addr, re.I): # Prefer an explicit consignor address over a generic/partial # CBE observation when it is longer and structurally valid. if not inv.get("suppliers_address") or len(ext_addr) > len(str(inv.get("suppliers_address"))): inv["suppliers_address"] = ext_addr # If a commercial invoice explicitly says FCA-SAINT WITZ and the CBE # term is absent, use that full term. Otherwise preserve CBE FOB. if not clean(inv.get("cour_be_incoterm")): m = re.search(r"\b(?:Invoice\s+Term|Incoterm|Terms\s+of\s+Delivery)\s*[:：-]?\s*([A-Za-z]{3}(?:-[A-Za-z][A-Za-z -]{2,40})?)\b", attached, re.I) if m: inv["cour_be_incoterm"] = clean(m.group(1)).upper() # Supplier fallback is always the CBE consignor when the invoice child # is otherwise empty. if not party_name(inv.get("cour_be_inv_supplier_name")) and party_name(result.get("cour_be_supplier_name")): inv["cour_be_inv_supplier_name"] = clean(result.get("cour_be_supplier_name")) if not address_value(inv.get("suppliers_address")) and address_value(supplier_addr): inv["suppliers_address"] = supplier_addr result["cour_be_invoicecount"] = len(invoices) def _final_cbe_xiii_label_authority(self, result): """Final CBE-XIII text authority for party/header/invoice fields. This pass deliberately uses only semantic labels and the text between those labels. It does not use absolute coordinates, page numbers, document-specific values, OCR, or neighbouring-column geometry. CBE-XIII is a compact two-column form. pdfplumber can return wrapped values such as: Name of Consignee: WIPRO GE HEALTHCARE PRIVATE LIMITED Address of Consignee: CO DHL SUPPLY CHAIN IN-DIA ... The correct boundary is therefore the next *semantic label*, not the end of the visual line. This pass is intentionally last so no broad extractor can overwrite the repaired value afterward. """ source = str(getattr(self, "full_text", "") or "") if not source or not re.search(r"Courier\s+Bill\s+Of\s+Entry\s*[- ]?XIII", source, re.I): return def clean(v): v = str(v or "") v = v.replace("\u00ad", "").replace("\ufeff", "").replace("\ufffe", "").replace("\uffff", "") # PDF line-wrap hyphens are formatting artifacts. Preserve real # separators such as the underscore in a CBE number. v = re.sub(r"(?<=[A-Za-z])[-\u2010\u2011\u2012\u2013\u2014]\s+(?=[A-Za-z0-9])", "", v) v = re.sub(r"\s+", " ", v) return v.strip(" \t\r\n:;,") def region_after(text, label_pattern, stop_patterns, limit=1200): m = re.search(label_pattern + r"\s*[:：-]?\s*", text, re.I) if not m: return "" tail = text[m.end():m.end() + limit] positions = [] for pat in stop_patterns: sm = re.search(pat, tail, re.I) if sm: positions.append(sm.start()) if positions: tail = tail[:min(positions)] return clean(tail) def valid_party(v): if not v or len(v) > 180: return False if self._looks_like_any_known_label(v): return False return bool(re.search(r"[A-Za-z]{2,}", v)) def valid_address(v): if not v or len(v) > 600: return False if re.search( r"\b(?:Import\s+Export\s+Code|IEC\s+Branch\s+Code|Special\s+Request|" r"No\s+of\s+Packages|Gross\s+Weight|Net\s+Weight|Assessable\s+Value|" r"Duty\s*\(Rs\.?\)|Invoice\s+Value|KYC\s+Document|State\s+Code|AD\s+Code)\b", v, re.I, ): return False return bool(re.search(r"[A-Za-z]{2,}", v)) # Work only inside the first CBE-XIII form section. Attached waybill / # export paperwork is deliberately excluded from these identity fields. form = re.search(r"Form\s+Courier\s+Bill\s+Of\s+Entry\s*[- ]?XIII", source, re.I) if not form: return cbe = source[form.start():] page_end = re.search(r"Page\s+1\s+of\s+\d+", cbe, re.I) if page_end: cbe = cbe[:page_end.end()] attached = re.search( r"(?:\*WAYBILL\s+DOC\*|SUBSIDIARY\s+EXPORT\s+CERTIFICATE|" r"DECLARATION\s+D['’]?EXPORTATION|COMMERCIAL\s+INVOICE)", cbe, re.I, ) if attached: cbe = cbe[:attached.start()] # ------------------------------------------------------------- # CBE-XIII number. The value can be wrapped after the final digits. # Remove whitespace *only inside this identifier*. # ------------------------------------------------------------- m = re.search( r"CBE\s*[- ]?XIII\s+Number\s*[:：-]?\s*(CBEXIII_[A-Z0-9_-]+(?:\s+[A-Z0-9_-]+)*?)(?=\s+Name\s+of\s+the\s+Authorized\s+Courier|\s+Address\s+of\s+Authorized\s+Courier|\s+IGM\s+DETAILS|$)", cbe, re.I, ) if m: cbe_no = re.sub(r"\s+", "", m.group(1)) cbe_no = re.sub(r"[^A-Za-z0-9_-]", "", cbe_no) if re.fullmatch(r"CBEXIII_[A-Za-z0-9_-]{8,}", cbe_no, re.I): result["cour_be_beno"] = cbe_no # ------------------------------------------------------------- # Authorized courier. Stop at the address label, so "Limited" on # the wrapped next line remains part of the courier name. # ------------------------------------------------------------- courier = region_after( cbe, r"Name\s+of\s+the\s+Authorized\s+Courier", [r"Address\s+of\s+Authorized\s+Courier", r"IGM\s+DETAILS"], 220, ) courier = re.sub(r"^Courier\s*:\s*", "", courier, flags=re.I) if valid_party(courier) and not re.search( r"\b(?:Address|IGM|Airlines|Flight|Airport|Date|Time)\b", courier, re.I ): result["cour_be_couriername"] = courier # ------------------------------------------------------------- # IGM row. The airline may wrap on a line by itself, so collect the # next few lines until a row containing flight + airport + date is found. # ------------------------------------------------------------- igm_header = re.search( r"Airlines\s+Flight\s+No\.?\s+Airport\s+Of\s+Arrival.*?Date\s+Of\s+Arrival(?:\s+Time\s+Of\s+Arrival)?", cbe, re.I | re.S, ) if igm_header: tail = cbe[igm_header.end():] tail = tail[:min([ x.start() for x in [ re.search(r"Airport\s+of\s+Shipment\s*:", tail, re.I), re.search(r"HAWB\s+Number\s*:", tail, re.I), ] if x ] or [len(tail)])] # Normalize only line-wrap formatting; retain line boundaries long # enough to distinguish an airline line from the flight row. lines = [clean(x) for x in tail.splitlines() if clean(x)] row = " ".join(lines[:6]) fm = re.search( r"^(.*?)\s+([A-Z0-9]{1,6}\s*\d{2,8})\s+([A-Z]{2,4})\s+(?:N/?A\s+)?" r"(\d{1,2}[/-]\d{1,2}[/-]\d{2,4})", row, re.I, ) if fm: airline = clean(fm.group(1)) # Do not allow header text into the airline. airline = re.sub( r"\b(?:First\s+Port\s+Of\s+Arrival|Date\s+Of\s+Arrival|Time\s+Of\s+Arrival)\b.*$", "", airline, flags=re.I, ).strip() if valid_party(airline): result["cour_be_airline"] = airline result["cour_be_flightno"] = clean(fm.group(2)).upper() result["cour_be_arrivalairport"] = fm.group(3).upper() try: result["cour_be_arrivaldate"] = datetime.strptime( fm.group(4), "%d/%m/%Y" ).strftime("%Y-%m-%d") except ValueError: try: result["cour_be_arrivaldate"] = datetime.strptime( fm.group(4), "%d-%m-%Y" ).strftime("%Y-%m-%d") except ValueError: pass # ------------------------------------------------------------- # Exact CBE-XIII party blocks. These are the strongest possible # boundaries and fix both truncation and cross-column interleaving. # ------------------------------------------------------------- supplier = region_after( cbe, r"Name\s+of\s+Consignor", [r"Address\s+of\s+Consignor", r"Name\s+of\s+Consignee"], 220, ) supplier_addr = region_after( cbe, r"Address\s+of\s+Consignor", [r"Name\s+of\s+Consignee", r"Address\s+of\s+Consignee", r"Import\s+Export\s+Code"], 650, ) importer = region_after( cbe, r"Name\s+of\s+Consignee", [r"Address\s+of\s+Consignee", r"Import\s+Export\s+Code"], 220, ) importer_addr = region_after( cbe, r"Address\s+of\s+Consignee", [r"Import\s+Export\s+Code", r"IEC\s+Branch\s+Code", r"Special\s+Request", r"No\s+of\s+Packages"], 650, ) if valid_party(supplier): result["cour_be_supplier_name"] = supplier if valid_address(supplier_addr): if not result.get("invoices"): result["invoices"] = [self._empty_invoice(1)] result["invoices"][0]["suppliers_address"] = supplier_addr if valid_party(importer): result["cour_be_importername"] = importer if valid_address(importer_addr): result["cour_be_importer_address"] = importer_addr if result.get("invoices"): inv = result["invoices"][0] if valid_party(supplier): inv["cour_be_inv_supplier_name"] = supplier if valid_address(supplier_addr): inv["suppliers_address"] = supplier_addr # ------------------------------------------------------------- # Item-level marks and invoice term are also explicit CBE-XIII labels. # ------------------------------------------------------------- marks = region_after( cbe, r"Marks\s+on\s+Packages", [r"Unit\s+of\s+Measure", r"Quantity", r"Invoice\s+Number"], 80, ) if marks and not re.search(r"\b(?:Unit|Quantity|Invoice|Currency|Rate)\b", marks, re.I): result["cour_be_marksnos"] = marks invs = result.setdefault("invoices", []) if not invs: invs.append(self._empty_invoice(1)) inv = invs[0] invoice_no = region_after( cbe, r"Invoice\s+Number", [r"Invoice\s+Value", r"Unit\s+Price", r"Currency\s+of\s+Unit\s+Price", r"Invoice\s+Term"], 100, ) if invoice_no: invoice_no = re.split(r"\s+(?:Invoice|Value|Currency|Unit|Rate|Term)\b", invoice_no, 1, flags=re.I)[0] inv["cour_be_invoice_no"] = clean(invoice_no) term = region_after( cbe, r"Invoice\s+Term", [r"Landing\s+Charges", r"Insurance", r"Freight", r"Discount\s+Amount", r"Assessable\s+Value"], 120, ) if term: term = clean(term) # CBE-XIII term is normally the first token/qualified token before # the next charge label. tm = re.match(r"((?:FOB|CIF|CFR|CIP|CPT|DAP|DDP|EXW|FCA|FAS|DAT)(?:-[A-Za-z][A-Za-z0-9 _-]*)?)", term, re.I) if tm: inv["cour_be_incoterm"] = clean(tm.group(1)).upper() # ------------------------------------------------------------- # Commercial invoice metadata. This is intentionally separate from # CBE identity. Search the full attached-document tail for explicit # Payment Terms and Date labels. A month-name date gets priority over # customs/logistics numeric dates. # ------------------------------------------------------------- attached_tail = source cbe_end = re.search(r"Page\s+1\s+of\s+\d+", source, re.I) if cbe_end: attached_tail = source[cbe_end.end():] payment_matches = re.findall( r"(?:Payment\s+Terms|Terms\s+of\s+Payment|Payment\s+Term)\s*[:：-]\s*([^\n\r]+)", attached_tail, re.I, ) for value in payment_matches: value = clean(value) if value and not self._looks_like_any_known_label(value): inv["cour_be_inv_pymt_terms"] = value break # Explicit commercial invoice date formats, including 31-JUL-2026. date_candidates = [] for dm in re.finditer( r"\bDate\s*[:：-]\s*(\d{1,2}[-/.](?:[A-Za-z]{3,9}|\d{1,2})[-/.]\d{2,4})", attached_tail, re.I, ): raw = clean(dm.group(1)) try: parsed = None for fmt in ( "%d-%b-%Y", "%d-%B-%Y", "%d/%b/%Y", "%d/%B/%Y", "%d-%m-%Y", "%d/%m/%Y", "%d.%m.%Y", ): try: parsed = datetime.strptime(raw, fmt).strftime("%Y-%m-%d") break except ValueError: pass if not parsed: continue context = attached_tail[max(0, dm.start()-600):dm.end()+600] score = 0 if re.search(r"\b(?:commercial\s+invoice|invoice|payment\s+terms|invoice\s+term)\b", context, re.I): score += 100 if re.search(r"\b(?:departure|permit|arrival|hawb|mawb|validation|date\s+edition)\b", context, re.I): score -= 100 date_candidates.append((score, parsed)) except Exception: continue if date_candidates: date_candidates.sort(key=lambda x: x[0], reverse=True) inv["cour_be_invoice_date"] = date_candidates[0][1] # If the commercial invoice explicitly provides an incoterm and the # CBE did not, accept the qualified term (e.g. FCA-SAINT WITZ). if not clean(inv.get("cour_be_incoterm")): im = re.search( r"(?:Invoice\s+Term|Incoterm(?:s)?|Terms\s+of\s+Delivery)\s*[:：-]\s*" r"((?:FOB|CIF|CFR|CIP|CPT|DAP|DDP|EXW|FCA|FAS|DAT)(?:-[A-Za-z][A-Za-z0-9 _-]{2,60})?)", attached_tail, re.I, ) if im: inv["cour_be_incoterm"] = clean(im.group(1)).upper() # Never let a legacy pass reintroduce a partial party/address after # this authority has established the exact CBE-XIII blocks. if valid_party(supplier): inv["cour_be_inv_supplier_name"] = supplier if valid_address(supplier_addr): inv["suppliers_address"] = supplier_addr if valid_party(importer): result["cour_be_importername"] = importer if valid_address(importer_addr): result["cour_be_importer_address"] = importer_addr result["cour_be_invoicecount"] = len(result.get("invoices") or []) # ================================================================ # V5 CBE-XIII TABLE AUTHORITY # ================================================================ def _v5_clean_table_text(self, value): """Normalize a pdfplumber table cell without destroying its content.""" value = str(value or "") value = value.replace("\u00ad", "").replace("\ufeff", "") value = value.replace("\u2010", "-").replace("\u2011", "-") value = value.replace("\u2012", "-").replace("\u2013", "-").replace("\u2014", "-") lines = [re.sub(r"\s+", " ", x).strip() for x in value.splitlines() if x.strip()] if not lines: return "" out = "" for line in lines: if not out: out = line continue # A PDF soft line-wrap hyphen belongs to the next line. if out.endswith("-") and line[:1].isalnum(): out = out[:-1] + line else: out += " " + line return re.sub(r"\s+", " ", out).strip(" \t\r\n:;,") def _v5_compact(self, value): return re.sub(r"[^a-z0-9]+", "", self._v5_clean_table_text(value).lower()) def _v5_table_find(self, predicate): """Return CBE-XIII tables using their own contents, not page numbers.""" matches = [] for table in getattr(self, "tables", []) or []: rows = table.get("rows") if isinstance(table, dict) else None if not rows: continue compact_all = self._v5_compact( " ".join( " ".join(str(c or "") for c in row) for row in rows[:12] ) ) if "formcourierbillofentryxiiicbexiii" not in compact_all: continue if predicate(rows): matches.append(table) return matches def _v5_nonempty_cells(self, row): return [ (idx, self._v5_clean_table_text(cell)) for idx, cell in enumerate(row or []) if self._v5_clean_table_text(cell) ] def _v5_row_value_after_label(self, row, aliases): """Read the value cell belonging to a label cell in the same table row.""" aliases_c = [self._v5_compact(a) for a in aliases] cells = list(row or []) for idx, cell in enumerate(cells): c = self._v5_compact(cell) if not c: continue if not any(c == a or a in c for a in aliases_c): continue for j in range(idx + 1, len(cells)): value = self._v5_clean_table_text(cells[j]) if value: return value return "" def _v5_row_by_label(self, rows, aliases): aliases_c = [self._v5_compact(a) for a in aliases] for row in rows: row_c = [self._v5_compact(c) for c in row] if any( c and any(c == a or a in c for a in aliases_c) for c in row_c ): return row return None def _v5_reconstruct_cell_from_words( self, page_no, raw_value, xright=None, y_hint=None, y_window=75, ): """ Reconstruct spacing from the original page words while using the table cell's text as the exact target. This is not a fixed bounding box. The start word, vertical location, and optional right boundary are all derived from the current PDF's table/value itself. """ target = self._v5_compact(raw_value) if not target: return "" page_words = list(getattr(self, "page_words", {}).get(page_no, []) or []) if not page_words: return self._v5_clean_table_text(raw_value) candidates = [] for word in page_words: wt = self._v5_compact(word.get("_text", word.get("text", ""))) if len(wt) < 2 or not target.startswith(wt): continue if y_hint is not None and abs(float(word.get("y0", word.get("top", 0))) - y_hint) > 35: continue candidates.append(word) def join_words(sequence): result = "" for word in sequence: t = str(word.get("_text", word.get("text", ""))) if not t: continue if not result: result = t elif result.endswith("-") and t[:1].isalnum(): result = result[:-1] + t else: result += " " + t return re.sub(r"\s+", " ", result).strip() for start in candidates: sy = float(start.get("y0", start.get("top", 0))) sx = float(start.get("x0", 0)) left = sx - 3.0 # If no explicit boundary was supplied, keep the search in the # current value column by stopping at the nearest right-side # label on the same visual line. This prevents adjacent columns # from being injected into wrapped values. boundary = xright if boundary is None: same_row = [ w for w in page_words if abs(float(w.get("y0", w.get("top", 0))) - sy) <= 2.5 and float(w.get("x0", 0)) > float(start.get("x1", sx)) ] # Use a known semantic label as a natural column boundary. label_tokens = { self._v5_compact(a).split(" ")[0] for group in ( getattr(self, "PARENT_ALIASES", {}).values(), getattr(self, "INVOICE_ALIASES", {}).values(), getattr(self, "ITEM_ALIASES", {}).values(), ) for aliases in group for a in aliases if self._v5_compact(a) } # The compact form above has no spaces, so also compare the # first lexical token from the raw word. for w in sorted(same_row, key=lambda z: float(z.get("x0", 0))): token = self._v5_compact(w.get("_text", w.get("text", ""))) if token in label_tokens or token in { "address", "flight", "airport", "date", "time", "country", "name", "iec", "special", "no", "gross", "net", "assessable", "duty", "invoice", }: boundary = float(w.get("x0", 0)) break band = [ w for w in page_words if sy - 1.0 <= float(w.get("y0", w.get("top", 0))) <= sy + y_window and float(w.get("x0", 0)) >= left and (boundary is None or float(w.get("x0", 0)) < boundary) ] band.sort( key=lambda z: ( float(z.get("y0", z.get("top", 0))), float(z.get("x0", 0)), ) ) try: start_index = next(i for i, w in enumerate(band) if w is start) except StopIteration: continue accumulated = "" sequence = [] for w in band[start_index:start_index + 40]: token = self._v5_compact(w.get("_text", w.get("text", ""))) if not token: continue trial = accumulated + token if not target.startswith(trial): break accumulated = trial sequence.append(w) if accumulated == target: return join_words(sequence) return self._v5_clean_table_text(raw_value) def _v5_find_word_start(self, page_no, raw_value, y_hint=None): target = self._v5_compact(raw_value) if not target: return None best = None for w in getattr(self, "page_words", {}).get(page_no, []) or []: token = self._v5_compact(w.get("_text", w.get("text", ""))) if len(token) < 2 or not target.startswith(token): continue y = float(w.get("y0", w.get("top", 0))) if y_hint is not None: distance = abs(y - y_hint) if distance > 35: continue else: distance = y score = (distance, y, float(w.get("x0", 0))) if best is None or score < best[0]: best = (score, w) return best[1] if best else None def _v5_cell_value(self, page_no, row, value_col, label_col=None, label_aliases=None): raw = "" if row and value_col < len(row): raw = self._v5_clean_table_text(row[value_col]) if not raw: return "" # Determine a visual starting word from the exact cell contents. start = self._v5_find_word_start(page_no, raw) if not start: return raw xright = None if label_col is not None and label_aliases: # The paired label is normally immediately before the value. # Find its rightmost word near the value start. label_tokens = { self._v5_compact(a).split()[0] for a in label_aliases if self._v5_compact(a) } sy = float(start.get("y0", start.get("top", 0))) right_candidates = [] for w in getattr(self, "page_words", {}).get(page_no, []) or []: wx = float(w.get("x0", 0)) wy = float(w.get("y0", w.get("top", 0))) token = self._v5_compact(w.get("_text", w.get("text", ""))) if wx >= float(start.get("x0", 0)): continue if abs(wy - sy) > 3.5: continue if token in label_tokens or token in {"address", "name", "courier", "number"}: continue # Find the first known label word to the right of the value on # the same line. This is the actual column boundary. for w in sorted( getattr(self, "page_words", {}).get(page_no, []) or [], key=lambda z: float(z.get("x0", 0)), ): wx = float(w.get("x0", 0)) wy = float(w.get("y0", w.get("top", 0))) if wx <= float(start.get("x1", start.get("x0", 0))): continue if abs(wy - sy) > 3.5: continue token = self._v5_compact(w.get("_text", w.get("text", ""))) if token in { "address", "name", "courier", "flight", "airport", "date", "time", "country", "iec", "special", "no", "gross", "net", "assessable", "duty", "invoice", }: xright = wx break return self._v5_reconstruct_cell_from_words( page_no, raw, xright=xright, y_hint=float(start.get("y0", 0)) ) def _final_cbe_xiii_table_authority(self, result): """ Final CBE-XIII authority. pdfplumber's detected CBE-XIII table preserves the logical cells even when the PDF text stream interleaves the two columns. The table is therefore the authoritative source for CBE-XIII header/party fields. The table itself is discovered by its form heading; no page number, document-specific coordinates, or sample values are used. """ tables = self._v5_table_find(lambda rows: any( "airlines" in self._v5_compact(row[0] if row else "") and "flightno" in self._v5_compact(row[1] if len(row) > 1 else "") for row in rows )) if not tables: return table = tables[0] rows = table.get("rows") or [] page_no = int(table.get("page") or 0) if not page_no: return def row_for(*aliases): return self._v5_row_by_label(rows, list(aliases)) def value_for(*aliases): row = row_for(*aliases) if not row: return "" aliases_c = [self._v5_compact(a) for a in aliases] for idx, cell in enumerate(row): cc = self._v5_compact(cell) if not cc: continue if any(cc == a or a in cc for a in aliases_c): for j in range(idx + 1, len(row)): if self._v5_clean_table_text(row[j]): return self._v5_clean_table_text(row[j]) return "" # CBE-XIII number is an identifier, so remove only visual whitespace. beno = value_for("CBE-XIII NUMBER", "CBEXIII NUMBER") if beno: beno = re.sub(r"\s+", "", beno) beno = re.sub(r"(?i)^CBEXIII_", "CBEXIII_", beno) if re.fullmatch(r"CBEXIII_[A-Za-z0-9_-]{8,}", beno): result["cour_be_beno"] = beno # Locate the exact logical cells and reconstruct their spaces from # the page words. The table cell remains the target string. pairs = [ ( "cour_be_couriername", ("NAME OF THE AUTHORIZED COURIER", "NAME OF AUTHORIZED COURIER"), ("ADDRESS OF AUTHORIZED COURIER",), 2, 0, ), ( "cour_be_supplier_name", ("NAME OF CONSIGNOR", "CONSIGNOR NAME"), ("ADDRESS OF CONSIGNOR",), 2, 0, ), ( "cour_be_importername", ("NAME OF CONSIGNEE", "CONSIGNEE NAME"), ("ADDRESS OF CONSIGNEE",), 2, 0, ), ] for field, labels, paired_labels, value_col, label_col in pairs: row = row_for(*labels) if not row or value_col >= len(row): continue raw = self._v5_clean_table_text(row[value_col]) if not raw: continue # For these rows the value is in the third logical cell. The # paired address label gives the dynamic right boundary. value = self._v5_reconstruct_cell_from_words(page_no, raw) if value: result[field] = value # Addresses are the right-hand value cells. for field, labels in [ ("cour_be_importer_address", ("ADDRESS OF CONSIGNEE",)), ]: row = row_for(*labels) if row and len(row) > 7: raw = self._v5_clean_table_text(row[7]) value = self._v5_reconstruct_cell_from_words(page_no, raw) if value: result[field] = value # Supplier address. row = row_for("ADDRESS OF CONSIGNOR") if row and len(row) > 7: raw = self._v5_clean_table_text(row[7]) value = self._v5_reconstruct_cell_from_words(page_no, raw) if value: # Never allow the supplier-name tail into the address. supplier = self._v5_clean_table_text(row[2]) if len(row) > 2 else "" supplier_reconstructed = self._v5_reconstruct_cell_from_words(page_no, supplier) if supplier_reconstructed: value = re.sub( r"\b" + re.escape( " ".join(supplier_reconstructed.split()[-2:]) ) + r"\b", "", value, flags=re.I, ).strip() result.setdefault("invoices", []) result["cour_be_supplier_name"] = supplier_reconstructed or result.get("cour_be_supplier_name", "") result["cour_be_inv_supplier_name"] = result.get("cour_be_supplier_name", "") result["invoices"][0]["cour_be_inv_supplier_name"] = result["cour_be_supplier_name"] if result["invoices"] else "" if value: result["invoices"][0]["suppliers_address"] = value if result["invoices"] else "" result["cour_be_supplier_name"] = supplier_reconstructed or result.get("cour_be_supplier_name", "") # Country of Origin authority: for CBE-XIII, the header-level # ``cour_be_coo`` field represents the actual origin of the goods, # which is printed inside DETAILS OF ITEM. Do not use the nearby # IGM-level ``Country of Exportation`` value for this field. Locate # the item section and its explicit Country of Origin label # dynamically from the detected CBE-XIII table; no page number, # coordinate, or document-specific country value is used. item_section = None for idx, row in enumerate(rows): if any(self._v5_compact(c) == "detailsofitem" for c in row): item_section = idx break if item_section is not None: item_coo = "" for row in rows[item_section + 1:]: labels = [self._v5_compact(c) for c in row] for i, cell in enumerate(row): if labels[i] == "countryoforigin": for j in range(i + 1, len(row)): candidate = self._v5_clean_table_text(row[j]) if candidate and self._country_validator(candidate): item_coo = candidate break if item_coo: break if item_coo: break if item_coo: item_coo = self._v5_reconstruct_cell_from_words(page_no, item_coo) or item_coo if self._country_validator(item_coo): result["cour_be_coo"] = item_coo # IGM data row immediately follows the semantic IGM header row. for idx, row in enumerate(rows): c = self._v5_compact(" ".join(row)) if "airlines" not in c or "flightno" not in c or "airportofarrival" not in c: continue if idx + 1 >= len(rows): break data = rows[idx + 1] if len(data) < 7: break flight = self._v5_clean_table_text(data[1]).replace(" ", "").upper() flight_start = self._v5_find_word_start(page_no, data[1]) flight_x = ( float(flight_start.get("x0", 0)) if flight_start else None ) airline = self._v5_reconstruct_cell_from_words( page_no, data[0], xright=flight_x, ) airport = self._v5_clean_table_text(data[3]).upper() arrival = self._v5_clean_table_text(data[6]) if airline: result["cour_be_airline"] = airline if re.fullmatch(r"[A-Z0-9]{1,6}\d{2,8}", flight): result["cour_be_flightno"] = flight if re.fullmatch(r"[A-Z]{2,5}", airport): result["cour_be_arrivalairport"] = airport parsed_date = self._v183_date(arrival) if parsed_date: result["cour_be_arrivaldate"] = parsed_date result["cour_be_inward_date"] = parsed_date break # Marks and invoice item values from the exact CBE-XIII cells. row = row_for("MARKS ON PACKAGES") if row and len(row) > 7: marks = self._v5_clean_table_text(row[7]) if marks: result["cour_be_marksnos"] = marks row = row_for("RATE OF EXCHANGE") if row and len(row) > 7: rate = self._v5_clean_table_text(row[7]) n = self._v183_numeric(rate) if n is not None and result.get("items"): result["items"][0]["cour_be_item_exchg_rate"] = float(n) row = row_for("INVOICE TERM") if row and len(row) > 2: term = self._v5_clean_table_text(row[2]) if term: for inv in result.get("invoices") or []: inv["cour_be_incoterm"] = term.upper() row = row_for("INVOICE NUMBER") if row and len(row) > 2: inv_no = self._v5_clean_table_text(row[2]) if inv_no: for inv in result.get("invoices") or []: inv["cour_be_invoice_no"] = inv_no row = row_for("CURRENCY OF INVOICE") if row and len(row) > 2: cur = self._v5_clean_table_text(row[2]) if cur: for inv in result.get("invoices") or []: inv["cour_be_inv_currency"] = cur.upper() def _final_invoice_text_authority_v5(self, result): """ Invoice-only fallback for text-based commercial invoice pages. It intentionally does not OCR. If a commercial invoice is an image-only page and the analyzer supplies no text for it, these fields cannot be recovered without an OCR-capable analyzer. """ invoices = result.get("invoices") or [] if not invoices: return inv = invoices[0] full = str(getattr(self, "full_text", "") or "") if not full: return def clean(v): v = str(v or "").replace("\u00ad", " ") v = re.sub(r"\s+", " ", v).strip(" :;\t\r\n") return v def fmt_date(raw): try: return self._v183_date(raw) except Exception: try: return self._format_date(raw) except Exception: return "" # Prefer explicit invoice labels. date_patterns = [ r"\bDate\s+of\s+Invoice\s*[:\-]?\s*(\d{1,2}[-/.]\s*[A-Za-z]{3,9}[-/.]\s*\d{2,4})", r"\bInvoice\s+Date\s*[:\-]?\s*(\d{1,2}[-/.]\s*[A-Za-z]{3,9}[-/.]\s*\d{2,4})", ] if not clean(inv.get("cour_be_invoice_date")): for pat in date_patterns: m = re.search(pat, full, re.I) if m: d = fmt_date(m.group(1)) if d: inv["cour_be_invoice_date"] = d break # Commercial invoices often use a bare "Date:" field. Score only # dates occurring near invoice-specific anchors and reject customs/ # logistics date contexts. if not clean(inv.get("cour_be_invoice_date")): candidates = [] for m in re.finditer( r"\bDate\s*[:\-]?\s*(\d{1,2}[-/.]\s*[A-Za-z]{3,9}[-/.]\s*\d{2,4}|\d{1,2}[-/.]\d{1,2}[-/.]\d{2,4})", full, re.I, ): d = fmt_date(m.group(1)) if not d: continue ctx = full[max(0, m.start()-600):m.end()+600] score = 0 if re.search(r"\b(?:invoice|commercial invoice|invoice no|invoice number|payment terms|incoterm)\b", ctx, re.I): score += 100 if re.search(r"\b(?:waybill|hawb|mawb|flight|arrival|departure|permit|validation|edition|challan)\b", ctx, re.I): score -= 80 candidates.append((score, d)) if candidates: candidates.sort(key=lambda x: x[0], reverse=True) if candidates[0][0] > 0: inv["cour_be_invoice_date"] = candidates[0][1] if not clean(inv.get("cour_be_inv_pymt_terms")): payment_patterns = [ r"\bPayment\s+Terms?\s*[:\-]?\s*([A-Za-z][A-Za-z0-9 ]{2,80}?)(?=\s+(?:Incoterm|Invoice\s+Date|Date|Invoice\s+No|Invoice\s+Number|Total|Currency|Description)\b|$)", r"\bTerms\s+of\s+Payment\s*[:\-]?\s*([A-Za-z][A-Za-z0-9 ]{2,80}?)(?=\s+(?:Incoterm|Invoice\s+Date|Date|Invoice\s+No|Invoice\s+Number|Total|Currency|Description)\b|$)", ] for pat in payment_patterns: m = re.search(pat, full, re.I | re.S) if m: value = clean(m.group(1)) if value and not self._looks_like_any_known_label(value) and self._v183_numeric(value) is None: inv["cour_be_inv_pymt_terms"] = value break def _final_v7_spaced_cbe_recovery(self, result): """ Final CBE-XIII recovery using the analyzer's page text as a second representation of the same document. Some pdfplumber settings weld glyphs in table cells. When a normal text representation is available, use it to restore word boundaries. Continuation lines are matched against the already-known compact table value, so only the missing suffix is appended. This is layout-relative and does not contain sample values. """ if not re.search(r"CBE\s*-?\s*XIII|CBEXIII", self.full_text, re.I): return def clean(v): v=self._clean(v) v=re.sub(r"[\u00ad\ufeff\ufffe\uffff]", "", v) return re.sub(r"\s+"," ",v).strip(" :;,-") def compact(v): return re.sub(r"[^A-Za-z0-9]+","",str(v or "")).upper() def repair_hyphen(v): v=clean(v) return re.sub(r"(?<=[A-Za-z])-\s*(?=[A-Za-z])","",v) lines=[] raw_lines=[] for page in self.pages: for raw in str(page.get("text") or "").splitlines(): raw_value=str(raw or "").strip() v=repair_hyphen(raw_value) if v: lines.append(v) raw_lines.append(raw_value) def continuation_for(compact_target, first_value): remaining=compact_target[len(compact(first_value)):] if not remaining: return "" for line in lines: lc=compact(line) if not lc: continue # Find a prefix of the line which is exactly the missing # compact suffix. Longest prefix wins. best="" for n in range(min(len(remaining),len(lc)),2,-1): if lc[:n] == remaining[:n]: best=line break if best: # Return only the token prefix which contributes to the # target, not the adjacent right-hand column. chars=0 out=[] target_len=len(remaining) for tok in best.split(): tc=compact(tok) if not tc: continue if chars + len(tc) <= target_len: out.append(tok) chars += len(tc) if chars == target_len: return clean(" ".join(out)) else: break return "" # Name of Consignee. target=clean(result.get("cour_be_importername")) if target: tc=compact(target) # Find the best explicit same-line value after the label. for line in lines: m=re.search( r"Name\s+of\s+Consignee\s*:\s*(.*?)(?=\s+Address\s+of\s+Consignee\s*:|$)", line,re.I, ) if not m: continue first=clean(m.group(1)) if not first: continue fc=compact(first) if tc.startswith(fc) and len(fc) < len(tc): tail=continuation_for(tc,first) if tail: candidate=clean(first+" "+tail) if compact(candidate)==tc: result["cour_be_importername"]=candidate elif tc in fc: result["cour_be_importername"]=first break # Name of Consignor. The explicit same-line value is preferred over # any compact table token. target=clean(result.get("cour_be_supplier_name")) if target: tc=compact(target) for line in lines: m=re.search( r"Name\s+of\s+Consignor\s*:\s*(.*?)(?=\s+Address\s+of\s+Consignor\s*:|$)", line,re.I, ) if not m: continue first=clean(m.group(1)) fc=compact(first) if fc and tc.startswith(fc): tail=continuation_for(tc,first) candidate=clean(first+" "+tail) if tail else first if compact(candidate)==tc: result["cour_be_supplier_name"]=candidate elif tc in fc: result["cour_be_supplier_name"]=first break # Authorized courier. Its legal-name continuation is on the same # visual row in some CBE-XIII layouts. target=clean(result.get("cour_be_couriername")) if target: tc=compact(target) for line in lines: m=re.search( r"Name\s+of\s+the\s+Authorized\s+Courier\s*:?\s*(.*?)(?=\s+Address\s+of\s+Authorized\s+Courier\s*:|$)", line,re.I, ) if not m: continue first=clean(m.group(1)) fc=compact(first) if fc and tc.startswith(fc): tail=continuation_for(tc,first) candidate=clean(first+" "+tail) if tail else first if compact(candidate)==tc: result["cour_be_couriername"]=candidate elif tc in fc: result["cour_be_couriername"]=first break # Airline. Read the IGM data row from the unrepaired source text so # a PDF line-break hyphen remains available for continuation detection. raw_all_lines=[] for page in self.pages: raw_all_lines.extend( [str(x or "").strip() for x in str(page.get("text") or "").splitlines() if str(x or "").strip()] ) for i, raw_header in enumerate(raw_all_lines): if not re.search(r"Airlines\s+Flight\s+No\.\s+Airport\s+Of\s+Arrival", raw_header, re.I): continue if i+1 >= len(raw_all_lines): continue raw_data=raw_all_lines[i+1] fm=re.search( r"^(.*?)\s+([A-Z0-9]{1,4}\s*\d{2,8})\s+([A-Z]{2,5})\s+" r"([A-Z0-9/.-]+)\s+(\d{1,2}[/-]\d{1,2}[/-]\d{2,4})", raw_data, re.I, ) if not fm: continue raw_airline=fm.group(1).strip() airline=clean(raw_airline) flight=re.sub(r"\s+"," ",fm.group(2)).upper() if raw_airline.endswith("-"): airline=clean(raw_airline[:-1]) if i+2 < len(raw_all_lines): continuation=clean(raw_all_lines[i+2]) # Only consume a continuation consisting of alphabetic # text; the following row is then outside the airline cell. if continuation and re.fullmatch(r"[A-Za-z][A-Za-z .'-]{1,80}", continuation): airline=clean(airline+continuation) if airline and self._strict_airline(airline): result["cour_be_airline"]=airline if self._flight_validator(flight): result["cour_be_flightno"]=flight break # Re-propagate invoice supplier after the parent value is final. supplier = clean(result.get("cour_be_supplier_name")) if supplier: for inv in result.get("invoices") or []: inv["cour_be_inv_supplier_name"] = supplier def _final_v6_contract_authority(self, result): """ Final contract-level cleanup and dynamic recovery. This pass deliberately runs last. It only: * removes fields that are outside the Courier parent schema; * normalizes flight identifiers into `` ``; * repairs address country word-splits using the country already extracted from this document; * prefers complete party/address observations from other pages of the same document when they are semantically identical; * recovers invoice Date/Payment Terms from text/table evidence when such evidence exists in the analyzer output. No OCR, fixed page number, fixed coordinate, or document-specific value is used. """ def clean(v): v = self._clean(v) v = v.replace("\u00ad", "").replace("\ufeff", "") v = re.sub(r"\s+", " ", v).strip(" :;\t\r\n") return v def compact(v): return re.sub(r"[^A-Za-z0-9]+", "", str(v or "")).upper() # -------------------------------------------------------------- # 1. Parent/child schema boundary: this field is invoice-only and # must NEVER exist at courier_bill_json root level. # -------------------------------------------------------------- result.pop("cour_be_inv_supplier_name", None) # -------------------------------------------------------------- # 2. Flight formatting. The semantic value remains the same; only # the delimiter required by the JSON contract is restored. # BA0119 -> BA 0119 # 3S0735 -> 3S 0735 # -------------------------------------------------------------- flight = clean(result.get("cour_be_flightno")) if flight: flight = re.sub(r"[\s\u00a0]+", "", flight).upper() m = re.fullmatch(r"([A-Z]{1,3})(\d{2,8})", flight) if m: result["cour_be_flightno"] = f"{m.group(1)} {m.group(2)}" else: m = re.fullmatch(r"([A-Z0-9]{1,4})[\s-]*(\d{2,8})", flight) if m: result["cour_be_flightno"] = f"{m.group(1)} {m.group(2)}" # -------------------------------------------------------------- # 3. Generic address repair for a country already present in the # document. This is intentionally derived from extracted document # evidence rather than a hard-coded country list. # -------------------------------------------------------------- country = clean(result.get("cour_be_countryofconsignment")) if country: country_compact = compact(country) for inv in result.get("invoices") or []: addr = clean(inv.get("suppliers_address")) if not addr: continue # A PDF column/text-order artifact can turn: # SINGAPORE SINGAPORE -> SINGAPO RESINGAPORE # or split the first occurrence of the country. Normalize # only when the compact prefix is demonstrably the country. for cut in range(len(country_compact) - 1, 2, -1): left = country_compact[:cut] right = country_compact[cut:] if not left or not right: continue pattern = re.compile( re.escape(left) + r"\s+" + re.escape(right), re.I, ) if pattern.search(addr): addr = pattern.sub(country, addr, count=1) break inv["suppliers_address"] = addr # -------------------------------------------------------------- # 4. Same-document semantic evidence is stronger than a compact # table token. Prefer a complete spaced occurrence for supplier and # consignee when the compact strings match. # -------------------------------------------------------------- full = str(getattr(self, "full_text", "") or "") spaced_text = re.sub(r"[\u00ad]", "", full) lines = [clean(x) for x in spaced_text.splitlines() if clean(x)] def best_same_document_occurrence(target): target = clean(target) if not target: return "" tc = compact(target) if not tc: return "" candidates = [] for line in lines: lc = compact(line) if not lc: continue if tc in lc: # A line can contain a label and a value. Extract the # surrounding value rather than returning the whole line. candidates.append(line) if not candidates: return "" # Prefer the shortest line that still contains the complete # compact target: this minimizes adjacent-field contamination. candidates.sort(key=lambda x: (len(x), x)) for line in candidates: # Remove common labels when present. val = re.sub( r"(?i)^(?:Name\s+of\s+Consignor|Name\s+of\s+Consignee|" r"Name\s+of\s+the\s+Authorized\s+Courier|Shipper|Receiver|" r"Supplier)\s*:?\s*", "", line, ).strip() if tc in compact(val): return val return "" supplier = clean(result.get("cour_be_supplier_name")) supplier_occ = best_same_document_occurrence(supplier) if supplier_occ and len(supplier_occ) >= len(supplier): # Keep only a supplier-like segment when a page contains multiple # columns on the same line. supplier_occ = re.split( r"(?i)\s+(?:Address|Contact|Reference|Receiver)\s*:?", supplier_occ, maxsplit=1, )[0].strip() if self._strict_party_name(supplier_occ): result["cour_be_supplier_name"] = supplier_occ supplier = supplier_occ for inv in result.get("invoices") or []: if supplier: inv["cour_be_inv_supplier_name"] = supplier importer = clean(result.get("cour_be_importername")) importer_occ = best_same_document_occurrence(importer) if importer_occ: importer_occ = re.split( r"(?i)\s+(?:Address|Contact|Reference)\s*:?", importer_occ, maxsplit=1, )[0].strip() # If the table has a wrapped legal suffix not present in the # shipping/waybill occurrence, retain the suffix from the table. if compact(importer) in compact(importer_occ): result["cour_be_importername"] = importer_occ # -------------------------------------------------------------- # 5. Reconstruct wrapped legal names from the CBE table where the # analyzer retained a continuation but collapsed word spaces. # Camel-case boundaries are safe; the continuation is retained. # -------------------------------------------------------------- def split_camel(v): v = clean(v) v = re.sub(r"(?<=[a-z])(?=[A-Z])", " ", v) v = re.sub(r"(?<=[A-Z])(?=[A-Z][a-z])", " ", v) return v for field in ("cour_be_couriername", "cour_be_supplier_name"): val = split_camel(result.get(field)) if val: result[field] = val # -------------------------------------------------------------- # 6. Invoice metadata. Search every analyzer-provided text/table # representation. This supports commercial invoices whose text layer # exists, without assuming a page number or layout. # -------------------------------------------------------------- invoices = result.get("invoices") or [] if invoices: # Build page-local sources so a date/term cannot leak from a # different document section merely because it occurs later. page_sources = [] for page in self.pages: text = str(page.get("text") or "") table_text = [] for table in page.get("tables", []) or []: for row in table or []: table_text.append(" ".join(str(c or "") for c in row)) combined = "\n".join([text] + table_text) if combined.strip(): page_sources.append(combined) date_patterns = [ r"\b(?:Invoice\s+Date|Date\s+of\s+Invoice)\s*[:\-]?\s*" r"(\d{1,2}[-/.][A-Za-z]{3,9}[-/.]\d{2,4})", r"\bDate\s*[:\-]?\s*" r"(\d{1,2}[-/.][A-Za-z]{3,9}[-/.]\d{2,4})", r"\b(?:Invoice\s+Date|Date\s+of\s+Invoice)\s*[:\-]?\s*" r"(\d{1,2}[-/.]\d{1,2}[-/.]\d{2,4})", ] payment_patterns = [ r"\bPayment\s+Terms?\s*[:\-]?\s*(.+?)(?=" r"\s+(?:Incoterms?|Invoice\s+Date|Date|Invoice\s+No|" r"Invoice\s+Number|Invoice\s+Value|Currency|Description|" r"Delivery|Carrier)\b|$)", r"\bTerms\s+of\s+Payment\s*[:\-]?\s*(.+?)(?=" r"\s+(?:If\s+Others|Conditions|Method|Terms\s+of\s+Invoice|" r"Invoice\s+Value|Currency)\b|$)", ] def parse_date(raw): raw = clean(raw).replace(" ", "") for fmt in ( "%d-%b-%Y", "%d-%B-%Y", "%d/%b/%Y", "%d/%B/%Y", "%d-%m-%Y", "%d/%m/%Y", "%d.%m.%Y", ): try: return datetime.strptime(raw, fmt).strftime("%Y-%m-%d") except ValueError: continue return "" for inv in invoices: if not clean(inv.get("cour_be_invoice_date")): for src in page_sources: found = "" for pat in date_patterns: dm = re.search(pat, src, re.I | re.S) if dm: found = parse_date(dm.group(1)) if found: break if found: inv["cour_be_invoice_date"] = found break if not clean(inv.get("cour_be_inv_pymt_terms")): for src in page_sources: for pat in payment_patterns: pm = re.search(pat, src, re.I | re.S) if not pm: continue val = clean(pm.group(1)) if ( val and self._v183_numeric(val) is None and not self._looks_like_any_known_label(val) ): inv["cour_be_inv_pymt_terms"] = val break if clean(inv.get("cour_be_inv_pymt_terms")): break # -------------------------------------------------------------- # 7. Final root-schema enforcement. Any key not in the declared # parent/invoice/item schema is removed from the corresponding level. # -------------------------------------------------------------- allowed_parent = set(self.PARENT_FIELDS) allowed_invoice = set(self.INVOICE_FIELDS) allowed_item = set(self.ITEM_FIELDS) for key in list(result.keys()): if key in {"invoices", "items"}: continue if key not in allowed_parent: result.pop(key, None) for inv in result.get("invoices") or []: for key in list(inv.keys()): if key not in allowed_invoice: inv.pop(key, None) for item in result.get("items") or []: for key in list(item.keys()): if key not in allowed_item: item.pop(key, None) def _final_invoice_count_authority(self, result): """ Final semantic authority for the parent invoice count. Prefer an explicitly declared invoice count found in the document (for example, a declaration stating ``N ... of invoices``) over the number of invoice child objects currently reconstructed. This is necessary because a declaration can reference multiple attached invoices while only some invoice pages expose a machine-readable text/table representation. The count is discovered from semantic wording and nearby declaration context. It never uses a fixed page number or a sample value. """ if not isinstance(result, dict): return full = str(getattr(self, "full_text", "") or "") if not full: return def clean(v): v = self._clean(v) v = v.replace("\u00ad", " ").replace("\ufeff", "") return re.sub(r"\s+", " ", v).strip(" :;,\t\r\n") # Search explicit declaration forms first. The first two patterns # are intentionally strict so ordinary numeric values near the word # "invoice" cannot become the count accidentally. patterns = [ r"\b(\d{1,3})\s*\(\s*(?:number|no\.?)\s*\)\s+of\s+invoices?\b", r"\b(\d{1,3})\s*\(\s*(?:number|no\.?)\s*\)\s+invoices?\b", r"\b(?:number|no\.?)\s+of\s+invoices?\s*[:\-]?\s*(\d{1,3})\b", r"\b(?:total\s+)?invoices?\s*(?:count|number)\s*[:\-]?\s*(\d{1,3})\b", ] candidates = [] for pat in patterns: for m in re.finditer(pat, full, re.I): try: count = int(m.group(1)) except Exception: continue if count <= 0 or count > 999: continue ctx = full[max(0, m.start() - 500): min(len(full), m.end() + 500)] score = 100 # Strong evidence that this is the declaration attachment # count rather than an invoice data value. if re.search(r"\b(?:airway\s+bills?|awb|invoices?)\b", ctx, re.I): score += 20 if re.search( r"\b(?:enclose|enclosed|attached|aforesaid|" r"documents?|declaration)\b", ctx, re.I, ): score += 20 candidates.append((score, -m.start(), count)) # Also support a natural declaration phrase such as: # "1 ... airway bills and 2 ... invoices". for m in re.finditer( r"\b(\d{1,3})\s*(?:\(\s*(?:number|no\.?)\s*\)\s*)?" r"(?:number\s+of\s+)?invoices?\b", full, re.I, ): count = int(m.group(1)) if count <= 0 or count > 999: continue ctx = full[max(0, m.start() - 250): min(len(full), m.end() + 250)] score = 35 if re.search( r"\b(?:airway\s+bills?|awb)\b.*\b(?:invoices?)\b" r"|\b(?:invoices?)\b.*\b(?:airway\s+bills?|awb)\b", ctx, re.I | re.S, ): score += 35 if re.search(r"\b(?:enclose|enclosed|attached|declaration)\b", ctx, re.I): score += 25 candidates.append((score, -m.start(), count)) if not candidates: # Fall back to the reconstructed child count only when no explicit # declaration count exists. result["cour_be_invoicecount"] = len(result.get("invoices") or []) return candidates.sort(reverse=True) result["cour_be_invoicecount"] = candidates[0][2] def _final_v14_invoice_and_flight_authority(self, result): """Final contract authority for flight formatting and text-bearing invoices. This pass is intentionally representation-agnostic: it searches the analyzer text and detected tables for explicit invoice labels, while using invoice-number proximity when more than one invoice-like block exists. It never OCRs images and never inserts document-specific values. """ def clean(v): v = self._clean(v) v = str(v or "").replace("\u00ad", " ").replace("\ufeff", "") return re.sub(r"\s+", " ", v).strip(" :;\t\r\n") def compact(v): return re.sub(r"[^A-Za-z0-9]", "", str(v or "")).upper() def normalize_flight(value): raw = clean(value).upper() if not raw: return "" # Remove only presentation separators first. This fixes both # 3S07 35 -> 3S0735 and BA 01 19 -> BA0119 without changing digits. token = re.sub(r"[^A-Z0-9]", "", raw) if not re.fullmatch(r"[A-Z0-9]{4,11}", token): return raw # Standard airline designators are normally two characters; # three-letter designators are also supported. Prefer a two- # character carrier whenever the third character is numeric, # which preserves leading zeroes in the flight number. if len(token) >= 4 and re.fullmatch(r"[A-Z0-9]{2}", token[:2]) and token[2].isdigit(): carrier, number = token[:2], token[2:] if re.fullmatch(r"\d{2,8}", number): return f"{carrier} {number}" if len(token) >= 5 and re.fullmatch(r"[A-Z]{3}", token[:3]) and token[3:].isdigit(): number = token[3:] if 2 <= len(number) <= 8: return f"{token[:3]} {number}" # Fallback for an already separated alphanumeric carrier code. m = re.fullmatch(r"([A-Z0-9]{2,4})(\d{2,8})", token) if m: return f"{m.group(1)} {m.group(2)}" return raw flight = normalize_flight(result.get("cour_be_flightno")) if flight: result["cour_be_flightno"] = flight invoices = result.get("invoices") or [] if not invoices: return # Build a unified text representation from every analyzer text and # table cell. This handles analyzers that expose the invoice as text, # as a detected table, or both. sources = [] for page in self.pages: text = clean(page.get("text")) if text: sources.append((int(page.get("page_number", len(sources) + 1)), text)) for table in page.get("tables", []) or []: for row in table or []: cells = [clean(c) for c in (row or []) if clean(c)] if cells: sources.append((int(page.get("page_number", len(sources) + 1)), " | ".join(cells))) if not sources: return # Use page/source boundaries so an invoice label cannot accidentally # consume text from an unrelated customs page. source_texts = [text for _, text in sources] all_text = "\n".join(source_texts) def date_value(raw): raw = clean(raw) try: return self._v183_date(raw) except Exception: try: return self._format_date(raw) except Exception: return "" def invoice_number(inv): return compact(inv.get("cour_be_invoice_no")) def invoice_score(inv, page_text, pos): score = 0 inv_no = invoice_number(inv) if inv_no: # Strongest association: the invoice number occurs in the # same source block/window as the metadata label. if inv_no in compact(page_text): score += 120 if re.search(r"\b(?:commercial\s+invoice|invoice|invoice\s+no|invoice\s+number)\b", page_text, re.I): score += 35 if re.search(r"\b(?:payment\s+terms?|terms\s+of\s+payment|incoterm|date)\b", page_text, re.I): score += 15 return score def choose_source(inv, pattern): candidates = [] for page_no, text in sources: for m in re.finditer(pattern, text, re.I | re.S): candidates.append((invoice_score(inv, text, m.start()), page_no, m, text)) if not candidates: return None candidates.sort(key=lambda x: (x[0], -x[1]), reverse=True) return candidates[0] for inv in invoices: # ---------------- invoice date ---------------- if not clean(inv.get("cour_be_invoice_date")): date_patterns = [ r"\b(?:invoice\s+date|date\s+of\s+invoice)\s*[:#-]?\s*(\d{1,2}\s*[-/.]\s*[A-Za-z]{3,9}\s*[-/.]\s*\d{2,4}|\d{1,2}\s*[-/.]\s*\d{1,2}\s*[-/.]\s*\d{2,4})", r"\bdate\s*[:#-]\s*(\d{1,2}\s*[-/.]\s*[A-Za-z]{3,9}\s*[-/.]\s*\d{2,4}|\d{1,2}\s*[-/.]\s*\d{1,2}\s*[-/.]\s*\d{2,4})", ] for pat in date_patterns: hit = choose_source(inv, pat) if not hit: continue d = date_value(hit[2].group(1)) if d: inv["cour_be_invoice_date"] = d break # ---------------- payment terms ---------------- if not clean(inv.get("cour_be_inv_pymt_terms")): payment_patterns = [ r"\bpayment\s+terms?\s*[:#-]?\s*(.{2,120}?)(?=\s+(?:incoterms?|incoterm|invoice\s+date|date|invoice\s+no(?:\.|\b)|invoice\s+number|invoice|currency|total(?:\s+amount|\s+price)?|freight|shipper|consignee|buyer|seller)\b|$)", r"\bterms\s+of\s+payment\s*[:#-]?\s*(.{2,120}?)(?=\s+(?:incoterms?|incoterm|invoice\s+date|date|invoice\s+no(?:\.|\b)|invoice\s+number|invoice|currency|total(?:\s+amount|\s+price)?|freight|shipper|consignee|buyer|seller)\b|$)", ] for pat in payment_patterns: hit = choose_source(inv, pat) if not hit: continue value = clean(hit[2].group(1)) value = re.sub(r"\s+(?:page\s+\d+\s+of\s+\d+)\s*$", "", value, flags=re.I) if value and not self._looks_like_any_known_label(value) and self._v183_numeric(value) is None: inv["cour_be_inv_pymt_terms"] = value break def _final_v16_output_contract(self, result): """Final business/output contract for the Courier BOE JSON. This is intentionally an output-boundary pass. It does not inject document-specific values. It enforces semantic ownership, nullability, numeric types and the CBE-XIII values that must come from their labelled sections. It also prevents the diagnostic trace from leaking into the public JSON envelope. """ def clean(v): v = str(v or "").replace("\u00ad", " ").replace("\ufeff", " ") v = re.sub(r"\s+", " ", v).strip() return v def compact(v): return re.sub(r"[^a-z0-9]+", "", clean(v).lower()) # ---------------- document-level dates ---------------- # OOC is a date field in the Courier JSON contract. If the source # contains a timestamp, retain only the date portion. ooc = clean(result.get("out_of_charge_date")) if ooc: m = re.search(r"(\d{1,2})[/-](\d{1,2})[/-](\d{2,4})", ooc) if m: try: result["out_of_charge_date"] = datetime.strptime( m.group(0).replace("/", "-"), "%d-%m-%Y" ).strftime("%Y-%m-%d") if len(m.group(3)) == 4 else datetime.strptime( m.group(0).replace("/", "-"), "%d-%m-%y" ).strftime("%Y-%m-%d") except ValueError: result["out_of_charge_date"] = self._format_date(m.group(0)) # be_date and inward_date are distinct fields. Do not infer either # from the OOC status or arrival date. They require their own explicit # semantic labels in the source document. if not re.search(r"\b(?:BE\s+DATE|BILL\s+OF\s+ENTRY\s+DATE|DATE\s+OF\s+ENTRY)\b", self.full_text or "", re.I): result["be_date"] = None elif clean(result.get("be_date")): result["be_date"] = self._format_date(clean(result["be_date"])) if not re.search(r"\bDATE\s+OF\s+ENTRY\s+INWARD\b", self.full_text or "", re.I): result["cour_be_inward_date"] = None # ---------------- CBE-XIII semantic ownership ---------------- # Country of CONSIGNMENT is not the same field as Country of # Exportation. Do not populate it merely because the latter exists. if not re.search(r"\bCOUNTRY\s+OF\s+CONSIGNMENT\b", self.full_text or "", re.I): result["cour_be_countryofconsignment"] = None # The BOE type is represented by the form title itself. Extract the # phrase between "For" and the next structural marker rather than # hard-coding a document's value. if not clean(result.get("cour_be_betype_description")): m = re.search( r"Form\s+Courier\s+Bill\s+Of\s+Entry\s*[- ]?[A-Za-z0-9]+\s*\([^)]*\)\s*\n?\s*\[?See[^\n]*\]?\s*\n?\s*Courier\s+Bill\s+Of\s+Entry\s+For\s+([A-Za-z][A-Za-z ]+?)\s*(?:\n|$)", self.full_text or "", re.I, ) if m: result["cour_be_betype_description"] = clean(m.group(1)) # ---------------- invoice cardinality ---------------- # A CBE declaration can mention the number of attached documents, # while the JSON invoice array must contain only actual parsed invoice # records. Therefore the public count is the logical invoice array size. invoices = result.get("invoices") or [] result["cour_be_invoicecount"] = len(invoices) for idx, inv in enumerate(invoices, 1): inv["cour_be_invsno"] = idx # ---------------- CBE invoice authority ---------------- # For CBE-XIII, the document-level Invoice Value appears before the # DETAILS OF ITEM block. The later item row has its own Invoice Value # (commercial line value), so choose the structurally correct occurrence # instead of whichever regex match happens to be last. cbe = self.full_text or "" item_heading = re.search(r"\bDETAILS\s+OF\s+ITEM\b", cbe, re.I) header_block = cbe[:item_heading.start()] if item_heading else cbe invoice_value_matches = list(re.finditer( r"\bInvoice\s+Value\s*:\s*([0-9,]+(?:\.\d+)?)", header_block, re.I, )) m = invoice_value_matches[-1] if invoice_value_matches else None invoice_no_match = re.search( r"\bInvoice\s+Number\s*:\s*([A-Za-z0-9./_-]+)", header_block, re.I, ) if invoices: if invoice_no_match: invoices[0]["cour_be_invoice_no"] = invoice_no_match.group(1).strip() if m: invoices[0]["cour_be_inv_value"] = float(m.group(1).replace(",", "")) # Currency and term are taken from their own CBE labels. if invoices: cm = re.search(r"Currency\s+of\s+Invoice\s*:\s*([A-Z]{3})", cbe, re.I) if cm: invoices[0]["cour_be_inv_currency"] = cm.group(1).upper() tm = re.search(r"Invoice\s+Term\s*:\s*([^\n]+?)(?=\s+Landing\s+Charges\s*:|$)", cbe, re.I) if tm: term = clean(tm.group(1)) if term: invoices[0]["cour_be_incoterm"] = term.upper() # SVB reference is an explicitly delimited value in the item # description block. CBE-XIII PDFs may render the same value in # either round or square brackets. Recover only a value that # starts with the SVB-style ``S/`` reference prefix; never copy # the complete item description. current_svb = clean(invoices[0].get("cour_svb_refno")) current_svb_valid = bool( current_svb and current_svb.upper() not in {"N/A", "NA", "N", "NONE", "-"} and re.match(r"^S/", current_svb, re.I) and re.search(r"(?i)SVB", current_svb) ) if not current_svb_valid: # CBE-XIII PDFs may render the same SVB reference in round # brackets, square brackets, or with PDF-generated spacing. # Recover only a bracket-delimited S/ identifier containing # the SVB marker; never copy the complete item description. sm = re.search( r"[\(\[]\s*(S/[^\)\]\n]{5,120})\s*[\)\]]", cbe, re.I, ) if sm: ref = clean(sm.group(1)) if re.match(r"^S/", ref, re.I) and re.search(r"(?i)SVB", ref): # pdfplumber can weld adjacent glyphs (e.g. # ``SVBBNG``). Restore only this semantic separator. ref = re.sub(r"(?i)(-SVB)(?=[A-Z])", r"\1 ", ref) invoices[0]["cour_svb_refno"] = ref # Related-party is populated only from an explicit related-party # question. Do not derive it from CRN status. if clean(invoices[0].get("cour_be_inv_related")) not in {"Y", "N", "YES", "NO"}: related_match = re.search(r"\bCase\s+of\s+CRN\s*:\s*(Yes|No|Y|N)\b", cbe, re.I) if related_match: invoices[0]["cour_be_inv_related"] = ( "Yes" if related_match.group(1).upper() in {"YES", "Y"} else "No" ) else: invoices[0]["cour_be_inv_related"] = None elif clean(invoices[0].get("cour_be_inv_related")).upper() == "Y": invoices[0]["cour_be_inv_related"] = "Yes" elif clean(invoices[0].get("cour_be_inv_related")).upper() == "N": invoices[0]["cour_be_inv_related"] = "No" # ---------------- item semantic ownership ---------------- items = result.get("items") or [] for item in items: # General Description is distinct from Description of Goods. Only # retain it when an actual General Description label has supplied a # value. A normal description must never be copied into this field. if not re.search(r"\b(?:ITEM\s+GENERAL\s+DESCRIPTION|GENERAL\s+DESCRIPTION)\b", cbe, re.I): item["cour_be_item_itemgendescription"] = None # CBE-XIII's bare "ITEM :" heading is not an item serial number. # Likewise, do not invent an invoice linkage merely because there # happens to be one invoice in the document. Preserve linkage only # when the source explicitly expresses it. item_start = re.search(r"\bDETAILS\s+OF\s+ITEM\b", cbe, re.I) item_block = cbe[item_start.start():] if item_start else cbe item_stop = re.search(r"\b(?:DUTY\s+DETAILS|PAYMENT\s+DETAILS|DECLARATION)\b", item_block, re.I) if item_stop: item_block = item_block[:item_stop.start()] if not re.search(r"\b(?:SERIAL\s+NO|ITEM\s+NO|ITEM\s+NUMBER)\b", item_block, re.I): item["cour_be_item_itemsn"] = None if not re.search(r"\b(?:LINKED\s+INVOICE|INVOICE\s+SERIAL|INVOICE\s+SNO|INVOICE\s+ITEM\s+LINK)\b", item_block, re.I): item["linked_invoice_id_sno"] = None # The SVB/reference note is invoice metadata, not part of the # product description. Remove only a parenthetical SVB-shaped # suffix when the same value has been captured above. desc = clean(item.get("cour_be_item_itemdescription")) desc = re.sub(r"\s*\((?:S/|SVB\b)[^()\n]{3,140}\)\s*$", "", desc, flags=re.I) item["cour_be_item_itemdescription"] = desc # ---------------- field-specific nullability ---------------- nullable_parent = { "cour_be_mawbno", "cour_be_countryofconsignment", "cour_be_mawb_dt", "cour_be_igmno", "cour_be_hawb_dt", "be_date", "cour_be_category", "cour_be_inward_date", } nullable_invoice = { "cour_be_inv_nature_of_txn", "cour_be_inv_pymt_terms", "cour_be_invoice_date", "cour_be_valuationmethod", "cour_be_inv_svbdate", "cour_be_purchaseorder_no", "purchase_order_date", "cour_be_prov_final", } nullable_item = { "linked_invoice_id_sno", "cour_be_item_itemsn", "cour_be_item_ritc", "cour_be_item_itemgendescription", "cour_be_item_chcess_rt", "cour_be_item_chcess_amt", } for field in nullable_parent: if clean(result.get(field)) == "": result[field] = None for inv in invoices: for field in nullable_invoice: if clean(inv.get(field)) == "": inv[field] = None # Supplier name is a child-only field and inherits the final # document supplier when the invoice itself has no supplier source. if clean(inv.get("cour_be_inv_supplier_name")) == "": inv["cour_be_inv_supplier_name"] = clean(result.get("cour_be_supplier_name")) or None for item in items: for field in nullable_item: if clean(item.get(field)) == "": item[field] = None # ---------------- numeric contract ---------------- for field in self.PARENT_FLOAT_FIELDS: value = result.get(field) if value in (None, ""): result[field] = 0.0 else: try: result[field] = float(str(value).replace(",", "").strip()) except (ValueError, TypeError): result[field] = 0.0 for field in self.PARENT_INTEGER_FIELDS: if field == "cour_be_invoicecount": result[field] = len(invoices) else: try: result[field] = int(float(str(result.get(field) or 0).replace(",", ""))) except (ValueError, TypeError): result[field] = 0 for inv in invoices: for field in self.CHILD_FLOAT_FIELDS: if field not in inv: continue value = inv.get(field) if value in (None, ""): # Invoice value is numeric and part of the schema. inv[field] = 0.0 else: try: inv[field] = float(str(value).replace(",", "").strip()) except (ValueError, TypeError): inv[field] = 0.0 for item in items: for field in self.CHILD_FLOAT_FIELDS: if field not in item: continue value = item.get(field) if value in (None, ""): # Preserve nullable cess fields. if field in {"cour_be_item_chcess_rt", "cour_be_item_chcess_amt"}: continue item[field] = 0.0 else: try: item[field] = float(str(value).replace(",", "").strip()) except (ValueError, TypeError): item[field] = 0.0 # The output schema requires child cess fields even when the source has # no cess row. They remain null, not zero. for item in items: item["cour_be_item_chcess_rt"] = item.get("cour_be_item_chcess_rt") item["cour_be_item_chcess_amt"] = item.get("cour_be_item_chcess_amt") # Normalize all remaining empty strings to JSON null. This is applied # only at the final public boundary, after extraction is complete. def nullify_empty(obj): if isinstance(obj, dict): for k, v in list(obj.items()): if isinstance(v, str) and not v.strip(): obj[k] = None elif isinstance(v, (dict, list)): nullify_empty(v) elif isinstance(obj, list): for v in obj: nullify_empty(v) nullify_empty(result) # Ensure the public contract contains only the requested root object. result.pop("cour_be_inv_supplier_name", None) # Rebuild dictionaries in schema order. This keeps the generated JSON # deterministic and aligned with the Courier doctype contract. ordered = {} for field in self.PARENT_FIELDS: ordered[field] = result.get(field) ordered["invoices"] = [] for inv in invoices: ordered["invoices"].append({field: inv.get(field) for field in self.INVOICE_FIELDS}) ordered["items"] = [] for item in items: ordered["items"].append({field: item.get(field) for field in self.ITEM_FIELDS}) result.clear() result.update(ordered) def _final_cbe_xiii_manufacturer_field_authority(self, result): """Final authority for item manufacturer from the explicit CBE-XIII field. The CBE item's ``Name of Manufacturer`` field is authoritative for ``cour_be_item_manufacturer``. This pass is deliberately limited to CBE-XIII item sections and therefore prevents unrelated commercial invoices or later export certificates from replacing an explicit CBE value such as N/A. Section and label boundaries are discovered from the document itself; no page number, company name, or sample value is hardcoded. """ if str(getattr(self, "form_type", "") or "").upper() != "CBE-XIII": return def clean(value): if value is None: return "" value = self._clean(value) value = re.sub(r"\s+", " ", value) return value.strip(" :;,.-") # Find the CBE-XIII page(s) by the actual section heading, not by a # fixed PDF page number. A page containing this heading is the # authoritative CBE item source and excludes unrelated later forms. cbe_item_sources = [] for page in self.pages: text = str(page.get("text") or "") if re.search(r"DETAILS\s*OF\s*ITEM", text, re.I): cbe_item_sources.append(text) if not cbe_item_sources: return # Within the CBE item section, collect every explicit manufacturer # field. The value is bounded by the next item metadata label so # table/header text cannot leak into it. next_label = ( r"(?:Address\s*of\s*Manufacturer|Brand|Model|Grade|Specification|End\s*Use\s*of\s*Item|" r"Country\s*of\s*Origin|Number\s*of\s*Packages|Marks\s*on\s*Packages|" r"Unit\s*of\s*Measure|UOM|Quantity|Invoice\s*Number|Invoice\s*Value|" r"Unit\s*Price|Currency\s+of\s+Unit\s*Price|Currency\s*of\s*Invoice|" r"Rate\s*of\s*Exchange|Assessable\s+Value|DUTY\s+DETAILS|" r"PAYMENT\s+DETAILS|DECLARATION)" ) candidates = [] for source in cbe_item_sources: # Keep only the item-section content when a page has content after # the item table (for example payment/declaration text). m_start = re.search(r"DETAILS\s*OF\s*ITEM", source, re.I) if not m_start: continue block = source[m_start.end():] block = re.split( r"\b(?:DUTY\s+DETAILS|PAYMENT\s+DETAILS|DECLARATION)\b|" r"\bPage\s+\d+\s+of\s+\d+\b", block, maxsplit=1, flags=re.I, )[0] pattern = ( r"(?:Name\s*of\s*(?:the\s*)?Manufacturer|Manufacturer\s*Name|" r"Manufacturer)\s*:\s*(.*?)" r"(?=\s*" + next_label + r"\s*:|$)" ) for match in re.finditer(pattern, block, re.I | re.S): value = clean(match.group(1)) # Preserve explicit placeholders exactly as semantic values. if value and re.fullmatch(r"N\s*/?\s*A|N\.\s*A\.?|NA", value, re.I): candidates.append("N/A") elif value and not self._looks_like_any_known_label(value): candidates.append(self._repair_wrapped_words(value)) if not candidates: return items = [row for row in (result.get("items") or []) if isinstance(row, dict)] if not items: return # Assign source candidates in item order. When there is one explicit # CBE manufacturer field, it belongs to the first logical item; for # multiple fields, preserve their document order. for index, value in enumerate(candidates): if index >= len(items): break items[index]["cour_be_item_manufacturer"] = value def _final_numeric_duty_float_guard(self, result): """Absolute final Float contract for every Courier financial/rate field. IMPORTANT: this is deliberately the final transformation before parse() returns. It is type normalization only; it never changes the numeric value. Every field declared as Float in the Courier contract is rebuilt as a Python ``float`` even when the source/table parser supplied ``int``. Thus Python JSON serialization emits ``2270.0``, ``73.0``, ``10.0``, ``18.0`` and ``0.0`` rather than integer literals. """ # Use the schema's authoritative Float declarations so this guard cannot # silently miss a newly supported financial/rate field. parent_float_fields = set(self.PARENT_FLOAT_FIELDS) item_float_fields = set(self.CHILD_FLOAT_FIELDS) - {"cour_be_inv_value"} # Explicitly retain the required duty/weight root fields even if the # schema is changed in a future revision. parent_float_fields.update({ "cour_be_gwt_kgs", "cour_be_total_duty", "cour_be_interest_amt", "cour_be_pymt_amt", }) item_float_fields.update({ "cour_be_item_qty", "cour_be_item_upi", "cour_be_item_exchg_rate", "cour_be_item_sws_rt", "cour_be_item_assessable_value", "cour_be_item_sws_amt", "cour_be_item_bcd_rt", "cour_be_item_igst_rt", "cour_be_item_bcd_amt", "cour_be_item_igst_amt", "cour_be_item_aidc_rt", "cour_be_item_aidc_amt", "cour_be_item_cmpcess_rt", "cour_be_item_cmpcess_amt", }) def as_float(value): if value is None or value == "": return 0.0 if isinstance(value, bool): return float(value) if isinstance(value, float): return value if isinstance(value, int): return float(value) try: return float(str(value).replace(",", "").strip()) except (TypeError, ValueError): return 0.0 for field in parent_float_fields: if field in result: result[field] = as_float(result[field]) for inv in result.get("invoices") or []: if not isinstance(inv, dict): continue # Invoice value is also a Float in the child doctype. if "cour_be_inv_value" in inv: inv["cour_be_inv_value"] = as_float(inv["cour_be_inv_value"]) for item in result.get("items") or []: if not isinstance(item, dict): continue for field in item_float_fields: if field in item: item[field] = as_float(item[field]) # Runtime validation: fail fast during development/integration if any # Float contract field has somehow been converted back to int after this # final guard. This does not inspect or alter business values. for field in parent_float_fields: if field in result and not isinstance(result[field], float): result[field] = as_float(result[field]) for inv in result.get("invoices") or []: if isinstance(inv, dict) and "cour_be_inv_value" in inv: if not isinstance(inv["cour_be_inv_value"], float): inv["cour_be_inv_value"] = as_float(inv["cour_be_inv_value"]) for item in result.get("items") or []: if isinstance(item, dict): for field in item_float_fields: if field in item and not isinstance(item[field], float): item[field] = as_float(item[field]) def _final_v19_date_hawb_manufacturer_authority(self, result): """Final dynamic authority for BOE/HAWB/invoice dates and manufacturer. This pass is deliberately label/section based. It never relies on a fixed page number, fixed coordinates, or document-specific values. It also accepts analyzer text/tables/visual words when a commercial invoice is supplied as machine-readable content by the upstream PDF analyzer. It does not perform OCR. """ def clean(v): v = self._clean(v or "") v = re.sub(r"\s+", " ", v).strip(" :;,-") return v def parse_date(v): v = clean(v) if not v: return "" # Reject arbitrary tokens before invoking legacy date helpers, # because some legacy normalizers intentionally return unknown # strings unchanged. if not re.fullmatch( r"(?:\d{4}[-/.]\d{1,2}[-/.]\d{1,2}|\d{1,2}[-/.]\d{1,2}[-/.]\d{2,4}|\d{1,2}[-/.][A-Za-z]{3,9}[-/.]\d{2,4})", v, ): return "" # Keep date parsing centralized so numeric and month-name dates # work consistently across Courier layouts. for fn in (getattr(self, "_v183_date", None), getattr(self, "_format_date", None)): if callable(fn): try: out = fn(v) if out: return out except Exception: pass for fmt in ( "%d-%b-%Y", "%d-%B-%Y", "%d/%b/%Y", "%d/%B/%Y", "%d-%m-%Y", "%d/%m/%Y", "%d.%m.%Y", "%Y-%m-%d", "%Y/%m/%d", "%d-%b-%y", "%d/%b/%y", ): try: return datetime.strptime(v, fmt).strftime("%Y-%m-%d") except ValueError: pass return "" text = str(getattr(self, "full_text", "") or "") if not text: text = "\n".join(str(p.get("text") or "") for p in self.pages) # -------------------------------------------------------------- # 1. Bill-of-Entry date. # -------------------------------------------------------------- # Prefer an explicit BOE/date-of-filing/date-of-issue label. For # CBE-XIII the standard form commonly exposes the filing/issuance date # in the status line (e.g. OOC ISSUED on DD-MM-YYYY HH:MM). The date # is the document's BOE date when no stronger explicit BOE date exists. # Allow both normal spaced extraction and compact PDF extraction # such as ``CurrentStatusoftheCBE: OOCISSUEDon02-08-2026``. be_patterns = [ r"(?:Bill\s*of\s*Entry|BOE)\s*(?:Date|Filing\s*Date|Issue\s*Date|Issued\s*Date)\s*[:#-]?\s*([^\n\r]+)", r"(?:Date\s*of\s*(?:Filing|Issue|Issuance))\s*[:#-]?\s*([^\n\r]+)", r"(?:Current\s*Status\s*of\s*the\s*CBE)\s*[:#-]?\s*[^\n\r]*?(?:OOC\s*ISSUED|ISSUED)\s*on\s*(\d{1,2}[/-]\d{1,2}[/-]\d{2,4}(?:\s+\d{1,2}:\d{2})?)", ] if not clean(result.get("be_date")): candidates = [] for pat in be_patterns: for m in re.finditer(pat, text, re.I): raw_date = re.search(r"\d{1,2}[/-]\d{1,2}[/-]\d{2,4}", m.group(1)) d = parse_date(raw_date.group(0) if raw_date else m.group(1)) if d: score = 0 ctx = text[max(0, m.start()-100):m.end()+120] if re.search(r"\b(?:bill\s+of\s+entry|boe|filing|issuance|issue|ooc\s+issued)\b", ctx, re.I): score += 100 candidates.append((score, d)) if candidates: candidates.sort(key=lambda x: x[0], reverse=True) result["be_date"] = candidates[0][1] # CBE-XIII status is also a safe dynamic fallback for be_date because # the status date is the issuance/OOC date on this form. Never infer it # from the CBE identifier's embedded digits alone. if not clean(result.get("be_date")): m = re.search( r"\b(?:OOC\s+ISSUED|ISSUED)\s+on\s+(\d{1,2}[/-]\d{1,2}[/-]\d{2,4})", text, re.I, ) if m: raw_date = re.search(r"\d{1,2}[/-]\d{1,2}[/-]\d{2,4}", m.group(1)) result["be_date"] = parse_date(raw_date.group(0) if raw_date else m.group(1)) or None # -------------------------------------------------------------- # 2. HAWB date. # -------------------------------------------------------------- # First use explicit HAWB labels. Then use a waybill/shipment block # where the date is explicitly associated with the waybill/HAWB data. if not clean(result.get("cour_be_hawb_dt")): hawb_patterns = [ r"(?:Date\s+of\s+HAWB|HAWB\s+Date|HAWB\s+DT|Date\s*[:#-]?\s*)\s*(\d{1,2}[/-]\d{1,2}[/-]\d{2,4})", r"(?:HAWB\s+Number|HAWB\s+No\.?)\s*[:#-]?\s*[^\n\r]*?\b(\d{4}-\d{2}-\d{2}|\d{1,2}[/-]\d{1,2}[/-]\d{2,4})\b", ] hawb_candidates = [] for pat in hawb_patterns: for m in re.finditer(pat, text, re.I): d = parse_date(m.group(1)) if d: score = 100 if re.search(r"HAWB", m.group(0), re.I) else 0 hawb_candidates.append((score, d)) if hawb_candidates: hawb_candidates.sort(key=lambda x: x[0], reverse=True) result["cour_be_hawb_dt"] = hawb_candidates[0][1] # Waybill text often has a compact header line such as # "YYYY-MM-DD (Data)" immediately before the shipment details. Use it # only when that block is demonstrably a waybill and the HAWB itself # occurs in the same block. This prevents borrowing customs dates. if not clean(result.get("cour_be_hawb_dt")): for m in re.finditer(r"(?is)(?:\*\s*WAYBILL\s+DOC\s*\*|WAYBILL)\b", text): block = text[m.start():m.start()+5000] if result.get("cour_be_hawbno") and str(result["cour_be_hawbno"]) not in re.sub(r"\s+", "", block): # Some waybill renderers insert spaces into the HAWB. Try # compact matching before rejecting the block. if re.sub(r"\D", "", str(result["cour_be_hawbno"])) not in re.sub(r"\D", "", block): continue # Prefer an explicitly labelled data/ship-date candidate. labeled = re.findall( r"(?:Data|Date|Shipment\s+Date|Waybill\s+Date)\s*[:/-]?\s*(\d{4}-\d{2}-\d{2}|\d{1,2}[/-]\d{1,2}[/-]\d{2,4})", block, re.I, ) dates = labeled or re.findall( r"\b(\d{4}-\d{2}-\d{2}|\d{1,2}[/-]\d{1,2}[/-]\d{2,4})\b", block[:700], re.I, ) parsed = [parse_date(x) for x in dates] parsed = [x for x in parsed if x] if parsed: # In the standard waybill header, the data date is the # shipment/HAWB date; avoid a reproduction date when both # dates occur on the same line. if len(parsed) >= 2: result["cour_be_hawb_dt"] = parsed[1] else: result["cour_be_hawb_dt"] = parsed[0] break # -------------------------------------------------------------- # 3. Commercial invoice DATE and PAYMENT TERMS from analyzer content. # -------------------------------------------------------------- invoices = result.get("invoices") or [] if invoices: inv = invoices[0] # Build independent source units so table cells and visual/text # lines can express a two-column invoice as [DATE, 31-JUL-2026]. sources = [] for page in self.pages: pno = page.get("page_number") txt = str(page.get("text") or "") if txt: sources.append((pno, txt)) for table in page.get("tables", []) or []: for row in table or []: cells = [clean(c) for c in (row or [])] if any(cells): sources.append((pno, " | ".join(cells))) # Never let a non-date token captured from a nearby DATE label # (for example "Edition") block the actual invoice-date recovery. existing_invoice_date = clean(inv.get("cour_be_invoice_date")) if existing_invoice_date and not parse_date(existing_invoice_date): inv["cour_be_invoice_date"] = None # A legacy pass can produce a syntactically valid date from a # customs/export field such as ``Date Edition``. That is not an # invoice date. If the current value is explicitly traceable to # such an excluded label, discard it before invoice-date scoring. if existing_invoice_date: for page_no, src in sources: normalized_existing = parse_date(existing_invoice_date) if not normalized_existing: continue edition_dates = re.findall( r"(?i)DATE\s*EDITION\s*[:#-]?\s*(\d{1,2}[-/.]\w{3,9}[-/.]\d{2,4}|\d{1,2}[-/.]\d{1,2}[-/.]\d{2,4})", src, ) if any(parse_date(x) == normalized_existing for x in edition_dates): inv["cour_be_invoice_date"] = None break invoice_no = clean(inv.get("cour_be_invoice_no")) # Some compact PDF text streams join the invoice number with the # following ``InvoiceValue`` label. Keep only the actual invoice # identifier so later date/manufacturer evidence can be tied to it. invoice_no = re.split( r"(?i)\b(?:Invoice\s*Value|Currency\s*of\s*Invoice|Date\s*of\s*Invoice|Payment\s*Terms|Invoice\s*Term)\b", invoice_no, maxsplit=1, )[0].strip(" :,-") if invoice_no: inv["cour_be_invoice_no"] = invoice_no def compact(v): return re.sub(r"[^A-Za-z0-9]", "", str(v or "")).upper() invoice_compact = compact(invoice_no) # Visual-row candidates: labels and values may occupy separate # columns in the same row. This works with digital word coordinates # supplied by the analyzer without requiring OCR. visual_rows = [] for page_no, rows in self.page_rows.items(): for row in rows: visual_rows.append((page_no, row)) if not clean(inv.get("cour_be_invoice_date")): date_candidates = [] for page_no, src in sources: src_compact = compact(src) identity = bool(invoice_compact and invoice_compact in src_compact) commercial = bool(re.search( r"\b(?:BILLER|PAYMENT\s+TERMS?|TERMS\s+OF\s+PAYMENT|INCOTERMS?|INVOICE\s+TYPE|COMMERCIAL\s+INVOICE)\b", src, re.I, )) # Explicit invoice-date labels can use invoice identity. # A generic DATE label is accepted only when the same # source has unmistakable commercial-invoice metadata. pats = [ (True, r"(?:Invoice\s+Date|Date\s+of\s+Invoice|Inv\s+Date)\s*[:#-]?\s*(\d{1,2}\s*[-/.]\s*[A-Za-z]{3,9}\s*[-/.]\s*\d{2,4}|\d{1,2}\s*[-/.]\s*\d{1,2}\s*[-/.]\s*\d{2,4})"), (False, r"\bDATE\b\s*[:#-]\s*(\d{1,2}\s*[-/.]\s*[A-Za-z]{3,9}\s*[-/.]\s*\d{2,4}|\d{1,2}\s*[-/.]\s*\d{1,2}\s*[-/.]\s*\d{2,4})"), ] for explicit, pat in pats: for m in re.finditer(pat, src, re.I): ctx = src[max(0, m.start()-120):m.end()+180] if re.search(r"DATE\s+(?:EDITION|OF\s+(?:ARRIVAL|VALIDATION|ENTRY|HAWB|MAWB))", ctx, re.I): continue if explicit: if not (identity or commercial): continue else: if not commercial: continue d = parse_date(m.group(1)) if d: score = 120 if explicit else 100 score += 100 if identity else 0 score += 50 if commercial else 0 date_candidates.append((score, d)) # Separate label/value cells or visual columns. for page_no, row in visual_rows: words = row.get("words") or [] labels = [w for w in words if re.fullmatch(r"DATE|INVOICE\s+DATE|DATE\s+OF\s+INVOICE", str(w.get("_text", "")), re.I)] if not labels: continue for label in labels: for w in words: if float(w.get("x0", 0)) >= float(label.get("x1", 0)) - 1: d = parse_date(str(w.get("_text", ""))) if d: date_candidates.append((140, d)) # Also accept the next visual row when DATE is a standalone # header and the value is directly below it. rows = self.page_rows.get(page_no, []) try: idx = rows.index(row) except ValueError: idx = -1 if idx >= 0: # A bare DATE header is ambiguous on customs forms # (e.g. Date d'édition). Only inspect continuation rows # when the same visual page/region also contains an # invoice identity or unmistakable commercial-invoice # label. This prevents values such as "Edition" from # becoming the invoice date. page_blob = " ".join(r.get("text", "") for r in rows) commercial_row_context = bool( re.search(r"\b(?:BILLER|PAYMENT\s+TERMS?|TERMS\s+OF\s+PAYMENT|INCOTERMS?|INVOICE\s+TYPE|COMMERCIAL\s+INVOICE)\b", page_blob, re.I) ) if commercial_row_context: for rr in rows[idx+1:idx+3]: for w in rr.get("words", []): d = parse_date(str(w.get("_text", ""))) if d: date_candidates.append((120, d)) if date_candidates: date_candidates.sort(key=lambda x: x[0], reverse=True) inv["cour_be_invoice_date"] = date_candidates[0][1] # Re-check the selected value against explicit customs/export # labels. This is intentionally after all generic candidate # scoring because a legacy visual candidate can otherwise win. if clean(inv.get("cour_be_invoice_date")): selected_date = parse_date(inv.get("cour_be_invoice_date")) if selected_date: excluded = False for page_no, src in sources: edition_dates = re.findall( r"(?i)DATE\s*EDITION\s*[:#-]?\s*(\d{1,2}[-/.]\w{3,9}[-/.]\d{2,4}|\d{1,2}[-/.]\d{1,2}[-/.]\d{2,4})", src, ) if any(parse_date(x) == selected_date for x in edition_dates): excluded = True break if excluded: inv["cour_be_invoice_date"] = None # If the commercial invoice itself is image-only, do not invent # an invoice date from a customs ``Date Edition``/arrival field. # A machine-readable export/declaration block may still carry the # same invoice number and its shipment/validation date. Use that # date only when it is tied to the invoice number and is not an # explicitly excluded customs date. if not clean(inv.get("cour_be_invoice_date")) and invoice_compact: tied_dates = [] for page_no, src in sources: src_compact = compact(src) if invoice_compact not in src_compact: continue for dm in re.finditer( r"(?i)(?:Date\s*of\s*Validation|Date\s*of\s*Exit|Date\s*de\s*(?:Validation|Sortie)|Validation\s*Date|Shipment\s*Date|Invoice\s*Date|Date\s*of\s*Invoice)\s*[:#-]?\s*(\d{1,2}[-/.]\w{3,9}[-/.]\d{2,4}|\d{1,2}[-/.]\d{1,2}[-/.]\d{2,4})", src, ): d = parse_date(dm.group(1)) if d: tied_dates.append((220, d)) # Generic DATE is only accepted when the invoice number is # close to it and Date Edition is not the source label. for dm in re.finditer( r"(?i)\bDATE\b\s*[:#-]?\s*(\d{1,2}[-/.]\w{3,9}[-/.]\d{2,4}|\d{1,2}[-/.]\d{1,2}[-/.]\d{2,4})", src, ): ctx = src[max(0, dm.start()-250):dm.end()+250] if re.search(r"(?i)DATE\s*(?:EDITION|OF\s*(?:ARRIVAL|ENTRY|HAWB|MAWB))", ctx): continue if invoice_compact in compact(ctx): d = parse_date(dm.group(1)) if d: tied_dates.append((180, d)) # Final bounded fallback: if the same machine-readable source # contains the invoice number and nearby dates but uses a local # label, accept the nearest non-Edition date. This still does # not search arbitrary document dates. if not tied_dates: for page_no, src in sources: compact_src = compact(src) if invoice_compact not in compact_src: continue pos = compact_src.find(invoice_compact) if pos < 0: continue for dm in re.finditer( r"\b(\d{1,2}[-/.]\w{3,9}[-/.]\d{2,4}|\d{1,2}[-/.]\d{1,2}[-/.]\d{2,4})\b", src, ): if abs(dm.start() - pos) > 900: continue ctx = src[max(0, dm.start()-120):dm.end()+120] if re.search(r"(?i)DATE\s*EDITION|DATE\s*OF\s*(?:ARRIVAL|ENTRY|HAWB|MAWB)", ctx): continue d = parse_date(dm.group(1)) if d: tied_dates.append((150, d)) if tied_dates: tied_dates.sort(key=lambda x: x[0], reverse=True) inv["cour_be_invoice_date"] = tied_dates[0][1] if not clean(inv.get("cour_be_inv_pymt_terms")): pay_candidates = [] for page_no, src in sources: src_compact = compact(src) identity = bool(invoice_compact and invoice_compact in src_compact) commercial = bool(re.search(r"\b(?:invoice|shipper|biller|consignee|sold\s+to|bill\s+to)\b", src, re.I)) if not (identity or commercial): continue for pat in ( r"(?:PAYMENT\s+TERMS?|TERMS\s+OF\s+PAYMENT|PAYMENT\s+TERM)\s*[:#-]?\s*([^\n\r|]+)", ): for m in re.finditer(pat, src, re.I): value = clean(m.group(1)) value = re.split(r"\b(?:INCOTERMS?|DATE|INVOICE|CURRENCY|TOTAL|FREIGHT|SHIPPER|CONSIGNEE)\b", value, maxsplit=1, flags=re.I)[0].strip(" :;,-") if value and not self._looks_like_any_known_label(value) and self._v183_numeric(value) is None: pay_candidates.append((120 + (50 if identity else 0), value)) # Two-column visual/table form: PAYMENT TERMS header followed # by the value in the same row or immediately adjacent row. for page_no, row in visual_rows: words = row.get("words") or [] labels = [w for w in words if re.fullmatch(r"PAYMENT\s+TERMS?|TERMS\s+OF\s+PAYMENT", str(w.get("_text", "")), re.I)] if not labels: continue for label in labels: for w in words: if float(w.get("x0", 0)) >= float(label.get("x1", 0)) - 1: value = clean(str(w.get("_text", ""))) if value and not re.fullmatch(r"PAYMENT|TERMS?|OF", value, re.I): pay_candidates.append((150, value)) rows = self.page_rows.get(page_no, []) try: idx = rows.index(row) except ValueError: idx = -1 if idx >= 0: for rr in rows[idx+1:idx+3]: value = clean(rr.get("text", "")) if value and not re.search(r"^(?:PAGE|INVOICE|INCOTERM|DATE)\b", value, re.I): pay_candidates.append((130, value)) if pay_candidates: pay_candidates.sort(key=lambda x: x[0], reverse=True) inv["cour_be_inv_pymt_terms"] = pay_candidates[0][1] # ---------------------------------------------------------- # 4. Manufacturer fallback from the commercial invoice. # ---------------------------------------------------------- # If the CBE explicitly says N/A, recover the strongest # manufacturer/producer party from the invoice content. Explicit # Manufacturer labels outrank Biller/Seller/Ship-to parties. for item in result.get("items") or []: current = clean(item.get("cour_be_item_manufacturer")) current = re.sub( r"(?i)^(?:N/?A|N\.A\.?)\s*(?:Address(?:\s+of\s+Manufacturer)?)?\s*$", "N/A", current, ).strip() current = re.sub(r"(?i)\s*Address\s*of\s*Manufacturer\s*$", "", current).strip() if re.match(r"(?i)^N/?\.?A\.?$", current.replace(" ", "")): current = "N/A" item["cour_be_item_manufacturer"] = current if current and current.upper() not in {"N/A", "NA", "N.A.", "N.A"}: continue candidates = [] for page_no, src in sources: # Some export declarations identify the actual exporter # immediately before an address/branch label rather than # using the word ``Manufacturer``. When the CBE says N/A, # that exporter is a valid document-local fallback for the # item manufacturer. No company name is hardcoded. exporter_patterns = [ r"(?is)(?:EXPÉDITEUR\s*/\s*EXPORTATEUR|EXPORTATEUR|EXPORTER)\s*.*?" r"(?:N°|NO\.?)[^\n]{0,80}\n?\s*" r"([A-Z][A-Z0-9&+.'/-]*(?:\s+[A-Z][A-Z0-9&+.'/-]*){1,8})" r"\s+(?=BUREAU\s+DE\s+RATTACHEMENT|ADDRESS|283\s+|\d{3,})", ] for pat in exporter_patterns: for em in re.finditer(pat, src, re.I): val = clean(em.group(1)) if val and not self._looks_like_any_known_label(val) and not self._looks_like_address(val): candidates.append((205, len(val), val)) # Explicit manufacturer labels are strongest. patterns = [ (300, r"(?:Manufacturer|Manufacturer\s+Name|Produced\s+By|Made\s+By)\s*[:#-]\s*([^\n\r|]+)"), # Commercial invoices frequently identify the producer # in the Biller/Seller block. This is only a fallback # when the CBE manufacturer is explicitly N/A. (180, r"(?:BILLER|SELLER|PRODUCER|MANUFACTURER|EXPORTER|EXPÉDITEUR\s*/\s*EXPORTATEUR)\s*[:#-]?\s*([^\n\r|]+)"), ] for score, pat in patterns: for m in re.finditer(pat, src, re.I): val = clean(m.group(1)) val = re.sub(r"(?i)\s*Address\s*of\s*Manufacturer\s*$", "", val).strip() if re.fullmatch(r"(?i)N/?\.?A\.?|NA", val.replace(" ", "")): continue if not val or self._looks_like_any_known_label(val): continue if re.search(r"\b(?:Address|Date|Invoice|Currency|Payment|Terms|Incoterm)\s*:", val, re.I): continue candidates.append((score, len(val), val)) # Visual label/value pairing supports two-column invoice forms. for page_no, row in visual_rows: words = row.get("words") or [] for label in words: if not re.fullmatch(r"MANUFACTURER|BILLER|PRODUCER|SELLER|EXPORTER|EXPÉDITEUR|EXPÉDITEUR\s*/\s*EXPORTATEUR", str(label.get("_text", "")), re.I): continue for w in words: if float(w.get("x0", 0)) > float(label.get("x1", 0)) + 1: val = clean(str(w.get("_text", ""))) if val and not self._looks_like_any_known_label(val): candidates.append((280 if str(label.get("_text", "")).upper() == "MANUFACTURER" else 170, len(val), val)) if candidates: candidates.sort(key=lambda x: (x[0], x[1]), reverse=True) item["cour_be_item_manufacturer"] = candidates[0][2] def _final_v21_cbe_xiv_issue_authority(self, result): """Final CBE-XIV authority for the verified text-layout issues. The CBE form/table is authoritative. The DHL inbound-charge invoice and repeated PDF headers are deliberately outside this extraction boundary. No OCR, fixed page numbers, or document-specific values are used. """ def clean(v): if v is None: return "" v = self._clean(v) v = re.sub(r"\s+", " ", v).strip(" :;,-") v = re.sub(r"\s*Page\s+\d+\s+of\s+\d+\s*$", "", v, flags=re.I) return v.strip(" :;,-") def norm_label(v): return re.sub(r"[^a-z0-9]+", "", str(v or "").lower()) # -------------------------------------------------------------- # Locate the CBE document dynamically from its own form marker and # footer numbering. This includes all continuation pages belonging to # the CBE while excluding later export declarations/invoices. # -------------------------------------------------------------- start_idx = None cbe_total = None for idx, page in enumerate(self.pages): text = str(page.get("text") or "") if re.search(r"Form\s+Regular\s+Bill\s+Of\s+entry\s*\(\s*CBE[- ]?XIV\s*\)|Courier\s+Bill\s+of\s+Entry[- ]?XIV", text, re.I): start_idx = idx fm = re.search(r"Page\s+\d+\s+of\s+(\d+)", text, re.I) if fm: cbe_total = int(fm.group(1)) break if start_idx is None: return cbe_pages = [] for idx in range(start_idx, len(self.pages)): text = str(self.pages[idx].get("text") or "") if idx == start_idx: cbe_pages.append(self.pages[idx]) continue footer = re.search(r"Page\s+(\d+)\s+of\s+(\d+)", text, re.I) if cbe_total is not None and footer and int(footer.group(2)) == cbe_total: cbe_pages.append(self.pages[idx]) if int(footer.group(1)) >= cbe_total: break continue # If a renderer omits the footer on a continuation page, include # it only while strong CBE-only section labels are present. if re.search(r"IGM\s+DETAILS|SUPPLIER\s+DETAILS|DETAILS\s+OF\s+INVOICE|DETAILS\s+OF\s+ITEM|DUTY\s+DETAILS|PAYMENT\s+DETAILS|DECLARATION\s*\(FOR\s+THE\s+COURIER", text, re.I): cbe_pages.append(self.pages[idx]) elif cbe_total is not None: break cbe_text = "\n".join(str(p.get("text") or "") for p in cbe_pages) def restore_spacing(candidate, source=cbe_text): """Restore spaces/case from the original text for a compacted table cell.""" candidate = clean(candidate) if not candidate: return "" target = re.sub(r"[^a-z0-9]+", "", candidate.lower()) if not target: return candidate compact_chars = [] source_pos = [] for pos, ch in enumerate(source): if ch.isalnum(): compact_chars.append(ch.lower()) source_pos.append(pos) compact_source = "".join(compact_chars) hit = compact_source.find(target) if hit < 0: return candidate a = source_pos[hit] b = source_pos[hit + len(target) - 1] + 1 return clean(source[a:b]) # Table rows are stronger than the flattened text stream for CBE-XIV. table_rows = [] for page in cbe_pages: for table in page.get("tables", []) or []: for row in table or []: cells = [clean(c) for c in (row or [])] if any(cells): table_rows.append(cells) def table_value(label): target = norm_label(label) for row in table_rows: for i, cell in enumerate(row): if norm_label(cell) == target: if i + 1 < len(row) and clean(row[i + 1]): return clean(row[i + 1]) # Handle a label and value that pdfplumber merged into one # cell, e.g. "Name: ABC". compact = norm_label(cell) if compact.startswith(target) and ":" in cell: value = clean(cell.split(":", 1)[1]) if value: return value return "" def section_row_index(marker): target = norm_label(marker) for ri, row in enumerate(table_rows): if any(target == norm_label(c) for c in row): return ri return None def row_field_value_after_section(marker, label, stop_marker=None): start = section_row_index(marker) if start is None: return "" target = norm_label(label) stop = norm_label(stop_marker) if stop_marker else None for row in table_rows[start + 1:]: labels = [norm_label(c) for c in row] if stop and any(stop == x for x in labels): break for i, cell in enumerate(row): if norm_label(cell) == target and i + 1 < len(row): return clean(row[i + 1]) return "" # 1. Importer. Table cell continuation is reconstructed naturally. importer = table_value("Name:") # There are several Name: rows. Prefer the row in the importer table # by locating the row whose label is Name and whose neighboring cell # is followed by Address in the same row. for row in table_rows: labels = [norm_label(c) for c in row] if "name" in labels and "address" in labels: ni = labels.index("name") ai = labels.index("address") if ni + 1 < len(row) and ai > ni: candidate = clean(row[ni + 1]) if candidate and self._strict_party_name(candidate): # The supplier row has the same shape; distinguish it # by being after the importer marker in document text. pass importer_block = re.search( r"PARTICULARS\s+OF\s+THE\s+IMPORTER(.*?)(?=PARTICULARS\s+OF\s+THE\s+HIGH\s+SEA\s+SELLER|SPECIAL\s+REQUESTS|Country\s+of\s+Origin)", cbe_text, re.I | re.S, ) if importer_block: im = re.search(r"Name\s*:\s*(.*?)\s+Address\s*:", importer_block.group(1), re.I | re.S) if im: candidate = clean(im.group(1)) candidate = re.sub(r"(?i)\s+(?:Account\s+Number|Invoice\s+Number)\s*:.*$", "", candidate).strip() # CBE-XIV may wrap the legal name below the Name/Address row. # Append only a genuine continuation row (typically starting # with a parenthesis/legal suffix), never an adjacent field. lines = importer_block.group(1).splitlines() for li, line in enumerate(lines): if re.search(r"\bName\s*:", line, re.I) and re.search(r"\bAddress\s*:", line, re.I): continuation = lines[li + 1] if li + 1 < len(lines) else "" continuation = clean(continuation) if continuation.startswith("(") and not re.search(r"\b(?:Category|Code|BOE|Address)\s*:", continuation, re.I): # Keep only the legal-name continuation. The same # PDF line may immediately continue into the # right-hand Address column. legal = re.match( r"^(\([^)]*\)\s+.*?\b(?:PRIVATE\s+LIMITED|LIMITED|LTD\.?|LLC|INC\.?|CORPORATION|CORP\.?)\b)", continuation, re.I, ) candidate = clean(candidate + " " + (legal.group(1) if legal else continuation)) break if self._strict_party_name(candidate): result["cour_be_importername"] = candidate else: # Table fallback for layouts where labels are separate cells. for row in table_rows: labels = [norm_label(c) for c in row] if labels[:1] == ["name"] and "address" in labels: candidate = clean(row[1] if len(row) > 1 else "") if self._strict_party_name(candidate): result["cour_be_importername"] = candidate break # The CBE table is authoritative when the visual text stream splits a # legal name across lines. Prefer the value cell in the importer row. table_importer = row_field_value_after_section( "PARTICULARS OF THE IMPORTER", "Name:", "PARTICULARS OF THE HIGH SEA SELLER" ) # Do not overwrite the raw CBE party extraction with pdfplumber's # compacted table-cell spelling; the raw section preserves legal-name # spacing and wrapped continuations. # 2. Courier from the authorized-courier section. courier_block = re.search( r"DETAILS\s+OF\s+AUTHORIZED\s+COURIER(.*?)(?=PARTICULARS\s+OF\s+THE\s+CUSTOMS\s+HOUSE\s+AGENT|PARTICULARS\s+OF\s+THE\s+IMPORTER)", cbe_text, re.I | re.S, ) if courier_block: cm = re.search( r"Name\s+of\s+the\s+Authorized\s+Courier\s*:\s*(.*?)\s+Address\s+of\s+Authorized\s+Courier\s*:", courier_block.group(1), re.I | re.S, ) if cm: candidate = clean(cm.group(1)) candidate = re.sub(r"(?i)^Courier\s*:\s*", "", candidate).strip() if self._strict_courier(candidate): result["cour_be_couriername"] = candidate table_courier = row_field_value_after_section( "DETAILS OF AUTHORIZED COURIER", "Name of the Authorized Courier", "PARTICULARS OF THE CUSTOMS HOUSE AGENT" ) # Prefer the raw authorized-courier block because the table extractor # may compact words and lose the legal-name spacing. if courier_block: raw_cb = courier_block.group(1) cm2 = re.search( r"Name\s+of\s+the\s+Authorized\b.*?\s+([A-Za-z][^\n]*?)\n[^\n]*\bCourier\s*:\s*([^\s]+)", raw_cb, re.I | re.S, ) if cm2: first = clean(cm2.group(1)) tail = clean(cm2.group(2)) if first and tail and tail.lower() not in {"address", "number"}: courier_candidate = clean(first + " " + tail) if self._strict_courier(courier_candidate): result["cour_be_couriername"] = courier_candidate # 3. Supplier + address from the actual SUPPLIER DETAILS table. supplier_block = re.search( r"SUPPLIER\s+DETAILS(.*?)(?=IF\s+SUPPLIER\s+IS\s+NOT\s+THE\s+SELLER|BROKER\s*/\s*AGENT\s+DETAILS|NATURE\s+OF\s+TRANSACTION)", cbe_text, re.I | re.S, ) if supplier_block: sb = supplier_block.group(1) sm = re.search(r"Name\s*:\s*(.*?)\s+Address\s*:\s*(.*)", sb, re.I | re.S) supplier = clean(sm.group(1)) if sm else "" address = clean(sm.group(2)) if sm else "" restored_supplier = restore_spacing(supplier) if self._strict_party_name(restored_supplier): supplier = restored_supplier result["cour_be_supplier_name"] = supplier elif self._strict_party_name(supplier): result["cour_be_supplier_name"] = supplier # Prefer the table cells around SUPPLIER DETAILS. The continuation # row on the next CBE page contains the country; only the country # itself is appended, while duplicate city/postcode spill is # discarded. # The raw supplier block contains the actual address text across # the page boundary. Start from that source; table cells may split # and compact the same address. supplier_address = address supplier_row_index = None for ri, row in enumerate(table_rows): if any(norm_label(c) == "supplierdetails" for c in row): supplier_row_index = ri break if supplier_row_index is not None and not supplier_address: for row in table_rows[supplier_row_index + 1:]: labels = [norm_label(c) for c in row] if "name" in labels and "address" in labels: ai = labels.index("address") if ai + 1 < len(row) and clean(row[ai + 1]): supplier_address = clean(row[ai + 1]) break # First continuation row after the supplier row. if len(row) >= 1 and clean(row[0]) and not any(norm_label(c) in {"ifsupplierisnotthe seller", "brokeragentdetails"} for c in row): if supplier_address: supplier_address = clean(supplier_address + ", " + clean(row[0])) break if not supplier_address: supplier_address = address supplier_address = restore_spacing(supplier_address) supplier_address = re.sub(r"(?i)Page\s+\d+\s+of\s+\d+", "", supplier_address) supplier_address = re.sub(r"\s*\*\s*,", ",", supplier_address) supplier_address = re.sub(r"\s*,\s*", ", ", supplier_address) supplier_address = re.sub(r"\s+", " ", supplier_address).strip(" ,") # Keep the first occurrence of the known document country after # the postal code. This handles PDF table spill such as # "..., LV-2167, Latvia, Marupe, Marupe, LATVIA, 2167". country = clean(result.get("cour_be_countryofconsignment") or result.get("cour_be_coo") or "") if country: cm_country = re.search(r"(?i)(LV-\d{3,6}|\d{4,6})\s*,?\s*" + re.escape(country) + r"\b", supplier_address) if cm_country: supplier_address = supplier_address[:cm_country.end()] if country: cmatch = re.search(r"(?i)(LV-\d{3,6}|\d{4,6})\s*,?\s*" + re.escape(country) + r"\b", supplier_address) if cmatch: supplier_address = supplier_address[:cmatch.end()] supplier_address = re.sub(r"\s*,\s*$", "", supplier_address) # If the address cell ends at a comma, recover the country from # the immediate continuation row or from the CBE country field. if supplier_address.endswith(",") or (country and re.search(r"(?i)\b" + re.escape(country) + r"\b", supplier_address) is None): for row in table_rows: for cell in row: cv = clean(cell) if not cv: continue if country and re.search(r"(?i)\b" + re.escape(country) + r"\b", cv): if re.search(r"(?i)\b(?:LV-\d{3,6}|\d{4,6})\s*,?\s*" + re.escape(country) + r"\b", supplier_address): break if supplier_address and re.search(r"(?i)\b" + re.escape(country) + r"\b", supplier_address) is None: source_country = re.search(r"(?i)\b" + re.escape(country) + r"\b", cv) country_value = source_country.group(0) if source_country else country supplier_address = supplier_address.rstrip(" ,") + ", " + country_value break if supplier_address: for inv in result.get("invoices") or []: inv["cour_be_inv_supplier_name"] = result.get("cour_be_supplier_name") or supplier inv["suppliers_address"] = supplier_address # 4. Invoice cardinality is authoritative when the CBE explicitly # declares Number of Invoices. count_match = re.search(r"Number\s+of\s+Invoices\s*:\s*(\d+)", cbe_text, re.I) explicit_count = int(count_match.group(1)) if count_match else None # 5. Rebuild invoice children exclusively from Details Of Invoice-N. inv_matches = list(re.finditer(r"Details\s+Of\s+Invoice\s*-\s*(\d+)", cbe_text, re.I)) if explicit_count is not None: inv_matches = [m for m in inv_matches if int(m.group(1)) <= explicit_count][:explicit_count] existing = result.get("invoices") or [] rebuilt = [] for pos, im in enumerate(inv_matches, 1): serial = int(im.group(1)) tail = cbe_text[im.end():] end = re.search(r"Details\s+Of\s+Invoice\s*-\s*\d+|DETAILS\s+OF\s+ITEM\s*[-:]\s*\d+|DECLARATION", tail, re.I) block = tail[:end.start()] if end else tail[:10000] inv = dict(existing[pos - 1]) if pos - 1 < len(existing) else self._empty_invoice(serial) inv["cour_be_invsno"] = serial def field_value(label, stops): mm = re.search(label + r"\s*:\s*(.*)", block, re.I | re.S) if not mm: return "" v = clean(mm.group(1)) if stops: v = re.split(r"(?i)\s+(?:" + "|".join(stops) + r")\s*:", v, maxsplit=1)[0] return clean(v) v = field_value(r"Invoice\s+Number", [r"Date\s+of\s+Invoice", r"Purchase\s+Order\s+Number", r"Date\s+of\s+Purchase\s+Order", r"Contract\s+Number"]) if v: inv["cour_be_invoice_no"] = v v = field_value(r"Date\s+of\s+Invoice", [r"Purchase\s+Order\s+Number", r"Date\s+of\s+Purchase\s+Order", r"Contract\s+Number"]) dm = re.search(r"\d{1,2}[/-]\d{1,2}[/-]\d{2,4}", v) if dm: inv["cour_be_invoice_date"] = self._format_date(dm.group(0)) v = field_value(r"Nature\s+of\s+Transaction", [r"If\s+Others", r"Terms\s+of\s+Payment", r"Conditions\s+or\s+Restrictions", r"Method\s+of\s+Valuation", r"Terms\s+of\s+Invoice"]) if v: inv["cour_be_inv_nature_of_txn"] = v v = field_value(r"Terms\s+of\s+Payment", [r"If\s+Others", r"Conditions\s+or\s+Restrictions", r"Method\s+of\s+Valuation", r"Terms\s+of\s+Invoice"]) if v: inv["cour_be_inv_pymt_terms"] = v v = field_value(r"Method\s+of\s+Valuation", [r"Terms\s+of\s+Invoice", r"Invoice\s+Value", r"Currency"]) if v: inv["cour_be_valuationmethod"] = v v = field_value(r"Terms\s+of\s+Invoice", [r"Invoice\s+Value", r"Currency", r"FREIGHT", r"INSURANCE"]) if v: inv["cour_be_incoterm"] = v v = field_value(r"Invoice\s+Value", [r"Currency", r"FREIGHT", r"INSURANCE"]) nm = re.search(r"-?\d[\d,]*(?:\.\d+)?", v) if nm: inv["cour_be_inv_value"] = float(nm.group(0).replace(",", "")) v = field_value(r"Currency", [r"FREIGHT", r"INSURANCE", r"LOADING"]) if re.fullmatch(r"[A-Za-z]{3}", v): inv["cour_be_inv_currency"] = v.upper() if result.get("cour_be_supplier_name"): inv["cour_be_inv_supplier_name"] = result["cour_be_supplier_name"] rebuilt.append(inv) if rebuilt: result["invoices"] = rebuilt elif explicit_count == 0: result["invoices"] = [] if explicit_count is not None: result["cour_be_invoicecount"] = explicit_count else: result["cour_be_invoicecount"] = len(result.get("invoices") or []) # 6. Item identity comes from Details Of Item-N. With one declared # invoice, every surviving item links to invoice serial 1. item_matches = list(re.finditer(r"Details\s+Of\s+Item\s*[-:]\s*(\d+)", cbe_text, re.I)) for idx, item in enumerate(result.get("items") or []): serial = int(item_matches[idx].group(1)) if idx < len(item_matches) else idx + 1 item["cour_be_item_itemsn"] = serial item["linked_invoice_id_sno"] = 1 if result.get("invoices") else None if explicit_count == 1 and result.get("invoices"): result["invoices"] = [result["invoices"][0]] result["invoices"][0]["cour_be_invsno"] = 1 result["cour_be_invoicecount"] = 1 for item in result.get("items") or []: item["linked_invoice_id_sno"] = 1 if not item.get("cour_be_item_itemsn"): item["cour_be_item_itemsn"] = 1 def _final_waybill_mawb_and_ritc_authority(self, result): """Recover waybill-derived MAWB fields and missing item tariff code.""" text = str(getattr(self, "full_text", "") or "") if not text: return def clean(v): return re.sub(r"\s+", " ", self._clean(v or "")).strip(" :;,-") def parse_date(v): v = clean(v).replace(" ", "") for fmt in ("%Y-%m-%d", "%Y/%m/%d", "%Y.%m.%d", "%d-%m-%Y", "%d/%m/%Y", "%d.%m.%Y", "%d-%b-%Y", "%d/%b/%Y", "%d-%B-%Y", "%d/%B/%Y"): try: return datetime.strptime(v, fmt).strftime("%Y-%m-%d") except ValueError: pass return "" # Compact CBE HAWB label recovery. if not clean(result.get("cour_be_hawbno")): for pat in ( r"HAWB\s*Number\s*[:：-]?\s*([0-9]{6,20})", r"House\s+Airway\s+Bill\s*\(\s*HAWB\s*\)\s*Number\s*[:：-]?\s*([0-9]{6,20})", r"House\s+Airway\s+Bill\s+Number\s*[:：-]?\s*([0-9]{6,20})", ): m = re.search(pat, text, re.I) if m: result["cour_be_hawbno"] = m.group(1) break # Explicit MAWB always has priority. if not clean(result.get("cour_be_mawbno")): for pat in ( r"Master\s+Airway\s+Bill\s*\(\s*MAWB\s*\)\s*Number\s*[:：-]?\s*([A-Za-z0-9][A-Za-z0-9./_-]{5,30})", r"Master\s+Airway\s+Bill\s+Number\s*[:：-]?\s*([A-Za-z0-9][A-Za-z0-9./_-]{5,30})", r"MAWB\s*(?:Number|No\.?)\s*[:：-]?\s*([A-Za-z0-9][A-Za-z0-9./_-]{5,30})", ): m = re.search(pat, text, re.I) if m: result["cour_be_mawbno"] = clean(m.group(1)) break # Standalone courier waybill: use its printed number only when no true # MAWB is present. The Data date is the shipment/waybill date. waybill_no = "" waybill_date = "" for wm in re.finditer(r"(?is)\*?\s*WAYBILL\s+DOC\s*\*?", text): block = text[wm.start():wm.start() + 2500] nums = re.findall(r"(?i)\bWAYBILL\s+((?:\d[ \t]*){8,20})", block) for raw in nums: val = re.sub(r"\s+", "", raw) if val.isdigit() and 8 <= len(val) <= 20: waybill_no = val m = re.search( r"\b(\d{4}[-/.]\d{1,2}[-/.]\d{1,2})\s*\(\s*Data\s*\)", block, re.I, ) if m: waybill_date = parse_date(m.group(1)) if waybill_no or waybill_date: break if not clean(result.get("cour_be_mawbno")) and waybill_no: result["cour_be_mawbno"] = waybill_no if not clean(result.get("cour_be_mawb_dt")) and waybill_date: result["cour_be_mawb_dt"] = waybill_date # Missing RITC: preserve an explicit RITC if already extracted. If the # document has no RITC label but exposes the item Customs tariff code, # use CTSH/CETSH as the semantic tariff fallback. for item in result.get("items") or []: if not isinstance(item, dict) or clean(item.get("cour_be_item_ritc")): continue code = "" for key in ("cour_be_item_ctsh", "cour_be_item_cetsh"): candidate = re.sub(r"\s+", "", str(item.get(key) or "")) if re.fullmatch(r"\d{4,12}", candidate): code = candidate break if not code: m = re.search( r"(?:Import\s+Country\s+HS\s+Code|Country\s+HS\s+Code|HS\s+Code|RITC|Tariff\s+(?:Item|Code))\s*[:#-]?\s*(\d{4,12})", text, re.I, ) if m: code = m.group(1) if code: item["cour_be_item_ritc"] = code def parse(self) -> dict[str, Any]: # Preserve the proven v18.3/final23 extraction pipeline first. base = self._parse_base() result = base.get("courier_bill_json", base) # Run existing recovery/duty logic, but do NOT run the old coordinate # authority that caused the reported CBE-XIII bounding-box regressions. self._final_dynamic_issue_fixes(result) self._normalize_courier_output_contract(result) self._finalize_parent_fields(result) self._finalize_children(result) self._schema_complete(result) result["cour_be_invoicecount"] = len(result.get("invoices") or []) for i, inv in enumerate(result.get("invoices") or [], 1): inv["cour_be_invsno"] = i # One final semantic authority pass. It is intentionally the last # business-field operation so broad legacy extractors cannot overwrite # the corrected values. No OCR is used. self._v183_final_semantic_authority(result) # Preserve duty-table authority after schema normalization as well. self._v183_duty_authority(result) # Final CBE-XIII semantic authority. This MUST be the last business # extraction pass because the generic parser can otherwise reintroduce # truncated/interleaved party values during reconciliation. self._final_cbe_xiii_label_authority(result) # V5: the detected CBE-XIII table is the authoritative source for # wrapped/interleaved header and party cells. This runs after all # generic semantic passes so those passes cannot reintroduce truncation. self._final_cbe_xiii_table_authority(result) # V5: invoice metadata is recovered from text-based attached invoices # when present. No OCR is performed here. self._final_invoice_text_authority_v5(result) result["cour_be_invoicecount"] = len(result.get("invoices") or []) for i, inv in enumerate(result.get("invoices") or [], 1): inv["cour_be_invsno"] = i self._schema_complete(result) self._final_v7_spaced_cbe_recovery(result) self._final_v6_contract_authority(result) self._final_invoice_count_authority(result) # V14 final authority: normalize malformed carrier/flight spacing and # recover invoice date/payment terms from analyzer text or tables. self._final_v14_invoice_and_flight_authority(result) # V15 is the final commercial-invoice authority. It deliberately runs # after all legacy/CBE passes so a generic CBE "Other" value cannot # overwrite an explicitly labelled commercial payment term. self._final_v15_commercial_invoice_authority(result) # V16 final output contract: enforce requested schema/types and remove # internal diagnostics from the public response. self._final_v16_output_contract(result) # V19 final authority MUST run after the public schema contract so # BOE/HAWB dates and invoice metadata cannot be reset by finalization. # It is semantic/visual only and performs no OCR. self._final_v19_date_hawb_manufacturer_authority(result) # V21 final CBE-XIV authority for the verified importer/courier/supplier, # invoice cardinality/identity/terms, supplier address, and item linkage. self._final_v21_cbe_xiv_issue_authority(result) # Final CBE country/category/item authority. Explicitly maps # Country of Exportation to country-of-consignment, recovers the # Government/Non-Government category, and guarantees item identity # and parent-invoice linkage from the detected CBE sections. self._final_cbe_country_category_item_authority(result) # Absolute final type boundary. This is deliberately after every # extraction, reconciliation and schema pass so no later operation can # turn Float duty values back into integer objects. # Final CBE-XIII manufacturer authority: the explicit Manufacturer # field outranks unrelated invoice/export-certificate party text. self._final_waybill_mawb_and_ritc_authority(result) self._final_waybill_ritc_svbdate_authority(result) self._final_cbe_xiii_manufacturer_field_authority(result) self._final_numeric_duty_float_guard(result) return {"courier_bill_json": result} @staticmethod def dumps_json(output: dict[str, Any], **kwargs) -> str: """Serialize parser output with Python's JSON encoder. This helper is provided for integrations that need the textual JSON representation. Python's encoder preserves Float values as ``2270.0`` rather than converting them to integer literals. It intentionally does not add file_id, processing_time, or any internal metadata. """ import json return json.dumps(output, **kwargs)