OCR multi-format (liste + detail) + matching accents + debug
This commit is contained in:
@@ -44,6 +44,14 @@
|
||||
Voir <a href="/admin/compta/depense/">dépenses</a> · <a href="/admin/compta/facture/">factures</a>.</div>
|
||||
{% endif %}
|
||||
|
||||
{% if ocr_debug %}
|
||||
<details class="card">
|
||||
<summary style="cursor:pointer;color:var(--muted)">Texte lu par l'OCR (debug — {{ ocr_debug|length }} ligne(s))</summary>
|
||||
<pre style="white-space:pre-wrap;font-size:12px;color:var(--muted);margin-top:8px">{% for l in ocr_debug %}{{ l }}
|
||||
{% endfor %}</pre>
|
||||
</details>
|
||||
{% endif %}
|
||||
|
||||
{% if not lignes %}
|
||||
<form method="post" enctype="multipart/form-data" class="card" id="srcForm">
|
||||
{% csrf_token %}
|
||||
|
||||
+124
-85
@@ -1,11 +1,18 @@
|
||||
"""
|
||||
Import d'opérations : par CSV (collé/uploadé) ou par OCR d'une capture d'écran
|
||||
(pytesseract). Dans les deux cas, un aperçu ÉDITABLE permet de corriger avant
|
||||
d'enregistrer (l'OCR n'est jamais parfait).
|
||||
Import d'opérations : CSV (collé/uploadé) ou OCR d'une capture d'écran.
|
||||
|
||||
OCR multi-format :
|
||||
- LISTE : le tableau d'opérations (date empilée à gauche, lignes). Découpage
|
||||
ancré sur le montant ; la date est reportée d'une ligne à l'autre
|
||||
(Indy ne l'affiche qu'une fois par jour). « UE » => intracom.
|
||||
- DÉTAIL : la fiche d'une opération (Libellé / Date / Montant / Ventilation).
|
||||
Extraction au mieux (les petits libellés gris sont mal lus).
|
||||
Dans tous les cas : aperçu ÉDITABLE + panneau de debug OCR.
|
||||
"""
|
||||
import csv
|
||||
import io
|
||||
import re
|
||||
import unicodedata
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from datetime import datetime, date as date_cls
|
||||
|
||||
@@ -20,6 +27,7 @@ MOIS = {"janvier": 1, "février": 2, "fevrier": 2, "mars": 3, "avril": 4, "mai":
|
||||
"juin": 6, "juillet": 7, "août": 8, "aout": 8, "septembre": 9,
|
||||
"octobre": 10, "novembre": 11, "décembre": 12, "decembre": 12}
|
||||
AMOUNT_RE = re.compile(r"^-?\d[\d .]*[.,]\d{2}$")
|
||||
DATE_RE = re.compile(r"^\d{2}/\d{2}/\d{4}$")
|
||||
|
||||
|
||||
def _dec(v, d="0"):
|
||||
@@ -27,8 +35,7 @@ def _dec(v, d="0"):
|
||||
s = str(v or d)
|
||||
for ch in (" ", " ", " "):
|
||||
s = s.replace(ch, "")
|
||||
s = s.replace(",", ".").strip()
|
||||
return Decimal(s or d)
|
||||
return Decimal(s.replace(",", ".").strip() or d)
|
||||
except (InvalidOperation, AttributeError):
|
||||
return Decimal(d)
|
||||
|
||||
@@ -43,12 +50,6 @@ def _date(s):
|
||||
return None
|
||||
|
||||
|
||||
def _activite(s):
|
||||
s = (s or "").strip().lower()
|
||||
return {"vente": "vente", "service_bic": "service_bic", "service_bnc": "service_bnc",
|
||||
"bic": "service_bic", "bnc": "service_bnc"}.get(s, "")
|
||||
|
||||
|
||||
def _regime(s):
|
||||
s = (s or "").strip().lower()
|
||||
if s in ("intracom", "intracommunautaire", "ue"):
|
||||
@@ -58,11 +59,21 @@ def _regime(s):
|
||||
return RegimeTva.NATIONALE
|
||||
|
||||
|
||||
def _noacc(s):
|
||||
return "".join(c for c in unicodedata.normalize("NFD", (s or "").lower())
|
||||
if unicodedata.category(c) != "Mn")
|
||||
|
||||
|
||||
def _match_cat(text, cats):
|
||||
low = (text or "").lower()
|
||||
for nom_low, c in cats.items():
|
||||
low = _noacc(text)
|
||||
items = [(_noacc(nom), c) for nom, c in cats.items()]
|
||||
for nom_low, c in items:
|
||||
if nom_low and nom_low in low:
|
||||
return c
|
||||
for nom_low, c in items:
|
||||
mots = [m for m in re.split(r"[ ,/.]+", nom_low) if len(m) > 4]
|
||||
if mots and mots[0] in low:
|
||||
return c
|
||||
return None
|
||||
|
||||
|
||||
@@ -73,14 +84,12 @@ def _row(dt, libelle, montant, taux=Decimal("0"), regime=RegimeTva.NATIONALE,
|
||||
typ = "Mouvement neutre" if neutre else "Vente"
|
||||
else:
|
||||
typ = "Dépense neutre" if neutre else "Dépense"
|
||||
return {
|
||||
"n": n, "date": dt.isoformat() if dt else "", "libelle": libelle,
|
||||
"categorie_id": categorie.pk if categorie else "",
|
||||
"taux": taux, "regime": regime, "montant": montant, "type": typ,
|
||||
}
|
||||
return {"n": n, "date": dt.isoformat() if dt else "", "libelle": libelle,
|
||||
"categorie_id": categorie.pk if categorie else "",
|
||||
"taux": taux, "regime": regime, "montant": montant, "type": typ}
|
||||
|
||||
|
||||
# ---- Parsing CSV --------------------------------------------------------
|
||||
# ---- CSV ----------------------------------------------------------------
|
||||
|
||||
def _parse_csv(text, cats):
|
||||
text = text.lstrip("")
|
||||
@@ -101,43 +110,33 @@ def _parse_csv(text, cats):
|
||||
cat = cats.get((c[2] or "").strip().lower())
|
||||
lignes.append(_row(dt, (c[1] or "").strip() or "(sans libellé)", montant,
|
||||
_dec(c[4]), _regime(c[5]), cat, n))
|
||||
return lignes, erreurs
|
||||
return lignes, erreurs, []
|
||||
|
||||
|
||||
# ---- Parsing OCR --------------------------------------------------------
|
||||
# ---- OCR ----------------------------------------------------------------
|
||||
|
||||
def _ocr_lang():
|
||||
try:
|
||||
import pytesseract
|
||||
return "fra" if "fra" in pytesseract.get_languages(config="") else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _parse_ocr(image_file, annee, cats):
|
||||
def _ocr_words(image_file):
|
||||
try:
|
||||
import pytesseract
|
||||
from pytesseract import Output
|
||||
from PIL import Image
|
||||
except Exception:
|
||||
return [], ["OCR indisponible (pytesseract / Pillow non installés)."]
|
||||
return None, ["OCR indisponible (pytesseract / Pillow non installés)."]
|
||||
try:
|
||||
img = Image.open(image_file)
|
||||
except Exception:
|
||||
return [], ["Image illisible."]
|
||||
return None, ["Image illisible."]
|
||||
if img.width < 1000:
|
||||
ratio = 1400 / img.width
|
||||
img = img.resize((int(img.width * ratio), int(img.height * ratio)))
|
||||
kwargs = {"output_type": Output.DICT}
|
||||
lang = _ocr_lang()
|
||||
if lang:
|
||||
kwargs["lang"] = lang
|
||||
r = 1400 / img.width
|
||||
img = img.resize((int(img.width * r), int(img.height * r)))
|
||||
try:
|
||||
data = pytesseract.image_to_data(img, **kwargs)
|
||||
lang = "fra" if "fra" in pytesseract.get_languages(config="") else None
|
||||
kw = {"output_type": Output.DICT}
|
||||
if lang:
|
||||
kw["lang"] = lang
|
||||
data = pytesseract.image_to_data(img, **kw)
|
||||
except Exception as ex:
|
||||
return [], [f"OCR impossible ({ex}). Vérifie l'installation de Tesseract, "
|
||||
f"ou utilise l'import CSV."]
|
||||
|
||||
return None, [f"OCR impossible ({ex}). Vérifie Tesseract, ou utilise le CSV."]
|
||||
words = []
|
||||
for i in range(len(data["text"])):
|
||||
t = data["text"][i].strip()
|
||||
@@ -146,58 +145,99 @@ def _parse_ocr(image_file, annee, cats):
|
||||
except (ValueError, TypeError):
|
||||
conf = -1
|
||||
if t and conf > 20:
|
||||
words.append((data["top"][i], data["left"][i], t))
|
||||
words.sort()
|
||||
words.append({"x": data["left"][i],
|
||||
"yc": data["top"][i] + data["height"][i] / 2, "t": t})
|
||||
return words, []
|
||||
|
||||
visual_rows, cur, cur_top = [], [], None
|
||||
for top, left, t in words:
|
||||
if cur_top is None or abs(top - cur_top) <= 18:
|
||||
cur.append((left, t))
|
||||
cur_top = top if cur_top is None else cur_top
|
||||
|
||||
def _debug_lines(words):
|
||||
out, cur, top = [], [], None
|
||||
for w in sorted(words, key=lambda w: (w["yc"], w["x"])):
|
||||
if top is None or abs(w["yc"] - top) <= 14:
|
||||
cur.append(w); top = w["yc"] if top is None else top
|
||||
else:
|
||||
visual_rows.append(cur); cur = [(left, t)]; cur_top = top
|
||||
out.append(" ".join(x["t"] for x in sorted(cur, key=lambda x: x["x"])))
|
||||
cur = [w]; top = w["yc"]
|
||||
if cur:
|
||||
visual_rows.append(cur)
|
||||
out.append(" ".join(x["t"] for x in sorted(cur, key=lambda x: x["x"])))
|
||||
return out
|
||||
|
||||
lignes = []
|
||||
for n, row in enumerate(visual_rows, start=1):
|
||||
toks = [t for _, t in sorted(row)]
|
||||
dt = None
|
||||
for i, t in enumerate(toks):
|
||||
if t.isdigit() and i + 1 < len(toks):
|
||||
m = MOIS.get(toks[i + 1].lower().strip(".,"))
|
||||
if m and 1 <= int(t) <= 31:
|
||||
dt = date_cls(annee, m, int(t)); break
|
||||
montant, amount_idx = None, None
|
||||
for i in range(len(toks) - 1, -1, -1):
|
||||
if "%" not in toks[i] and AMOUNT_RE.match(toks[i]):
|
||||
montant = _dec(toks[i]); amount_idx = i; break
|
||||
|
||||
def _parse_liste(words, annee, cats):
|
||||
anchors = sorted([w for w in words if "%" not in w["t"] and AMOUNT_RE.match(w["t"])],
|
||||
key=lambda w: w["yc"])
|
||||
lignes, last = [], None
|
||||
for n, a in enumerate(anchors, start=1):
|
||||
band = sorted([w for w in words if abs(w["yc"] - a["yc"]) <= 22], key=lambda w: w["x"])
|
||||
day = mois = None
|
||||
taux = Decimal("0")
|
||||
for t in toks:
|
||||
regime = RegimeTva.NATIONALE
|
||||
reste = []
|
||||
for w in band:
|
||||
t = w["t"]
|
||||
if w["x"] < 150 and t.isdigit() and 1 <= int(t) <= 31:
|
||||
day = int(t); continue
|
||||
if w["x"] < 150 and t.lower().strip(".,") in MOIS:
|
||||
mois = MOIS[t.lower().strip(".,")]; continue
|
||||
if "%" in t:
|
||||
taux = _dec(t.replace("%", ""))
|
||||
if dt is None or montant is None or montant == 0:
|
||||
continue
|
||||
reste, skip = [], False
|
||||
for i, t in enumerate(toks):
|
||||
if i == amount_idx or "%" in t or t.upper() == "TVA":
|
||||
taux = _dec(t.replace("%", "")); continue
|
||||
if t.upper() == "UE":
|
||||
regime = RegimeTva.INTRACOM; continue
|
||||
if w is a or (t == a["t"] and w["x"] == a["x"]):
|
||||
continue
|
||||
if t.upper() == "TVA":
|
||||
continue
|
||||
if t.isdigit() and i + 1 < len(toks) and toks[i + 1].lower().strip(".,") in MOIS:
|
||||
skip = True; continue
|
||||
if skip and t.lower().strip(".,") in MOIS:
|
||||
skip = False; continue
|
||||
reste.append(t)
|
||||
if day and mois:
|
||||
dt = date_cls(annee, mois, day); last = dt
|
||||
else:
|
||||
dt = last
|
||||
texte = " ".join(reste).strip()
|
||||
cat = _match_cat(texte, cats)
|
||||
libelle = texte
|
||||
if cat:
|
||||
libelle = re.sub(re.escape(cat.nom), "", texte, flags=re.I).strip(" -·,")
|
||||
lignes.append(_row(dt, libelle or texte or "(sans libellé)", montant, taux,
|
||||
RegimeTva.NATIONALE, cat, n))
|
||||
erreurs = [] if lignes else [
|
||||
"Aucune ligne exploitable détectée. Réessaie avec une capture plus nette, "
|
||||
"ou utilise le CSV."]
|
||||
return lignes, erreurs
|
||||
lignes.append(_row(dt, libelle or texte or "(sans libellé)", _dec(a["t"]),
|
||||
taux, regime, cat, n))
|
||||
return lignes
|
||||
|
||||
|
||||
def _parse_detail(words, cats):
|
||||
dtok = next((w for w in words if DATE_RE.match(w["t"])), None)
|
||||
dt = _date(dtok["t"]) if dtok else None
|
||||
amounts = [w for w in words if "%" not in w["t"] and AMOUNT_RE.match(w["t"])]
|
||||
montant = _dec(amounts[0]["t"]) if amounts else Decimal("0")
|
||||
y_ref = dtok["yc"] if dtok else 99999
|
||||
haut_gauche = [w for w in words if w["x"] < 450 and w["yc"] < y_ref
|
||||
and w["t"].lower() not in ("informations", "date", "libellé", "libelle")]
|
||||
libelle = " ".join(w["t"] for w in sorted(haut_gauche, key=lambda w: (w["yc"], w["x"])))
|
||||
libelle = libelle.strip(" ‘'\"") or "(à compléter)"
|
||||
droite = [w for w in words if w["x"] > 480 and not AMOUNT_RE.match(w["t"]) and "%" not in w["t"]
|
||||
and w["t"].lower() not in ("ventilation(s)", "ventilation", "catégorie",
|
||||
"categorie", "montant", "ttc", "tva")]
|
||||
texte_cat = " ".join(w["t"] for w in sorted(droite, key=lambda w: (w["yc"], w["x"])))
|
||||
cat = _match_cat(texte_cat, cats)
|
||||
if montant == 0:
|
||||
return []
|
||||
return [_row(dt, libelle, montant, Decimal("0"), RegimeTva.NATIONALE, cat, 1)]
|
||||
|
||||
|
||||
def _parse_ocr(image_file, annee, cats):
|
||||
words, erreurs = _ocr_words(image_file)
|
||||
if words is None:
|
||||
return [], erreurs, []
|
||||
debug = _debug_lines(words)
|
||||
allt = " ".join(w["t"] for w in words).lower()
|
||||
is_detail = ("ventilation" in allt) or ("informations" in allt and "montant" in allt)
|
||||
if is_detail:
|
||||
lignes = _parse_detail(words, cats); fmt = "détail"
|
||||
else:
|
||||
lignes = _parse_liste(words, annee, cats); fmt = "liste"
|
||||
if not lignes:
|
||||
erreurs.append(f"Format « {fmt} » détecté mais aucune ligne exploitable. "
|
||||
f"Regarde le texte OCR ci-dessous et complète à la main, "
|
||||
f"ou utilise le CSV.")
|
||||
return lignes, erreurs, debug
|
||||
|
||||
|
||||
# ---- Création -----------------------------------------------------------
|
||||
@@ -224,7 +264,6 @@ def _creer(entreprise, dt, libelle, montant, taux, regime, categorie):
|
||||
prix_unitaire_ht=ht, taux_tva=taux)
|
||||
f.recompute_totaux()
|
||||
return "vente"
|
||||
# montant_tva est calculé automatiquement dans Depense.save() depuis le taux.
|
||||
Depense.objects.create(entreprise=entreprise, date=dt, libelle=libelle,
|
||||
categorie=categorie, regime_tva=regime,
|
||||
taux_tva=taux, montant_ttc=-montant)
|
||||
@@ -272,18 +311,18 @@ def import_csv(request):
|
||||
ctx["resultat"] = compteur
|
||||
return render(request, "compta/import_csv.html", ctx)
|
||||
|
||||
# Aperçu : CSV ou OCR
|
||||
image = request.FILES.get("image")
|
||||
if image:
|
||||
annee = int(request.POST.get("annee") or date_cls.today().year)
|
||||
lignes, erreurs = _parse_ocr(image, annee, cats)
|
||||
lignes, erreurs, debug = _parse_ocr(image, annee, cats)
|
||||
ctx["source"] = "ocr"
|
||||
ctx["ocr_debug"] = debug
|
||||
else:
|
||||
texte = request.POST.get("texte", "")
|
||||
fichier = request.FILES.get("fichier")
|
||||
if fichier:
|
||||
texte = fichier.read().decode("utf-8-sig", "ignore")
|
||||
lignes, erreurs = _parse_csv(texte, cats)
|
||||
lignes, erreurs, debug = _parse_csv(texte, cats)
|
||||
ctx["texte"] = texte
|
||||
ctx["source"] = "csv"
|
||||
ctx["lignes"] = lignes
|
||||
|
||||
Reference in New Issue
Block a user