diff --git a/app/main.py b/app/main.py index d96da4b..ae4ad5f 100644 --- a/app/main.py +++ b/app/main.py @@ -1,24 +1,187 @@ -from fastapi import FastAPI -from fastapi.staticfiles import StaticFiles +from decimal import Decimal +from fastapi import FastAPI, Request +from fastapi.staticfiles import StaticFiles +from fastapi.templating import Jinja2Templates +from sqlalchemy import create_engine, func +from sqlalchemy.orm import sessionmaker + +from app.core.config import DATABASE_URL +from app.models.document import Document +from app.models.extracted_field import ExtractedField +from app.models.receipt_line_item import ReceiptLineItem from app.routes.documents import router as documents_router from app.routes.health import router as health_router from app.routes.ingest import router as ingest_router +from app.routes.line_items import router as line_items_router from app.routes.queue import router as queue_router from app.routes.trash import router as trash_router app = FastAPI(title="document-processor") app.mount("/static", StaticFiles(directory="app/static"), name="static") - app.mount("/files", StaticFiles(directory="/mnt/storage/document-processor"), name="files") app.include_router(health_router) app.include_router(documents_router) app.include_router(ingest_router) +app.include_router(line_items_router) app.include_router(queue_router) app.include_router(trash_router) +templates = Jinja2Templates(directory="app/templates") + +engine = create_engine(DATABASE_URL) +SessionLocal = sessionmaker(bind=engine) + + +def _to_str(value) -> str: + if value is None: + return "" + return str(value) + @app.get("/") -def root(): - return {"app": "document-processor", "status": "running"} +def root_dashboard(request: Request): + db = SessionLocal() + try: + total_documents = db.query(func.count(Document.id)).scalar() or 0 + active_documents = ( + db.query(func.count(Document.id)) + .filter(Document.is_trashed.is_(False)) + .scalar() + or 0 + ) + reviewed_documents = ( + db.query(func.count(Document.id)) + .filter(Document.is_trashed.is_(False)) + .filter(Document.review_status == "reviewed") + .scalar() + or 0 + ) + pending_review_documents = ( + db.query(func.count(Document.id)) + .filter(Document.is_trashed.is_(False)) + .filter(Document.review_status != "reviewed") + .scalar() + or 0 + ) + trashed_documents = ( + db.query(func.count(Document.id)) + .filter(Document.is_trashed.is_(True)) + .scalar() + or 0 + ) + extracted_documents = db.query(func.count(ExtractedField.id)).scalar() or 0 + total_line_items = db.query(func.count(ReceiptLineItem.id)).scalar() or 0 + cocktail_count = ( + db.query(func.count(ReceiptLineItem.id)) + .filter(func.lower(ReceiptLineItem.item_category) == "cocktail") + .scalar() + or 0 + ) + + quality_candidates = ( + db.query(ReceiptLineItem.extra_json) + .filter(func.lower(ReceiptLineItem.item_category) == "cocktail") + .all() + ) + + rated_cocktails = 0 + pending_cocktail_reviews = 0 + na_cocktails = 0 + rating_sum = Decimal("0") + + for (extra_json,) in quality_candidates: + extra = extra_json or {} + status = str(extra.get("quality_status") or "").strip().lower() + rating_raw = str(extra.get("quality_rating") or "").strip() + + rating_dec = None + if rating_raw: + try: + rating_dec = Decimal(rating_raw) + except Exception: + rating_dec = None + + if status == "na": + na_cocktails += 1 + elif rating_dec is not None: + rated_cocktails += 1 + rating_sum += rating_dec + else: + pending_cocktail_reviews += 1 + + avg_cocktail_rating = "" + if rated_cocktails > 0: + avg_cocktail_rating = str((rating_sum / Decimal(rated_cocktails)).quantize(Decimal("0.01"))) + + recent_documents = ( + db.query(Document) + .filter(Document.is_trashed.is_(False)) + .order_by(Document.updated_at.desc()) + .limit(10) + .all() + ) + + recent_line_items = ( + db.query(ReceiptLineItem) + .join(Document, ReceiptLineItem.document_id == Document.id) + .filter(Document.is_trashed.is_(False)) + .order_by(ReceiptLineItem.updated_at.desc()) + .limit(10) + .all() + ) + + recent_line_item_rows = [] + for item in recent_line_items: + doc = item.document + merchant = "" + transaction_date = "" + if doc and doc.extracted_fields: + extracted = sorted( + doc.extracted_fields, + key=lambda x: x.updated_at or x.created_at, + reverse=True, + )[0] + merchant = extracted.merchant_normalized or extracted.merchant_raw or "" + if extracted.transaction_date: + transaction_date = extracted.transaction_date.isoformat() + + extra = item.extra_json or {} + recent_line_item_rows.append( + { + "document_id": doc.document_id if doc else "", + "merchant": merchant, + "transaction_date": transaction_date, + "description": item.normalized_description or item.raw_description or "", + "category": item.item_category or "", + "line_total": _to_str(item.line_total), + "quality_rating": _to_str(extra.get("quality_rating")), + "quality_status": _to_str(extra.get("quality_status")), + } + ) + + return templates.TemplateResponse( + request=request, + name="dashboard.html", + context={ + "request": request, + "active_page": "", + "total_documents": total_documents, + "active_documents": active_documents, + "reviewed_documents": reviewed_documents, + "pending_review_documents": pending_review_documents, + "trashed_documents": trashed_documents, + "extracted_documents": extracted_documents, + "total_line_items": total_line_items, + "cocktail_count": cocktail_count, + "rated_cocktails": rated_cocktails, + "pending_cocktail_reviews": pending_cocktail_reviews, + "na_cocktails": na_cocktails, + "avg_cocktail_rating": avg_cocktail_rating, + "recent_documents": recent_documents, + "recent_line_items": recent_line_item_rows, + }, + ) + finally: + db.close() diff --git a/app/routes/line_items.py b/app/routes/line_items.py index 3626dea..4096c83 100644 --- a/app/routes/line_items.py +++ b/app/routes/line_items.py @@ -36,18 +36,79 @@ def _to_decimal(value: str | None) -> Decimal | None: return None +def _line_item_extra(item: ReceiptLineItem) -> dict: + return dict(item.extra_json or {}) + + def _line_item_quality_rating(item: ReceiptLineItem) -> str: - extra = item.extra_json or {} - value = extra.get("quality_rating") + value = _line_item_extra(item).get("quality_rating") return "" if value is None else str(value) def _line_item_quality_note(item: ReceiptLineItem) -> str: - extra = item.extra_json or {} - value = extra.get("quality_note") + value = _line_item_extra(item).get("quality_note") return "" if value is None else str(value) +def _line_item_quality_status(item: ReceiptLineItem) -> str: + value = _line_item_extra(item).get("quality_status") + return "" if value is None else str(value) + + +def _is_quality_queue_candidate(item: ReceiptLineItem) -> bool: + if (item.item_category or "").lower() != "cocktail": + return False + + extra = _line_item_extra(item) + status = str(extra.get("quality_status") or "").strip().lower() + rating = str(extra.get("quality_rating") or "").strip() + + if status == "na": + return False + if rating: + return False + + return True + + +def _build_row(item: ReceiptLineItem) -> dict | None: + document = item.document + if document is None: + return None + + extracted = get_current_extracted_fields(document) + merchant_value = "" + transaction_date = "" + + if extracted is not None: + merchant_value = ( + extracted.merchant_normalized + or extracted.merchant_raw + or "" + ) + if extracted.transaction_date: + transaction_date = extracted.transaction_date.isoformat() + + if not transaction_date and document.created_at: + transaction_date = document.created_at.date().isoformat() + + return { + "line_item_id": item.id, + "document_id": document.document_id, + "transaction_date": transaction_date, + "merchant": merchant_value, + "description": item.normalized_description or item.raw_description or "", + "raw_description": item.raw_description or "", + "quantity": _decimal_to_str(item.quantity), + "line_total": _decimal_to_str(item.line_total), + "category": item.item_category or "", + "confidence": _decimal_to_str(item.confidence), + "quality_rating": _line_item_quality_rating(item), + "quality_note": _line_item_quality_note(item), + "quality_status": _line_item_quality_status(item), + } + + @router.post("/{line_item_id}/review", response_class=RedirectResponse) def save_line_item_review( line_item_id: int, @@ -58,32 +119,51 @@ def save_line_item_review( date_to: str = Form(""), rating_min: str = Form(""), rating_max: str = Form(""), + return_to: str = Form("list"), quality_rating: str = Form(""), quality_note: str = Form(""), + quality_status: str = Form(""), db: Session = Depends(get_db), ): item = db.query(ReceiptLineItem).filter(ReceiptLineItem.id == line_item_id).first() if item is None: return RedirectResponse(url="/line-items/", status_code=303) - extra = dict(item.extra_json or {}) + extra = _line_item_extra(item) rating_clean = quality_rating.strip() note_clean = quality_note.strip() + status_clean = quality_status.strip().lower() - if rating_clean: - extra["quality_rating"] = rating_clean - else: + if status_clean == "na": + extra["quality_status"] = "na" extra.pop("quality_rating", None) - - if note_clean: - extra["quality_note"] = note_clean + if note_clean: + extra["quality_note"] = note_clean + else: + extra.pop("quality_note", None) else: - extra.pop("quality_note", None) + if rating_clean: + extra["quality_rating"] = rating_clean + extra["quality_status"] = "rated" + else: + extra.pop("quality_rating", None) + if status_clean == "rated": + extra["quality_status"] = "rated" + else: + extra.pop("quality_status", None) + + if note_clean: + extra["quality_note"] = note_clean + else: + extra.pop("quality_note", None) item.extra_json = extra db.commit() + if return_to == "quality_queue": + return RedirectResponse(url="/queue/?tab=quality", status_code=303) + redirect_url = ( f"/line-items/?q={q}&merchant={merchant}&category={category}" f"&date_from={date_from}&date_to={date_to}" @@ -92,6 +172,50 @@ def save_line_item_review( return RedirectResponse(url=redirect_url, status_code=303) +@router.get("/queue", response_class=HTMLResponse) +def quality_queue( + request: Request, + db: Session = Depends(get_db), +): + items = ( + db.query(ReceiptLineItem) + .options( + selectinload(ReceiptLineItem.document).selectinload(Document.extracted_fields) + ) + .order_by(ReceiptLineItem.id.asc()) + .all() + ) + + rows = [] + for item in items: + if not _is_quality_queue_candidate(item): + continue + row = _build_row(item) + if row is not None: + rows.append(row) + + rows.sort( + key=lambda row: ( + row["transaction_date"] or "", + row["merchant"] or "", + row["description"] or "", + ) + ) + + next_row = rows[0] if rows else None + + return templates.TemplateResponse( + request=request, + name="line_items/queue.html", + context={ + "request": request, + "rows": rows, + "next_row": next_row, + "active_page": "line_items", + }, + ) + + @router.get("/", response_class=HTMLResponse) def list_line_items( request: Request, @@ -122,45 +246,21 @@ def list_line_items( rows: list[dict] = [] for item in items: - document = item.document - if document is None: + row = _build_row(item) + if row is None: continue - extracted = get_current_extracted_fields(document) - merchant_value = "" - transaction_date = "" + quality_rating_dec = _to_decimal(row["quality_rating"]) - if extracted is not None: - merchant_value = ( - extracted.merchant_normalized - or extracted.merchant_raw - or "" - ) - if extracted.transaction_date: - transaction_date = extracted.transaction_date.isoformat() - - if not transaction_date and document.created_at: - transaction_date = document.created_at.date().isoformat() - - description_value = ( - item.normalized_description - or item.raw_description - or "" - ) - category_value = item.item_category or "" - quality_rating_value = _line_item_quality_rating(item) - quality_note_value = _line_item_quality_note(item) - quality_rating_dec = _to_decimal(quality_rating_value) - - if q_norm and q_norm not in description_value.lower(): + if q_norm and q_norm not in row["description"].lower(): continue - if merchant_norm and merchant_norm not in merchant_value.lower(): + if merchant_norm and merchant_norm not in row["merchant"].lower(): continue - if category_norm and category_norm not in category_value.lower(): + if category_norm and category_norm not in row["category"].lower(): continue - if date_from and (not transaction_date or transaction_date < date_from): + if date_from and (not row["transaction_date"] or row["transaction_date"] < date_from): continue - if date_to and (not transaction_date or transaction_date > date_to): + if date_to and (not row["transaction_date"] or row["transaction_date"] > date_to): continue if rating_min_dec is not None: if quality_rating_dec is None or quality_rating_dec < rating_min_dec: @@ -169,22 +269,7 @@ def list_line_items( if quality_rating_dec is None or quality_rating_dec > rating_max_dec: continue - rows.append( - { - "line_item_id": item.id, - "document_id": document.document_id, - "transaction_date": transaction_date, - "merchant": merchant_value, - "description": description_value, - "raw_description": item.raw_description or "", - "quantity": _decimal_to_str(item.quantity), - "line_total": _decimal_to_str(item.line_total), - "category": category_value, - "confidence": _decimal_to_str(item.confidence), - "quality_rating": quality_rating_value, - "quality_note": quality_note_value, - } - ) + rows.append(row) rows.sort( key=lambda row: ( diff --git a/app/templates/dashboard.html b/app/templates/dashboard.html new file mode 100644 index 0000000..6101cb3 --- /dev/null +++ b/app/templates/dashboard.html @@ -0,0 +1,141 @@ + + + + + Dashboard + + + +
+ {% include "partials/sidebar.html" %} + +
+
+
+

Dashboard

+

Overview of document processing, extraction coverage, and line item review status.

+
+
+ +
+
+ Open documents + Open queue + Open line items + Open summary + Open ingest +
+
+ +
+

Document overview

+
+
Total documents{{ total_documents }}
+
Active documents{{ active_documents }}
+
Reviewed documents{{ reviewed_documents }}
+
Pending OCR/review{{ pending_review_documents }}
+
Extracted field rows{{ extracted_documents }}
+
Trashed documents{{ trashed_documents }}
+
+
+ +
+

Line item overview

+
+
Total line items{{ total_line_items }}
+
Cocktail items{{ cocktail_count }}
+
Rated cocktails{{ rated_cocktails }}
+
Pending cocktail ratings{{ pending_cocktail_reviews }}
+
Cocktails marked N/A{{ na_cocktails }}
+
Average cocktail rating{{ avg_cocktail_rating or "—" }}
+
+
+ +
+

Recent documents

+ {% if recent_documents %} +
+ + + + + + + + + + + {% for doc in recent_documents %} + + + + + + + {% endfor %} + +
DocumentTypeReview statusUpdated
{{ doc.document_id }}{{ doc.document_type }}{{ doc.review_status }}{{ doc.updated_at }}
+
+ {% else %} +

No documents found.

+ {% endif %} +
+ +
+

Recent line items

+ {% if recent_line_items %} +
+ + + + + + + + + + + + + + + {% for row in recent_line_items %} + + + + + + + + + + + {% endfor %} + +
DateMerchantItemCategoryTotalRatingStatusDocument
{{ row.transaction_date }}{{ row.merchant }}{{ row.description }}{{ row.category }}{{ row.line_total }}{{ row.quality_rating }}{{ row.quality_status }}{{ row.document_id }}
+
+ {% else %} +

No line items found.

+ {% endif %} +
+
+
+ + + +