Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 11 additions & 20 deletions .github/workflows/_test-integrations.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,17 @@ on:
permissions:
contents: read

env:
MINDEE_API_KEY: ${{ secrets.MINDEE_API_KEY_SE_TESTS }}
WORKFLOW_ID: ${{ secrets.WORKFLOW_ID_SE_TESTS }}
MINDEE_V2_API_KEY: ${{ secrets.MINDEE_V2_SE_TESTS_API_KEY }}
MINDEE_V2_SE_TESTS_FINDOC_MODEL_ID: ${{ secrets.MINDEE_V2_SE_TESTS_FINDOC_MODEL_ID }}
MINDEE_V2_SE_TESTS_BLANK_PDF_URL: ${{ secrets.MINDEE_V2_SE_TESTS_BLANK_PDF_URL }}
MINDEE_V2_SE_TESTS_CLASSIFICATION_MODEL_ID: ${{ secrets.MINDEE_V2_SE_TESTS_CLASSIFICATION_MODEL_ID }}
MINDEE_V2_SE_TESTS_CROP_MODEL_ID: ${{ secrets.MINDEE_V2_SE_TESTS_CROP_MODEL_ID }}
MINDEE_V2_SE_TESTS_SPLIT_MODEL_ID: ${{ secrets.MINDEE_V2_SE_TESTS_SPLIT_MODEL_ID }}
MINDEE_V2_SE_TESTS_OCR_MODEL_ID: ${{ secrets.MINDEE_V2_SE_TESTS_OCR_MODEL_ID }}

jobs:
pytest:
name: Run Integration Tests
Expand Down Expand Up @@ -46,16 +57,6 @@ jobs:
python -m pip install pip
pip install -e '.[test]'
- name: Run Integration Testing
env:
MINDEE_API_KEY: ${{ secrets.MINDEE_API_KEY_SE_TESTS }}
WORKFLOW_ID: ${{ secrets.WORKFLOW_ID_SE_TESTS }}
MINDEE_V2_API_KEY: ${{ secrets.MINDEE_V2_SE_TESTS_API_KEY }}
MINDEE_V2_SE_TESTS_FINDOC_MODEL_ID: ${{ secrets.MINDEE_V2_SE_TESTS_FINDOC_MODEL_ID }}
MINDEE_V2_SE_TESTS_BLANK_PDF_URL: ${{ secrets.MINDEE_V2_SE_TESTS_BLANK_PDF_URL }}
MINDEE_V2_SE_TESTS_CLASSIFICATION_MODEL_ID: ${{ secrets.MINDEE_V2_SE_TESTS_CLASSIFICATION_MODEL_ID }}
MINDEE_V2_SE_TESTS_CROP_MODEL_ID: ${{ secrets.MINDEE_V2_SE_TESTS_CROP_MODEL_ID }}
MINDEE_V2_SE_TESTS_SPLIT_MODEL_ID: ${{ secrets.MINDEE_V2_SE_TESTS_SPLIT_MODEL_ID }}
MINDEE_V2_SE_TESTS_OCR_MODEL_ID: ${{ secrets.MINDEE_V2_SE_TESTS_OCR_MODEL_ID }}
run: |
pytest --cov mindee -m integration

Expand Down Expand Up @@ -105,15 +106,5 @@ jobs:
pip install -e '.[test]'
shell: bash
- name: Run Integration Testing
env:
MINDEE_API_KEY: ${{ secrets.MINDEE_API_KEY_SE_TESTS }}
WORKFLOW_ID: ${{ secrets.WORKFLOW_ID_SE_TESTS }}
MINDEE_V2_API_KEY: ${{ secrets.MINDEE_V2_SE_TESTS_API_KEY }}
MINDEE_V2_SE_TESTS_FINDOC_MODEL_ID: ${{ secrets.MINDEE_V2_SE_TESTS_FINDOC_MODEL_ID }}
MINDEE_V2_SE_TESTS_BLANK_PDF_URL: ${{ secrets.MINDEE_V2_SE_TESTS_BLANK_PDF_URL }}
MINDEE_V2_SE_TESTS_CLASSIFICATION_MODEL_ID: ${{ secrets.MINDEE_V2_SE_TESTS_CLASSIFICATION_MODEL_ID }}
MINDEE_V2_SE_TESTS_CROP_MODEL_ID: ${{ secrets.MINDEE_V2_SE_TESTS_CROP_MODEL_ID }}
MINDEE_V2_SE_TESTS_SPLIT_MODEL_ID: ${{ secrets.MINDEE_V2_SE_TESTS_SPLIT_MODEL_ID }}
MINDEE_V2_SE_TESTS_OCR_MODEL_ID: ${{ secrets.MINDEE_V2_SE_TESTS_OCR_MODEL_ID }}
run: |
pytest -m "integration and not pypdfium2 and not pillow"
1 change: 0 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
* :sparkles: add simple fields typed accessors
* :sparkles: allow passing a webhook to the CLI
* :recycle: update internals and bump dependencies

### Fixes
* :bug: :boom: rework extraction methods
* :bug: :boom: harmonize Crop and Split
Expand Down
4 changes: 2 additions & 2 deletions mindee/v2/error/mindee_http_error_v2.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
import json

from mindee.parsing.common.string_dict import StringDict
from mindee.v2.parsing import ErrorItem, ErrorResponse
from mindee.v2.parsing.error import ErrorItem, ErrorResponse, IErrorResponse


class MindeeHTTPErrorV2(RuntimeError, ErrorResponse):
class MindeeHTTPErrorV2(RuntimeError, IErrorResponse):
Comment thread
ianardee marked this conversation as resolved.
"""An exception relating to HTTP calls."""

def __init__(self, response: ErrorResponse) -> None:
Expand Down
2 changes: 1 addition & 1 deletion mindee/v2/mindee_http/mindee_api_v2.py
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,7 @@ def req_get_search_models(
self, name: str | None, model_type: str | None
) -> SearchResponse:
"""
Deprecated. Use `search` instead.
Deprecated. Use `req_search` instead.
"""
get_caller: Callable
if self.http_client is None or self.http_client.is_closed:
Expand Down
3 changes: 2 additions & 1 deletion mindee/v2/parsing/error/__init__.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
from mindee.v2.parsing.error.error_item import ErrorItem
from mindee.v2.parsing.error.error_response import ErrorResponse
from mindee.v2.parsing.error.ierror_response import IErrorResponse

__all__ = ["ErrorItem", "ErrorResponse"]
__all__ = ["ErrorItem", "ErrorResponse", "IErrorResponse"]
44 changes: 30 additions & 14 deletions mindee/v2/parsing/error/error_response.py
Original file line number Diff line number Diff line change
@@ -1,22 +1,14 @@
from mindee.parsing.common import CommonResponse
from mindee.parsing.common.string_dict import StringDict
from mindee.v2.parsing.error.error_item import ErrorItem
from mindee.v2.parsing.error.ierror_response import IErrorResponse


class ErrorResponse:
class ErrorResponse(CommonResponse, IErrorResponse):
Comment thread
ianardee marked this conversation as resolved.
"""Error response detailing a problem. The format adheres to RFC 9457."""

status: int
"""The HTTP status code returned by the server."""
detail: str
"""A human-readable explanation specific to the occurrence of the problem."""
title: str
"""A short, human-readable summary of the problem."""
code: str
"""A machine-readable code specific to the occurrence of the problem."""
errors: list[ErrorItem]
"""A list of explicit error details."""

def __init__(self, raw_response: StringDict):
super().__init__(raw_response)
self.status = raw_response["status"]
self.detail = raw_response["detail"]
self.title = raw_response["title"]
Expand All @@ -26,5 +18,29 @@ def __init__(self, raw_response: StringDict):
except KeyError:
self.errors = []

def __str__(self):
return f"HTTP {self.status} - {self.title} :: {self.code} - {self.detail}"
def __str__(self) -> str:
"""To make the error prettier to display."""

result = [
"Error Details",
"=============",
f":HTTP Status: {self.status}",
f":Title: {self.title}",
f":Code: {self.code}",
f":Detail: {self.detail}",
]

if self.errors:
result.append("")
result.append("Error Items")
result.append("-----------")

for i, error in enumerate(self.errors):
result.append(f"**Error {i + 1}:**")
result.append(f" :Pointer: {getattr(error, 'pointer', '')}")
result.append(f" :Detail: {getattr(error, 'detail', '')}")

if i < len(self.errors) - 1:
result.append("")

return "\n".join(result) + "\n"
22 changes: 22 additions & 0 deletions mindee/v2/parsing/error/ierror_response.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
from typing import Protocol

from mindee.v2.parsing.error.error_item import ErrorItem


class IErrorResponse(Protocol):
"""Error response detailing a problem. The format adheres to RFC 9457."""

status: int
"""The HTTP status code returned by the server."""

detail: str
"""A human-readable explanation specific to the occurrence of the problem."""

title: str
"""A short, human-readable summary of the problem."""

code: str
"""A machine-readable code specific to the occurrence of the problem."""

errors: list[ErrorItem]
"""A list of explicit details on the problem."""
38 changes: 20 additions & 18 deletions tests/input/test_apply_page_options.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
PathInput,
)
from mindee.input.page_options import KEEP_ONLY, REMOVE, PageOptions
from tests.utils import FILE_TYPES_DIR, V1_PRODUCT_DATA_DIR
from tests.utils import FILE_TYPES_PATH, V1_PRODUCT_PATH

pdfium = pytest.importorskip("pypdfium2")

Expand All @@ -22,7 +22,9 @@ def _assert_page_options(input_source: LocalInputSource, numb_pages: int):
# I.e., each page is read and rendered as a rasterized image.
# These images are then compared as raw byte sequences.
cut_pdf = pdfium.PdfDocument(input_source.file_object)
pdf = pdfium.PdfDocument(FILE_TYPES_DIR / "pdf" / f"multipage_cut-{numb_pages}.pdf")
pdf = pdfium.PdfDocument(
FILE_TYPES_PATH / "pdf" / f"multipage_cut-{numb_pages}.pdf"
)
for idx in range(len(pdf)):
pdf_page = pdf.get_page(idx)
pdf_page_render = pdfium.PdfPage.render(pdf_page)
Expand All @@ -35,14 +37,14 @@ def _assert_page_options(input_source: LocalInputSource, numb_pages: int):


def test_pdf_reconstruct_ok():
input_source = PathInput(FILE_TYPES_DIR / "pdf" / "multipage.pdf")
input_source = PathInput(FILE_TYPES_PATH / "pdf" / "multipage.pdf")
input_source.process_pdf(behavior=KEEP_ONLY, on_min_pages=2, page_indexes=range(5))
assert isinstance(input_source.file_object, io.BytesIO)


@pytest.mark.parametrize("numb_pages", [1, 2, 3])
def test_process_pdf_cut_n_pages(numb_pages: int):
input_source = PathInput(FILE_TYPES_DIR / "pdf" / "multipage.pdf")
input_source = PathInput(FILE_TYPES_PATH / "pdf" / "multipage.pdf")
assert input_source.page_count == 12
input_source.process_pdf(
behavior=KEEP_ONLY, on_min_pages=2, page_indexes=[0, -2, -1][:numb_pages]
Expand All @@ -53,7 +55,7 @@ def test_process_pdf_cut_n_pages(numb_pages: int):

@pytest.mark.parametrize("numb_pages", [1, 2, 3])
def test_apply_pages_pdf_cut_n_pages(numb_pages: int):
input_source = PathInput(FILE_TYPES_DIR / "pdf" / "multipage.pdf")
input_source = PathInput(FILE_TYPES_PATH / "pdf" / "multipage.pdf")
assert input_source.page_count == 12
input_source.apply_page_options(
PageOptions(on_min_pages=2, page_indexes=[0, -2, -1][:numb_pages])
Expand All @@ -63,7 +65,7 @@ def test_apply_pages_pdf_cut_n_pages(numb_pages: int):


def test_pdf_keep_5_first_pages():
input_source = PathInput(FILE_TYPES_DIR / "pdf" / "multipage.pdf")
input_source = PathInput(FILE_TYPES_PATH / "pdf" / "multipage.pdf")
assert input_source.page_count == 12
input_source.process_pdf(
behavior=KEEP_ONLY, on_min_pages=2, page_indexes=[0, 1, 2, 3, 4]
Expand All @@ -72,7 +74,7 @@ def test_pdf_keep_5_first_pages():


def test_pdf_keep_invalid_pages():
input_source = PathInput(FILE_TYPES_DIR / "pdf" / "multipage.pdf")
input_source = PathInput(FILE_TYPES_PATH / "pdf" / "multipage.pdf")
assert input_source.page_count == 12
input_source.process_pdf(
behavior=KEEP_ONLY, on_min_pages=2, page_indexes=[0, 1, 17]
Expand All @@ -81,7 +83,7 @@ def test_pdf_keep_invalid_pages():


def test_pdf_remove_5_last_pages():
input_source = PathInput(FILE_TYPES_DIR / "pdf" / "multipage.pdf")
input_source = PathInput(FILE_TYPES_PATH / "pdf" / "multipage.pdf")
assert input_source.is_pdf() is True
input_source.process_pdf(
behavior=REMOVE, on_min_pages=2, page_indexes=[-5, -4, -3, -2, -1]
Expand All @@ -90,7 +92,7 @@ def test_pdf_remove_5_last_pages():


def test_pdf_remove_5_first_pages():
input_source = PathInput(FILE_TYPES_DIR / "pdf" / "multipage.pdf")
input_source = PathInput(FILE_TYPES_PATH / "pdf" / "multipage.pdf")
assert input_source.is_pdf() is True
input_source.process_pdf(
behavior=REMOVE, on_min_pages=2, page_indexes=list(range(5))
Expand All @@ -99,14 +101,14 @@ def test_pdf_remove_5_first_pages():


def test_pdf_remove_invalid_pages():
input_source = PathInput(FILE_TYPES_DIR / "pdf" / "multipage.pdf")
input_source = PathInput(FILE_TYPES_PATH / "pdf" / "multipage.pdf")
assert input_source.is_pdf() is True
input_source.process_pdf(behavior=REMOVE, on_min_pages=2, page_indexes=[16])
assert input_source.page_count == 12


def test_pdf_keep_no_pages():
input_source = PathInput(FILE_TYPES_DIR / "pdf" / "multipage.pdf")
input_source = PathInput(FILE_TYPES_PATH / "pdf" / "multipage.pdf")
assert input_source.is_pdf() is True
# empty page indexes
with pytest.raises(RuntimeError):
Expand All @@ -119,7 +121,7 @@ def test_pdf_keep_no_pages():


def test_pdf_remove_all_pages():
input_source = PathInput(FILE_TYPES_DIR / "pdf" / "multipage.pdf")
input_source = PathInput(FILE_TYPES_PATH / "pdf" / "multipage.pdf")
assert input_source.is_pdf() is True
with pytest.raises(RuntimeError):
input_source.process_pdf(
Expand All @@ -128,37 +130,37 @@ def test_pdf_remove_all_pages():


def test_pdf_input_from_file():
with open(FILE_TYPES_DIR / "pdf" / "multipage.pdf", "rb") as fp:
with open(FILE_TYPES_PATH / "pdf" / "multipage.pdf", "rb") as fp:
input_source = FileInput(fp)
assert input_source.is_pdf() is True
input_source.process_pdf(behavior=KEEP_ONLY, on_min_pages=2, page_indexes=[0])
assert input_source.page_count == 1


def test_pdf_input_from_base64():
with open(V1_PRODUCT_DATA_DIR / "invoices" / "invoice_10p.txt") as fp:
with open(V1_PRODUCT_PATH / "invoices" / "invoice_10p.txt") as fp:
input_source = Base64Input(fp.read(), filename="invoice_10p.pdf")
assert input_source.is_pdf() is True
input_source.process_pdf(behavior=KEEP_ONLY, on_min_pages=2, page_indexes=[0])
assert input_source.page_count == 1


def test_pdf_input_from_bytes():
with open(V1_PRODUCT_DATA_DIR / "invoices" / "invoice_10p.pdf", "rb") as fp:
with open(V1_PRODUCT_PATH / "invoices" / "invoice_10p.pdf", "rb") as fp:
input_source = BytesInput(fp.read(), filename="invoice_10p.pdf")
assert input_source.is_pdf() is True
input_source.process_pdf(behavior=KEEP_ONLY, on_min_pages=2, page_indexes=[0])
assert input_source.page_count == 1


def test_pdf_blank_check():
input_source = PathInput(FILE_TYPES_DIR / "pdf" / "blank.pdf")
input_source = PathInput(FILE_TYPES_PATH / "pdf" / "blank.pdf")
with pytest.raises(MindeeError):
input_source.process_pdf(behavior=KEEP_ONLY, on_min_pages=2, page_indexes=[0])

input_source = PathInput(FILE_TYPES_DIR / "pdf" / "blank_1.pdf")
input_source = PathInput(FILE_TYPES_PATH / "pdf" / "blank_1.pdf")
with pytest.raises(MindeeError):
input_source.process_pdf(behavior=KEEP_ONLY, on_min_pages=2, page_indexes=[0])

input_not_blank = PathInput(FILE_TYPES_DIR / "pdf" / "not_blank_image_only.pdf")
input_not_blank = PathInput(FILE_TYPES_PATH / "pdf" / "not_blank_image_only.pdf")
assert input_not_blank.page_count == 1
Loading
Loading