diff --git a/CHANGELOG.md b/CHANGELOG.md index 73917a0..0787983 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Send a Wikimedia-compliant default `User-Agent` in `stream_file` when downloading from `upload.wikimedia.org` (#214) - Add fuzzy rule to rewrite URLs of lesbases.anct.gouv.fr (#334) - Report progress from `save_large_file` through a new `on_progress` callback and periodic logging, and stream the download to a `.part` file that is only moved into place once complete (#348) +- New `get_pdf_title` and `get_epub_title` helpers to get a document's title from its own metadata without extracting its full text content, cheaper than `get_pdf_index_data`/`get_epub_index_data` when only the title is needed (openzim/warc2zim#462) ### Changed diff --git a/src/zimscraperlib/zim/indexing.py b/src/zimscraperlib/zim/indexing.py index 3413e15..e1b0c01 100644 --- a/src/zimscraperlib/zim/indexing.py +++ b/src/zimscraperlib/zim/indexing.py @@ -104,14 +104,51 @@ def get_epub_index_data( ) -def _get_document_index_data( +def get_pdf_title( *, + content: str | bytes | None = None, + fileobj: io.BytesIO | None = None, + filepath: pathlib.Path | None = None, +) -> str: + """Returns the title of a given PDF, built from its own metadata + + Cheaper than get_pdf_index_data when only the title is needed, since it does + not extract the document's full text content. + + PDF can be passed either as content or fileobject or filepath + """ + return _get_document_title( + filetype="pdf", content=content, fileobj=fileobj, filepath=filepath + ) + + +def get_epub_title( + *, + content: str | bytes | None = None, + fileobj: io.BytesIO | None = None, + filepath: pathlib.Path | None = None, +) -> str: + """Returns the title of a given EPUB, built from its own metadata + + Cheaper than get_epub_index_data when only the title is needed, since it does + not extract the document's full text content. + + EPUB can be passed either as content or fileobject or filepath + """ + return _get_document_title( + filetype="epub", content=content, fileobj=fileobj, filepath=filepath + ) + + +def _open_document( filetype: str, + *, content: str | bytes | None = None, fileobj: io.BytesIO | None = None, filepath: pathlib.Path | None = None, -) -> IndexData: - """IndexData of any document PyMuPDF can read, title built from its metadata""" +) -> pymupdf.Document: + """Open a document with PyMuPDF, passed either as content, fileobject or + filepath""" # do not display all pymupdf errors, we will filter them afterwards pymupdf.TOOLS.mupdf_display_errors( # pyright: ignore[reportUnknownMemberType] @@ -119,11 +156,15 @@ def _get_document_index_data( ) if content: - doc = pymupdf.open(stream=content, filetype=filetype) + return pymupdf.open(stream=content, filetype=filetype) elif fileobj: - doc = pymupdf.open(stream=fileobj, filetype=filetype) + return pymupdf.open(stream=fileobj, filetype=filetype) else: - doc = pymupdf.open(filename=filepath, filetype=filetype) + return pymupdf.open(filename=filepath, filetype=filetype) + + +def _get_metadata_title(doc: pymupdf.Document) -> str: + """Build a title from a document's own metadata (title/author/subject)""" metadata = ( # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType] doc.metadata ) @@ -137,19 +178,12 @@ def _get_document_index_data( ) if parts: # pragma: no branch (always metadata in test documents) title = " - ".join(parts) + return title - def get_page_content(page: pymupdf.Page) -> str: - text = ( # pyright: ignore[reportUnknownVariableType] - page.get_text() # pyright: ignore[reportUnknownMemberType] - ) - if not isinstance(text, str): # pragma: no cover - raise Exception("Unexpected text content") - return text - content = "\n".join(get_page_content(page) for page in doc) - - # build list of messages and filter messages which are known to not be relevant - # in our use-case +def _log_mupdf_warnings(): + """Log PyMuPDF warnings accumulated since errors were last silenced/fetched, + filtering out messages which are known to not be relevant in our use-case""" mupdf_messages = "\n".join( warning for warning in pymupdf.TOOLS.mupdf_warnings().splitlines() @@ -162,6 +196,45 @@ def get_page_content(page: pymupdf.Page) -> str: f"PyMuPDF issues:\n{mupdf_messages}" ) # pragma: no cover (no known error in test documents) + +def _get_document_title( + *, + filetype: str, + content: str | bytes | None = None, + fileobj: io.BytesIO | None = None, + filepath: pathlib.Path | None = None, +) -> str: + """Title of any document PyMuPDF can read, built from its own metadata""" + doc = _open_document(filetype, content=content, fileobj=fileobj, filepath=filepath) + title = _get_metadata_title(doc) + _log_mupdf_warnings() + return title + + +def _get_document_index_data( + *, + filetype: str, + content: str | bytes | None = None, + fileobj: io.BytesIO | None = None, + filepath: pathlib.Path | None = None, +) -> IndexData: + """IndexData of any document PyMuPDF can read, title built from its metadata""" + + doc = _open_document(filetype, content=content, fileobj=fileobj, filepath=filepath) + title = _get_metadata_title(doc) + + def get_page_content(page: pymupdf.Page) -> str: + text = ( # pyright: ignore[reportUnknownVariableType] + page.get_text() # pyright: ignore[reportUnknownMemberType] + ) + if not isinstance(text, str): # pragma: no cover + raise Exception("Unexpected text content") + return text + + content = "\n".join(get_page_content(page) for page in doc) + + _log_mupdf_warnings() + return IndexData( title=title, content=content, diff --git a/tests/zim/test_indexing.py b/tests/zim/test_indexing.py index 3b1bed2..0279ee9 100644 --- a/tests/zim/test_indexing.py +++ b/tests/zim/test_indexing.py @@ -9,7 +9,9 @@ from zimscraperlib.zim.indexing import ( IndexData, get_epub_index_data, + get_epub_title, get_pdf_index_data, + get_pdf_title, ) from zimscraperlib.zim.items import StaticItem @@ -322,6 +324,30 @@ def test_get_pdf_index_data( assert index_data.get_keywords() == "" +@pytest.mark.parametrize( + "pdf_no, expected_title", + [ + (1, "Microsoft Word - Placeholder Documentation.docx - c8clark"), + ( + 2, + "Military Dermatology (Redacted) - Office of the Surgeon General - 1994", + ), + ], +) +def test_get_pdf_title( + *, + pdf_no: int, + expected_title: str, + encrypted_pdf_file: pathlib.Path, + big_pdf_file: pathlib.Path, +): + """get_pdf_title returns the same title as get_pdf_index_data, cheaper""" + assert ( + get_pdf_title(filepath=encrypted_pdf_file if pdf_no == 1 else big_pdf_file) + == expected_title + ) + + def test_indexing_item_pdf_custom_title( tmp_path: pathlib.Path, encrypted_pdf_file: pathlib.Path ): @@ -394,6 +420,21 @@ def test_get_epub_index_data( warning.assert_not_called() +@pytest.mark.parametrize("source", ["content", "fileobj", "filepath"]) +def test_get_epub_title(source: str, epub_file: pathlib.Path, mocker: MockerFixture): + """get_epub_title returns the same title as get_epub_index_data, cheaper""" + warning = mocker.patch.object(indexing.logger, "warning") + if source == "content": + title = get_epub_title(content=epub_file.read_bytes()) + elif source == "fileobj": + title = get_epub_title(fileobj=io.BytesIO(epub_file.read_bytes())) + else: + title = get_epub_title(filepath=epub_file) + assert title == "Lighthouse Almanac - Ada Quillfeather" + # MuPDF does not know EPUB 3 but reads it fine, this must not be reported + warning.assert_not_called() + + @pytest.mark.parametrize("source", ["content", "fileobj", "filepath"]) def test_indexing_item_epub( tmp_path: pathlib.Path, epub_file: pathlib.Path, source: str