From 8443a4fd8ae39be4bcc22d004e153fac6340a013 Mon Sep 17 00:00:00 2001 From: Sriram-Nambiar Date: Tue, 20 Oct 2026 14:38:08 +0530 Subject: [PATCH] Support multi-archive search and SearchResult with snippets --- .gitignore | 1 + CHANGELOG.md | 3 + README.md | 5 +- libzim/libwrapper.h | 10 ++- libzim/libzim.pyx | 131 ++++++++++++++++++++++++++--------- libzim/search.pyi | 18 ++++- libzim/zim.pxd | 3 + tests/test_libzim_creator.py | 84 +++++++++++++++++++++- 8 files changed, 216 insertions(+), 39 deletions(-) diff --git a/.gitignore b/.gitignore index e95ed3f5..d6abd155 100644 --- a/.gitignore +++ b/.gitignore @@ -241,6 +241,7 @@ libzim.dylib libzim.*.dylib zim*.dll libicu*.dll +icu*.dll zim.lib # pkg-config file diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f1ede6d..dafbfb56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Breaking**: `SuggestionSearch.getEstimatedMatches()` renamed `SuggestionSearch.get_estimated_matches()` (#200) - **Breaking**: `SuggestionSearch.getResults()` renamed `SuggestionSearch.get_results()` (#200) - Modernize type hints in `libzim.pyx` to use PEP 604/585 syntax (e.g. `str | bytes`, `dict[...]`) instead of `typing.Union`/`typing.Dict`/etc. (#281) +- **Breaking**: `SearchResultSet` now yields `SearchResult` objects instead of entry path strings (#230, #226) ### Removed @@ -36,6 +37,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add support for Python 3.15 (#281) - Build wheels against the stable ABI (abi3) so a single wheel per platform covers all supported CPython versions (#263) - Add `libzim.illustration` module with the `IllustrationInfo` value object, and width/height/scale-aware illustration APIs: `Creator.add_illustration()`/`add_illustration_info()`, `Archive.get_illustration_item()`/`get_illustration_item_info()`, `Archive.get_illustrations_info()` (#232, #233) +- Support multi-archive searching via `Searcher.add_archive()` (#230) +- Add `SearchResult` object with `archive`, `path`, `title`, and `snippet` (#230, #226) ## [3.13.1] - 2026-10-05 diff --git a/README.md b/README.md index 7e1970ad..f9f83b11 100644 --- a/README.md +++ b/README.md @@ -61,11 +61,12 @@ print(bytes(entry.get_item().content).decode("UTF-8")) # searching using full-text index search_string = "Welcome" query = Query().set_query(search_string) -searcher = Searcher(zim) +searcher = Searcher(zim) # or Searcher() followed by searcher.add_archive(...) search = searcher.search(query) search_count = search.get_estimated_matches() print(f"there are {search_count} matches for {search_string}") -print(list(search.get_results(0, search_count))) +for result in search.get_results(0, search_count): + print(result.archive, result.path, result.title, result.snippet) # accessing suggestions search_string = "kiwix" diff --git a/libzim/libwrapper.h b/libzim/libwrapper.h index df1e71bb..22692e77 100644 --- a/libzim/libwrapper.h +++ b/libzim/libwrapper.h @@ -206,10 +206,18 @@ class Search : public Wrapper class Searcher : public Wrapper { public: - Searcher() = default; + Searcher() : Wrapper(zim::Searcher(std::vector())) {}; Searcher(const wrapper::Archive& a) : Wrapper(zim::Searcher(*a)) {}; Searcher(const zim::Searcher& o) : Wrapper(o) {}; + void addArchive(const wrapper::Archive& a) + { + if (!mp_base) { + mp_base.reset(new zim::Searcher(std::vector())); + } + mp_base->addArchive(*a); + } + FORWARD(void, setVerbose) FORWARD(wrapper::Search, search) }; diff --git a/libzim/libzim.pyx b/libzim/libzim.pyx index fb458ba9..10c7d98c 100644 --- a/libzim/libzim.pyx +++ b/libzim/libzim.pyx @@ -1694,55 +1694,90 @@ cdef class Query: return self +cdef class SearchResult: + """Identifiable search result containing archive, path, title, and snippet.""" + __module__ = search_module_name + cdef readonly Archive archive + cdef readonly str path + cdef readonly str title + cdef readonly str snippet + + def __init__(self, Archive archive, str path, str title, str snippet): + self.archive = archive + self.path = path + self.title = title + self.snippet = snippet + + def __repr__(self) -> str: + return f"" + + def __str__(self) -> str: + return self.path + + def __eq__(self, other): + if isinstance(other, str): + return self.path == other + if isinstance(other, SearchResult): + return self.archive == (other).archive and self.path == (other).path + return False + + def __hash__(self): + return hash((self.archive.filename.expanduser().resolve(), self.path)) + + cdef class SearchResultSet: - """Iterator over a Search result: entry paths.""" + """Iterator over search results.""" __module__ = search_module_name cdef zim.SearchResultSet c_resultset + cdef list _archives @staticmethod - cdef from_resultset(zim.SearchResultSet _resultset): + cdef from_resultset(zim.SearchResultSet _resultset, list archives): cdef SearchResultSet resultset = SearchResultSet() resultset.c_resultset = move(_resultset) + resultset._archives = archives return resultset - def __iter__(self) -> Iterator[str]: - """Entry paths found in Archive for Search""" + def __iter__(self) -> Iterator[SearchResult]: + """Search results found in Archive(s) for Search""" cdef zim.SearchIterator current = self.c_resultset.begin() cdef zim.SearchIterator end = self.c_resultset.end() + cdef int archive_idx + cdef Archive archive + cdef str path, title, snippet while current != end: - yield current.getPath().decode('UTF-8') + archive_idx = current.getFileIndex() + archive = self._archives[archive_idx] + path = current.getPath().decode('UTF-8') + title = current.getTitle().decode('UTF-8') + snippet = current.getSnippet().decode('UTF-8') + yield SearchResult(archive, path, title, snippet) preincrement(current) + cdef class Search: - """Search request over a ZIM Archive.""" + """Search request over ZIM Archive(s).""" __module__ = search_module_name cdef zim.Search c_search + cdef list _archives - # Factory functions - Currently Cython can't use classmethods @staticmethod - cdef from_search(zim.Search _search): - """Creates a python ReadArticle from a C++ Article (zim::) -> ReadArticle - - Args: - _item (Item): A C++ Item - - Returns: - (Item): Casted item""" + cdef from_search(zim.Search _search, list archives): cdef Search search = Search() search.c_search = move(_search) + search._archives = archives return search def get_estimated_matches(self) -> pyint: - """Estimated number of results in Archive for the search. - + """Estimated number of results in Archive(s) for the search. Returns: - The number of estimeated results for this search. + The number of estimated results for this search. """ return self.c_search.getEstimatedMatches() def get_results(self, start: pyint, count: pyint) -> SearchResultSet: - """Iterator over Entry paths found in Archive for the search. + """Iterator over search results found in Archive(s) for the search. Args: start (int): The beginning of the range to get (offset of the first result). @@ -1751,14 +1786,18 @@ cdef class Search: Returns: A set of results for this search. """ - return SearchResultSet.from_resultset(move(self.c_search.getResults(start, count))) + return SearchResultSet.from_resultset( + move(self.c_search.getResults(start, count)), + self._archives + ) cdef class Searcher: - """ZIM Archive Searcher. + """ZIM Archive Searcher supporting single and multi-archive search. Args: - archive (Archive): ZIM Archive to search. + archive (Archive, optional): Initial ZIM Archive to search. + *extra_archives (Archive): Additional ZIM Archives to search across. Attributes: *c_searcher (zim.Searcher): a pointer to a C++ Searcher object. @@ -1766,28 +1805,53 @@ cdef class Searcher: __module__ = search_module_name cdef zim.Searcher c_searcher + cdef list _archives - def __cinit__(self, object archive: Archive): - """Constructs a Searcher from a ZIM Archive. + def __cinit__(self, object archive = None, *extra_archives): + """Constructs a Searcher, optionally bound to one or more ZIM Archives. Args: - archive (Archive): ZIM Archive to search.. - """ + archive (Archive or list of Archives, optional): Initial Archive(s). + *extra_archives (Archive): Additional Archives. + """ + self.c_searcher = zim.Searcher() + self._archives = [] + if archive is not None: + if isinstance(archive, (list, tuple)): + for a in archive: + self.add_archive(a) + else: + self.add_archive(archive) + for a in extra_archives: + self.add_archive(a) + + def add_archive(self, Archive archive): + """Bind an Archive to this Searcher. - self.c_searcher = move(zim.Searcher(archive.c_archive)) + Args: + archive (Archive): ZIM Archive to add. + """ + self.c_searcher.addArchive(archive.c_archive) + self._archives.append(archive) def search(self, object query: Query) -> Search: - """Create a Search object for a query of this Searcher's ZIM Archive. + """Create a Search object for a query across all bound ZIM Archives. Returns: - A Search object for querying this Searcher's ZIM Archive. + A Search object for querying this Searcher's ZIM Archive(s). """ - return Search.from_search(move(self.c_searcher.search(query.c_query))) + return Search.from_search( + move(self.c_searcher.search((query).c_query)), + self._archives + ) + search_module_doc = """libzim search module - Query to prepare a query from a string -- Searcher to perform a search over a libzim.reader.Archive +- Searcher to perform a search over one or more libzim.reader.Archive objects +- Search to execute the query +- SearchResult containing archive, path, title, and snippet Usage: @@ -1796,11 +1860,12 @@ archive = libzim.reader.Archive(fpath) searcher = Searcher(archive) query = Query().set_query("foo") search = searcher.search(query) -for path in search.get_results(10, 10) # get result from 10 to 20 (10 results) - print(path, archive.get_entry_by_path(path).title) +for result in search.get_results(10, 10): + print(result.archive, result.path, result.title, result.snippet) ```""" search_public_objects = [ Query, + SearchResult, SearchResultSet, Search, Searcher, diff --git a/libzim/search.pyi b/libzim/search.pyi index 6c53c75c..25e0a639 100644 --- a/libzim/search.pyi +++ b/libzim/search.pyi @@ -8,13 +8,27 @@ from libzim.reader import Archive class Query: def set_query(self, query: str) -> Self: ... +class SearchResult: + archive: Archive + path: str + title: str + snippet: str + def __init__( + self, archive: Archive, path: str, title: str, snippet: str + ) -> None: ... + class SearchResultSet: - def __iter__(self) -> Iterator[str]: ... + def __iter__(self) -> Iterator[SearchResult]: ... class Search: def get_estimated_matches(self) -> int: ... def get_results(self, start: int, count: int) -> SearchResultSet: ... class Searcher: - def __init__(self, archive: Archive) -> None: ... + def __init__( + self, + archive: Archive | list[Archive] | tuple[Archive, ...] | None = None, + *extra_archives: Archive, + ) -> None: ... + def add_archive(self, archive: Archive) -> None: ... def search(self, query: Query) -> Search: ... diff --git a/libzim/zim.pxd b/libzim/zim.pxd index 51ee4b17..238b9efc 100644 --- a/libzim/zim.pxd +++ b/libzim/zim.pxd @@ -106,6 +106,8 @@ cdef extern from "zim/search_iterator.h" namespace "zim": bint operator!=(SearchIterator) string getPath() string getTitle() + string getSnippet() + int getFileIndex() # Import the python wrappers (ObjWrapper) from libwrapper. @@ -206,6 +208,7 @@ cdef extern from "libwrapper.h" namespace "wrapper": cdef cppclass Searcher: Searcher() Searcher(const Archive& archive) except + + void addArchive(const Archive& archive) except + setVerbose(bool verbose) Search search(Query query) except + diff --git a/tests/test_libzim_creator.py b/tests/test_libzim_creator.py index a20e27fb..a398f872 100644 --- a/tests/test_libzim_creator.py +++ b/tests/test_libzim_creator.py @@ -14,7 +14,11 @@ IllustrationInfo, ) from libzim.reader import Archive # pyright: ignore [reportMissingModuleSource] -from libzim.search import Query, Searcher # pyright: ignore [reportMissingModuleSource] +from libzim.search import ( # pyright: ignore [reportMissingModuleSource] + Query, + Searcher, + SearchResult, +) from libzim.suggestion import ( # pyright: ignore [reportMissingModuleSource] SuggestionSearcher, ) @@ -1064,3 +1068,81 @@ def test_get_entry_by_title_utf8(fpath): assert zim.has_entry_by_title("Non Existent Title 🔍") is False with pytest.raises(KeyError): zim.get_entry_by_title("Non Existent Title 🔍") + + +def test_multi_archive_search_and_snippets(tmpdir): + p1 = pathlib.Path(tmpdir / "multi1.zim") + p2 = pathlib.Path(tmpdir / "multi2.zim") + + item1 = StaticItem( + path="python_doc", + title="Python Guide", + content="Python is an open source programming language with clean syntax.", + mimetype="text/html", + ) + item2 = StaticItem( + path="ruby_doc", + title="Ruby Guide", + content="Ruby is an open source programming language with elegant blocks.", + mimetype="text/html", + ) + + with Creator(p1).config_indexing(True, "eng") as c: + c.add_item(item1) + + with Creator(p2).config_indexing(True, "eng") as c: + c.add_item(item2) + + zim1 = Archive(p1) + zim2 = Archive(p2) + + searcher = Searcher() + searcher.add_archive(zim1) + searcher.add_archive(zim2) + + query = Query().set_query("programming") + search = searcher.search(query) + + assert search.get_estimated_matches() == 2 + results = list(search.get_results(0, 10)) + assert len(results) == 2 + + for r in results: + assert isinstance(r, SearchResult) + assert r.path in ("python_doc", "ruby_doc") + assert isinstance(r.snippet, str) + assert len(r.snippet) > 0 + + archives_found = [r.archive for r in results] + assert zim1 in archives_found + assert zim2 in archives_found + assert len(set(results)) == 2 + + paths_found = {r.path for r in results} + assert paths_found == {"python_doc", "ruby_doc"} + + r1, r2 = results[0], results[1] + assert repr(r1) == f"" + assert str(r1) == r1.path + + same_r1 = SearchResult(r1.archive, r1.path, r1.title, r1.snippet) + assert r1 == same_r1 + assert r1 == r1.path + assert r1 != "other_doc" + assert r1 != SearchResult(r1.archive, "other_doc", r1.title, r1.snippet) + assert r1 != SearchResult(r2.archive, r1.path, r1.title, r1.snippet) + assert (r1 == 123) is False + assert (r1 == None) is False # noqa: E711 + assert (r1 == zim1) is False + + assert hash(r1) == hash((r1.archive.filename.expanduser().resolve(), r1.path)) + assert len({r1, same_r1}) == 1 + + multi_searcher = Searcher([zim1, zim2]) + assert multi_searcher.search(query).get_estimated_matches() == 2 + + multi_searcher_extra = Searcher(zim1, zim2) + assert multi_searcher_extra.search(query).get_estimated_matches() == 2 + + multi_searcher_tuple = Searcher((zim1, zim2)) + assert multi_searcher_tuple.search(query).get_estimated_matches() == 2