diff --git a/README.md b/README.md index f91ce2e..97ec7be 100644 --- a/README.md +++ b/README.md @@ -44,11 +44,12 @@ Below is a list of features that we would like to implement or have been request | Allow DVE to run on Python 3.12+ | 0.8.0 | Yes | | Upgrade to Pydantic 2.0 | 0.9.0 | Yes | | Upgrade DuckDB to v1.4 | 0.10.0 | Yes | +| Polars upgrade to v1+ | 0.10.0 | No | | Uplift Pyspark to 4.0+ | TBA | No | -| Polars upgrade to v1+ | TBA | No | -| DuckDB upgrade to v1.5+ | TBA | No | -| Python 3.13 & 3.14 upgrade | TBA | No | -| Create a more user friendly interface for building and modifying dischema files | TBA | No | +| DuckDB upgrade to v1.5+ | TBA | No | +| Python 3.13 & 3.14 upgrade | TBA | No | +| Create a more user friendly interface for building and modifying dischema files | TBA | No | +| Polars upgrade to v2+ | TBA | No | If you are interested in getting any of the unreleased features listed above available, then please read the [Contributing](#Contributing) section and then submit us a pull request. diff --git a/poetry.lock b/poetry.lock index 18f789e..aed52e7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1227,6 +1227,37 @@ typing-extensions = {version = ">=4.6.0", markers = "python_version < \"3.13\""} [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "fastexcel" +version = "0.21.0" +description = "A fast excel file reader for Python, written in Rust" +optional = false +python-versions = ">=3.10" +groups = ["dev", "test"] +files = [ + {file = "fastexcel-0.21.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c3e7ab5d8c8b6c5a787aaf2b64604bd8b93b94694920a2ed731ea556a81d9a35"}, + {file = "fastexcel-0.21.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:768b663728cb5f29e159428fdf3a3f74e379534c2f0304b300bd95039d482abe"}, + {file = "fastexcel-0.21.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c6e66906fe3b9f68f94c4c94e2ac21b6eebd862b703983c8e0c009f91c71754"}, + {file = "fastexcel-0.21.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ddb458fecbbf1804c0952155fb99d18025d86e345b57a5435e0553944f25578"}, + {file = "fastexcel-0.21.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0376944edf90c98008b49b200f7354122ba9abac6c21bab76487655738b041b7"}, + {file = "fastexcel-0.21.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e919a4eaa15330341744cfee33d1f87d041d08228ce68809790e3738e80811e8"}, + {file = "fastexcel-0.21.0-cp310-abi3-win_amd64.whl", hash = "sha256:e1db4666a0790b48c76bb5a43cda06ffecebb22706f9ac6b3f07bcb0e7336134"}, + {file = "fastexcel-0.21.0-cp310-abi3-win_arm64.whl", hash = "sha256:86af0a1e3c3d8657916ea434f11636df4e4b49e0cf665b4ea39349a83d4ca3c8"}, + {file = "fastexcel-0.21.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:f6cf28f5f3fed1f34aa15bf021d2c04bf947720df70f54b131258c913bc3b4cf"}, + {file = "fastexcel-0.21.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ef2a6953e8350966d32632e3bc064edaab64ea2899f2027e564269fa7d75fb58"}, + {file = "fastexcel-0.21.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f8fdbfd80647714a2b3d49de2517d0466f6c046aa215c16fb569c48aef8d0ee"}, + {file = "fastexcel-0.21.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47c6f42b3b82a158e4e6c4e1ed53ba0b96cec132d1fed828c8411e6f6ba5caab"}, + {file = "fastexcel-0.21.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bce27f751cf1661f823088e89c11375448d19e425e3c3aa993c356720305c873"}, + {file = "fastexcel-0.21.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:1a5742e598516734740ef4142cf3328d6ef6c8e43947d9a66d6a91a5d9bfa3ec"}, + {file = "fastexcel-0.21.0-cp314-cp314t-win_amd64.whl", hash = "sha256:fe52f6053aac6ff3b8cc879052b671af9cb3ada16853b1c8b4bcac44574e4c10"}, + {file = "fastexcel-0.21.0.tar.gz", hash = "sha256:07313c1267ab47ba639abf1122efd5985a1fb08efc996194f422ab17f06149c5"}, +] + +[package.extras] +pandas = ["pandas (>=1.4.4)", "pyarrow (>=8.0.0)"] +polars = ["polars (>=1)"] +pyarrow = ["pyarrow (>=8.0.0)"] + [[package]] name = "filelock" version = "3.29.5" @@ -2370,44 +2401,67 @@ testing = ["coverage", "pytest", "pytest-benchmark"] [[package]] name = "polars" -version = "0.20.31" +version = "1.44.2" description = "Blazingly fast DataFrame library" optional = false -python-versions = ">=3.8" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "polars-0.20.31-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:86454ade5ed302bbf87f145cfcb1b14f7a5765a9440e448659e1f3dba6ac4e79"}, - {file = "polars-0.20.31-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:67f2fe842262b7e1b9371edad21b760f6734d28b74c78dda88dff1bf031b9499"}, - {file = "polars-0.20.31-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24b82441f93409e0e8abd6f427b029db102f02b8de328cee9a680f84b84e3736"}, - {file = "polars-0.20.31-cp38-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:87f43bce4d41abf8c8c5658d881e4b8378e5c61010a696bfea8b4106b908e916"}, - {file = "polars-0.20.31-cp38-abi3-win_amd64.whl", hash = "sha256:2d7567c9fd9d3b9aa93387ca9880d9e8f7acea3c0a0555c03d8c0c2f0715d43c"}, - {file = "polars-0.20.31.tar.gz", hash = "sha256:00f62dec6bf43a4e2a5db58b99bf0e79699fe761c80ae665868eaea5168f3bbb"}, + {file = "polars-1.44.2-py3-none-any.whl", hash = "sha256:1bb331f17a40d9d931101533dcd33637b66edc61eb377b07020dac16a0f0377b"}, + {file = "polars-1.44.2.tar.gz", hash = "sha256:86c8e26b6c2de8c8d344bb910b74dfc47b118ac3fe0f19b44909467990a0b281"}, ] +[package.dependencies] +polars-runtime-32 = "1.44.2" + [package.extras] -adbc = ["adbc-driver-manager", "adbc-driver-sqlite"] -all = ["polars[adbc,async,cloudpickle,connectorx,deltalake,fastexcel,fsspec,gevent,iceberg,numpy,pandas,plot,pyarrow,pydantic,sqlalchemy,timezone,xlsx2csv,xlsxwriter]"] -async = ["nest-asyncio"] +adbc = ["adbc-driver-manager[dbapi]", "adbc-driver-sqlite[dbapi]"] +all = ["polars[async,cloudpickle,database,deltalake,excel,fsspec,graph,iceberg,numpy,pandas,plot,pyarrow,pydantic,style,timezone]"] +async = ["gevent"] +calamine = ["fastexcel (>=0.9)"] cloudpickle = ["cloudpickle"] connectorx = ["connectorx (>=0.3.2)"] -deltalake = ["deltalake (>=0.15.0)"] -fastexcel = ["fastexcel (>=0.9)"] +database = ["polars[adbc,connectorx,sqlalchemy]"] +deltalake = ["deltalake (>=1.0.0,!=1.5.*)"] +excel = ["polars[calamine,openpyxl,xlsx2csv,xlsxwriter]"] fsspec = ["fsspec"] -gevent = ["gevent"] -iceberg = ["pyiceberg (>=0.5.0)"] -matplotlib = ["matplotlib"] +gpu = ["cudf-polars-cu12"] +graph = ["matplotlib"] +iceberg = ["pyiceberg (>=0.9.0)"] numpy = ["numpy (>=1.16.0)"] openpyxl = ["openpyxl (>=3.0.0)"] -pandas = ["pandas", "pyarrow (>=7.0.0)"] -plot = ["hvplot (>=0.9.1)"] +pandas = ["pandas", "polars[pyarrow]"] +plot = ["altair (>=5.4.0)"] +polars-cloud = ["polars_cloud (>=0.9.0)"] pyarrow = ["pyarrow (>=7.0.0)"] pydantic = ["pydantic"] -pyxlsb = ["pyxlsb (>=1.0)"] -sqlalchemy = ["pandas", "sqlalchemy"] -timezone = ["backports-zoneinfo ; python_version < \"3.9\"", "tzdata ; platform_system == \"Windows\""] +rt64 = ["polars-runtime-64 (==1.44.2)"] +rtcompat = ["polars-runtime-compat (==1.44.2)"] +sqlalchemy = ["polars[pandas]", "sqlalchemy"] +style = ["great-tables (>=0.8.0)"] +timezone = ["tzdata ; platform_system == \"Windows\""] xlsx2csv = ["xlsx2csv (>=0.8.0)"] xlsxwriter = ["xlsxwriter"] +[[package]] +name = "polars-runtime-32" +version = "1.44.2" +description = "Blazingly fast DataFrame library" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "polars_runtime_32-1.44.2-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:1fd536720668ba203a16a20b08cd6b23057e407a0279cf36b2f35f879d6e3208"}, + {file = "polars_runtime_32-1.44.2-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:e0fd43720c8222ae39919c8ff891636d53b352706087120e62f83544dd3ff782"}, + {file = "polars_runtime_32-1.44.2-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bbf9b45040291dc1c6c588c837019c33557bde25ec536562a9cca9e1f6dfcc45"}, + {file = "polars_runtime_32-1.44.2-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1bafb441e99199a62c63bf1bbdc0ea09ee9776dbac2bf31452b5000fb1df2f7"}, + {file = "polars_runtime_32-1.44.2-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:10c0c695a418407617b5159db7d9a21074a733e4c6d61275b6762f25cb31ca99"}, + {file = "polars_runtime_32-1.44.2-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:c4a09fb14aad711526346efc0cb2015c2fd0555ce4118b6524e5debbaea65ff5"}, + {file = "polars_runtime_32-1.44.2-cp310-abi3-win_amd64.whl", hash = "sha256:8598e7a20efba70bb74978c7df7af7c606ff4d79b9b48fdd808250b189bc9a13"}, + {file = "polars_runtime_32-1.44.2-cp310-abi3-win_arm64.whl", hash = "sha256:d51040d3ab40157f6db3c62be59cab5b80fb3c8d158924769c4982a1c8eef730"}, + {file = "polars_runtime_32-1.44.2.tar.gz", hash = "sha256:b84842f7d621aaca7a52e165e19a24f89db45f8aa13744941430218419a14a67"}, +] + [[package]] name = "pre-commit" version = "4.3.0" @@ -3558,18 +3612,6 @@ files = [ [package.extras] dev = ["pytest", "setuptools"] -[[package]] -name = "xlsx2csv" -version = "0.8.4" -description = "xlsx to csv converter" -optional = false -python-versions = "*" -groups = ["dev", "test"] -files = [ - {file = "xlsx2csv-0.8.4-py3-none-any.whl", hash = "sha256:52ab873fc7b2f2ca75d14aee8bd1985a9f5c1bcb3cc7b80df7a5d57a40a67473"}, - {file = "xlsx2csv-0.8.4.tar.gz", hash = "sha256:2aa809888826f6af5b26c77fc7f613f2bbeada0d8cc09e5a58e0f59684bb6911"}, -] - [[package]] name = "xmltodict" version = "1.0.4" @@ -3638,4 +3680,4 @@ type = ["pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "6d996888c9d149407fb885b454745bf763f9cba56404a29148ffe0b9fd57b0de" +content-hash = "2088f09f963ea03be49857428d8132386dd085244e4acfe1eb9a4870976a8463" diff --git a/pyproject.toml b/pyproject.toml index 5cac028..958020d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ lxml = "6.1.1" numpy = "1.26.4" openpyxl = "3.1.5" pandas = "2.3.3" -polars = "0.20.31" +polars = "1.44.2" pyarrow = "23.0.1" pydantic = "2.13.4" pyspark = ">=3.5.0,<=3.5.5" @@ -75,7 +75,7 @@ requests = "2.33.0" # dependency of `moto` Werkzeug = "3.1.6" pytest = "9.0.3" pytest-lazy-fixtures = "1.4.0" # switched from https://github.com/TvoroG/pytest-lazy-fixture as it's no longer supported -xlsx2csv = "0.8.4" # polars requirement +fastexcel = "0.21.0" # polars requirement [tool.poetry.group.lint] optional = true diff --git a/tests/test_pipeline/test_spark_pipeline.py b/tests/test_pipeline/test_spark_pipeline.py index 642a7ea..52bd6a1 100644 --- a/tests/test_pipeline/test_spark_pipeline.py +++ b/tests/test_pipeline/test_spark_pipeline.py @@ -420,7 +420,7 @@ def test_error_report_where_report_is_expected( # pylint: disable=redefined-out report_records = ( pl.read_excel(report_uri) .filter(pl.col("Data Summary").is_not_null()) - .select(pl.col("Data Summary"), pl.col("_duplicated_0")) + .select(pl.col("Data Summary"), pl.col("__UNNAMED__1")) .rows() ) assert report_records == [ @@ -482,8 +482,8 @@ def test_error_report_where_report_is_expected( # pylint: disable=redefined-out "Error Code": "LONG_ORBIT", "Data Item Submission Name": "orbitalPeriod", "Errors and Warnings": "Planet has long orbital period", - "Record Index": 1, - "Value": 365.20001220703125, + "Record Index": "1", + "Value": "365.20001220703125", "ID": None, "Category": "Bad value", } @@ -495,8 +495,8 @@ def test_error_report_where_report_is_expected( # pylint: disable=redefined-out "Error Code": "STRONG_GRAVITY", "Data Item Submission Name": "gravity", "Errors and Warnings": "Planet has too strong gravity", - "Record Index": 1, - "Value": 9.800000190734863, + "Record Index": "1", + "Value": "9.800000190734863", "ID": None, "Category": "Bad value", }