From eca745923f3d943676aef08fa1255a06f14e7f37 Mon Sep 17 00:00:00 2001 From: Bruce Merry Date: Sat, 9 Dec 2023 18:50:50 +0200 Subject: [PATCH 1/4] Update dependencies to more recent versions Mostly they're updated to the latest. I've held back on two packages: - pyparsing: the exception messages have changed, and not always in a good way (they're less intelligible); that was breaking unit tests for the errors. - aiomonitor: the new version has an HTTP console, but I'm not sure what the implications of that are when run under katsdpcontroller. I've opened NGC-1179 to deal with that. There are a few code changes needed to deal with the upgrades (mostly to deal with deprecation warnings): - pytest-asyncio no longer wants you to override the `event_loop` fixture. See [here](https://pytest-asyncio.readthedocs.io/en/latest/how-to-guides/index.html) for it's recommended replacements. - xarray warns that Dataset.dims should be replaced by Dataset.sizes - signal.py was using a 1D array with 1 element as if it was a scalar. numpy is now warning about that. The type annotation has also been fixed. --- .../antenna_channelised_voltage/__init__.py | 4 + .../baseline_correlation_products/__init__.py | 5 + qualification/conftest.py | 8 - qualification/requirements.txt | 95 ++++++----- requirements-dev.txt | 150 +++++++++--------- requirements.txt | 63 ++++---- scratch/benchmarks/requirements.txt | 34 ++-- src/katgpucbf/dsim/send.py | 10 +- src/katgpucbf/dsim/server.py | 2 +- src/katgpucbf/dsim/signal.py | 7 +- test/test_utils.py | 13 +- 11 files changed, 190 insertions(+), 201 deletions(-) diff --git a/qualification/antenna_channelised_voltage/__init__.py b/qualification/antenna_channelised_voltage/__init__.py index 03aaab67a..72678f8f2 100644 --- a/qualification/antenna_channelised_voltage/__init__.py +++ b/qualification/antenna_channelised_voltage/__init__.py @@ -23,6 +23,7 @@ import asyncio import numpy as np +import pytest from numpy.typing import ArrayLike from katgpucbf import DIG_SAMPLE_BITS, N_POLS @@ -30,6 +31,9 @@ from .. import BaselineCorrelationProductsReceiver, CorrelatorRemoteControl from ..reporter import Reporter +# Make the event_loop package-scoped +pytestmark = pytest.mark.asyncio(scope="package") + async def sample_tone_response_hdr( correlator: CorrelatorRemoteControl, diff --git a/qualification/baseline_correlation_products/__init__.py b/qualification/baseline_correlation_products/__init__.py index 6e031999e..87b7343fa 100644 --- a/qualification/baseline_correlation_products/__init__.py +++ b/qualification/baseline_correlation_products/__init__.py @@ -1 +1,6 @@ # noqa: D104 + +import pytest + +# Make the event_loop package-scoped +pytestmark = pytest.mark.asyncio(scope="package") diff --git a/qualification/conftest.py b/qualification/conftest.py index 7560d4cf4..f0c95f8a0 100644 --- a/qualification/conftest.py +++ b/qualification/conftest.py @@ -171,14 +171,6 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: metafunc.parametrize("n_channels, narrowband_decimation", configs, indirect=True) -# Need to redefine this from pytest-asyncio to have it at package scope -@pytest.fixture(scope="package") -def event_loop(): # noqa: D103 - loop = asyncio.get_event_loop_policy().new_event_loop() - yield loop - loop.close() - - @pytest.fixture(scope="package") def n_antennas(request: pytest.FixtureRequest): # noqa: D401 """Number of antennas, i.e. size of the array.""" diff --git a/qualification/requirements.txt b/qualification/requirements.txt index 5b3b09cad..11087b115 100644 --- a/qualification/requirements.txt +++ b/qualification/requirements.txt @@ -9,7 +9,7 @@ aioconsole==0.6.2 # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt # aiomonitor -aiohttp==3.8.6 +aiohttp==3.9.1 # via # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt @@ -34,7 +34,7 @@ appdirs==1.4.4 # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt # katsdpsigproc -async-timeout==4.0.2 +async-timeout==4.0.3 # via # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt @@ -47,36 +47,34 @@ attrs==23.1.0 # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt # aiohttp -certifi==2023.7.22 +certifi==2023.11.17 # via # -c qualification/../requirements-dev.txt # requests -charset-normalizer==3.2.0 +charset-normalizer==3.3.2 # via # -c qualification/../requirements-dev.txt - # -c qualification/../requirements.txt - # aiohttp # requests -click==8.1.6 +click==8.1.7 # via # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt # dask -cloudpickle==2.2.1 +cloudpickle==3.0.0 # via # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt # dask -contourpy==1.1.0 +contourpy==1.2.0 # via matplotlib -cycler==0.11.0 +cycler==0.12.1 # via matplotlib -dask==2023.7.1 +dask==2023.12.0 # via # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt # katgpucbf (setup.cfg) -dateparser==1.1.8 +dateparser==1.2.0 # via prometheus-api-client decorator==5.1.1 # via @@ -84,15 +82,15 @@ decorator==5.1.1 # -c qualification/../requirements.txt # aiokatcp # katsdpsigproc -docutils==0.18.1 +docutils==0.20.1 # via # -c qualification/../requirements-dev.txt # katgpucbf (setup.cfg) -exceptiongroup==1.1.2 +exceptiongroup==1.2.0 # via # -c qualification/../requirements-dev.txt # pytest -fonttools==4.41.1 +fonttools==4.46.0 # via matplotlib frozenlist==1.4.0 # via @@ -100,7 +98,7 @@ frozenlist==1.4.0 # -c qualification/../requirements.txt # aiohttp # aiosignal -fsspec==2023.6.0 +fsspec==2023.12.1 # via # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt @@ -112,13 +110,13 @@ hiredis==2.2.3 # katsdptelstate httmock==1.4.0 # via prometheus-api-client -idna==3.4 +idna==3.6 # via # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt # requests # yarl -importlib-metadata==6.8.0 +importlib-metadata==7.0.0 # via # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt @@ -142,9 +140,9 @@ katsdptelstate==0.13 # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt # katgpucbf (setup.cfg) -kiwisolver==1.4.4 +kiwisolver==1.4.5 # via matplotlib -llvmlite==0.39.1 +llvmlite==0.41.1 # via # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt @@ -154,7 +152,7 @@ locket==1.0.0 # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt # partd -mako==1.2.4 +mako==1.3.0 # via # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt @@ -164,11 +162,11 @@ markupsafe==2.1.3 # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt # mako -matplotlib==3.7.2 +matplotlib==3.8.2 # via # katgpucbf (setup.cfg) # prometheus-api-client -msgpack==1.0.5 +msgpack==1.0.7 # via # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt @@ -184,13 +182,13 @@ netifaces==0.11.0 # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt # katsdpservices -numba==0.56.4 +numba==0.58.1 # via # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt # katgpucbf (setup.cfg) # katsdpsigproc -numpy==1.23.5 +numpy==1.26.2 # via # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt @@ -207,7 +205,7 @@ numpy==1.23.5 # xarray ordered-set==4.1.0 # via pylatex -packaging==23.1 +packaging==23.2 # via # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt @@ -215,32 +213,32 @@ packaging==23.1 # matplotlib # pytest # xarray -pandas==2.0.3 +pandas==2.1.4 # via # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt # katsdpsigproc # prometheus-api-client # xarray -partd==1.4.0 +partd==1.4.1 # via # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt # dask -pillow==10.0.1 +pillow==10.1.0 # via matplotlib -pluggy==1.2.0 +pluggy==1.3.0 # via # -c qualification/../requirements-dev.txt # pytest -prometheus-api-client==0.5.3 +prometheus-api-client==0.5.4 # via katgpucbf (setup.cfg) prometheus-async==22.2.0 # via # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt # katgpucbf (setup.cfg) -prometheus-client==0.17.1 +prometheus-client==0.19.0 # via # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt @@ -251,7 +249,7 @@ pygelf==0.4.2 # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt # katsdpservices -pylatex==1.4.1 +pylatex==1.4.2 # via katgpucbf (setup.cfg) pyparsing==3.0.9 # via @@ -259,7 +257,7 @@ pyparsing==3.0.9 # -c qualification/../requirements.txt # katgpucbf (setup.cfg) # matplotlib -pytest==7.4.0 +pytest==7.4.3 # via # -c qualification/../requirements-dev.txt # katgpucbf (setup.cfg) @@ -267,11 +265,11 @@ pytest==7.4.0 # pytest-check # pytest-custom-exit-code # pytest-reportlog -pytest-asyncio==0.21.1 +pytest-asyncio==0.23.2 # via # -c qualification/../requirements-dev.txt # katgpucbf (setup.cfg) -pytest-check==2.2.0 +pytest-check==2.2.2 # via katgpucbf (setup.cfg) pytest-custom-exit-code==0.3.0 # via @@ -286,7 +284,7 @@ python-dateutil==2.8.2 # dateparser # matplotlib # pandas -pytz==2023.3 +pytz==2023.3.post1 # via # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt @@ -297,19 +295,19 @@ pyyaml==6.0.1 # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt # dask -redis==4.6.0 +redis==5.0.1 # via # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt # katsdptelstate -regex==2023.6.3 +regex==2023.10.3 # via dateparser requests==2.31.0 # via # -c qualification/../requirements-dev.txt # httmock # prometheus-api-client -scipy==1.11.1 +scipy==1.11.4 # via # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt @@ -341,7 +339,7 @@ toolz==0.12.0 # -c qualification/../requirements.txt # dask # partd -typing-extensions==4.7.1 +typing-extensions==4.8.0 # via # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt @@ -352,32 +350,29 @@ tzdata==2023.3 # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt # pandas -tzlocal==5.0.1 +tzlocal==5.2 # via dateparser -urllib3==2.0.7 +urllib3==2.1.0 # via # -c qualification/../requirements-dev.txt # requests -wrapt==1.15.0 +wrapt==1.16.0 # via # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt # prometheus-async -xarray==2023.7.0 +xarray==2023.12.0 # via # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt # katgpucbf (setup.cfg) -yarl==1.9.2 +yarl==1.9.4 # via # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt # aiohttp -zipp==3.16.2 +zipp==3.17.0 # via # -c qualification/../requirements-dev.txt # -c qualification/../requirements.txt # importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/requirements-dev.txt b/requirements-dev.txt index 41d159a2e..d9b11ffa1 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,7 +8,7 @@ aioconsole==0.6.2 # via # -c requirements.txt # aiomonitor -aiohttp==3.8.6 +aiohttp==3.9.1 # via # -c requirements.txt # prometheus-async @@ -31,11 +31,11 @@ appdirs==1.4.4 # -c requirements.txt # katsdpsigproc # pycuda -asttokens==2.2.1 +asttokens==2.4.1 # via stack-data async-solipsism==0.5 # via katgpucbf (setup.cfg) -async-timeout==4.0.2 +async-timeout==4.0.3 # via # -c requirements.txt # aiohttp @@ -46,33 +46,28 @@ attrs==23.1.0 # via # -c requirements.txt # aiohttp -babel==2.12.1 +babel==2.13.1 # via sphinx -backcall==0.2.0 - # via ipython -build==0.10.0 +build==1.0.3 # via pip-tools -certifi==2023.7.22 +certifi==2023.11.17 # via requests -cfgv==3.3.1 +cfgv==3.4.0 # via pre-commit -charset-normalizer==3.2.0 - # via - # -c requirements.txt - # aiohttp - # requests -click==8.1.6 +charset-normalizer==3.3.2 + # via requests +click==8.1.7 # via # -c requirements.txt # dask # pip-tools -cloudpickle==2.2.1 +cloudpickle==3.0.0 # via # -c requirements.txt # dask -coverage==7.2.7 +coverage==7.3.2 # via pytest-cov -dask==2023.7.1 +dask==2023.12.0 # via # -c requirements.txt # katgpucbf (setup.cfg) @@ -84,26 +79,28 @@ decorator==5.1.1 # katsdpsigproc distlib==0.3.7 # via virtualenv -docutils==0.18.1 +docutils==0.20.1 # via # pybtex-docutils # sphinx # sphinx-rtd-theme # sphinxcontrib-bibtex -exceptiongroup==1.1.2 - # via pytest +exceptiongroup==1.2.0 + # via + # ipython + # pytest execnet==2.0.2 # via pytest-xdist -executing==1.2.0 +executing==2.0.1 # via stack-data -filelock==3.12.2 +filelock==3.13.1 # via virtualenv frozenlist==1.4.0 # via # -c requirements.txt # aiohttp # aiosignal -fsspec==2023.6.0 +fsspec==2023.12.1 # via # -c requirements.txt # dask @@ -111,24 +108,24 @@ hiredis==2.2.3 # via # -c requirements.txt # katsdptelstate -identify==2.5.26 +identify==2.5.33 # via pre-commit -idna==3.4 +idna==3.6 # via # -c requirements.txt # requests # yarl imagesize==1.4.1 # via sphinx -importlib-metadata==6.8.0 +importlib-metadata==7.0.0 # via # -c requirements.txt # dask iniconfig==2.0.0 # via pytest -ipython==8.14.0 +ipython==8.18.1 # via -r requirements-dev.in -jedi==0.19.0 +jedi==0.19.1 # via ipython jinja2==3.1.2 # via sphinx @@ -146,7 +143,7 @@ katsdptelstate==0.13 # katgpucbf (setup.cfg) latexcodec==2.0.1 # via pybtex -llvmlite==0.39.1 +llvmlite==0.41.1 # via # -c requirements.txt # numba @@ -154,7 +151,7 @@ locket==1.0.0 # via # -c requirements.txt # partd -mako==1.2.4 +mako==1.3.0 # via # -c requirements.txt # katsdpsigproc @@ -166,7 +163,7 @@ markupsafe==2.1.3 # mako matplotlib-inline==0.1.6 # via ipython -msgpack==1.0.5 +msgpack==1.0.7 # via # -c requirements.txt # katsdptelstate @@ -181,12 +178,12 @@ netifaces==0.11.0 # katsdpservices nodeenv==1.8.0 # via pre-commit -numba==0.56.4 +numba==0.58.1 # via # -c requirements.txt # katgpucbf (setup.cfg) # katsdpsigproc -numpy==1.23.5 +numpy==1.26.2 # via # -c requirements.txt # katgpucbf (setup.cfg) @@ -197,7 +194,7 @@ numpy==1.23.5 # scipy # spead2 # xarray -packaging==23.1 +packaging==23.2 # via # -c requirements.txt # build @@ -205,42 +202,40 @@ packaging==23.1 # pytest # sphinx # xarray -pandas==2.0.3 +pandas==2.1.4 # via # -c requirements.txt # katsdpsigproc # xarray parso==0.8.3 # via jedi -partd==1.4.0 +partd==1.4.1 # via # -c requirements.txt # dask -pexpect==4.8.0 - # via ipython -pickleshare==0.7.5 +pexpect==4.9.0 # via ipython -pip-tools==7.1.0 +pip-tools==7.3.0 # via -r requirements-dev.in -platformdirs==3.10.0 +platformdirs==4.1.0 # via # -c requirements.txt # pytools # virtualenv -pluggy==1.2.0 +pluggy==1.3.0 # via pytest -pre-commit==3.3.3 +pre-commit==3.5.0 # via -r requirements-dev.in prometheus-async==22.2.0 # via # -c requirements.txt # katgpucbf (setup.cfg) -prometheus-client==0.17.1 +prometheus-client==0.19.0 # via # -c requirements.txt # katgpucbf (setup.cfg) # prometheus-async -prompt-toolkit==3.0.39 +prompt-toolkit==3.0.41 # via ipython ptyprocess==0.7.0 # via pexpect @@ -250,9 +245,9 @@ pybtex==0.24.0 # via # pybtex-docutils # sphinxcontrib-bibtex -pybtex-docutils==1.0.2 +pybtex-docutils==1.0.3 # via sphinxcontrib-bibtex -pycuda==2022.2.2 +pycuda==2023.1 # via # -c requirements.txt # katsdpsigproc @@ -260,7 +255,7 @@ pygelf==0.4.2 # via # -c requirements.txt # katsdpservices -pygments==2.15.1 +pygments==2.17.2 # via # ipython # sphinx @@ -270,7 +265,7 @@ pyparsing==3.0.9 # katgpucbf (setup.cfg) pyproject-hooks==1.0.0 # via build -pytest==7.4.0 +pytest==7.4.3 # via # katgpucbf (setup.cfg) # pytest-asyncio @@ -278,15 +273,15 @@ pytest==7.4.0 # pytest-custom-exit-code # pytest-mock # pytest-xdist -pytest-asyncio==0.21.1 +pytest-asyncio==0.23.2 # via katgpucbf (setup.cfg) pytest-cov==4.1.0 # via -r requirements-dev.in pytest-custom-exit-code==0.3.0 # via katgpucbf (setup.cfg) -pytest-mock==3.11.1 +pytest-mock==3.12.0 # via katgpucbf (setup.cfg) -pytest-xdist==3.3.1 +pytest-xdist==3.5.0 # via -r requirements-dev.in python-dateutil==2.8.2 # via @@ -296,7 +291,7 @@ pytools==2023.1.1 # via # -c requirements.txt # pycuda -pytz==2023.3 +pytz==2023.3.post1 # via # -c requirements.txt # pandas @@ -306,13 +301,13 @@ pyyaml==6.0.1 # dask # pre-commit # pybtex -redis==4.6.0 +redis==5.0.1 # via # -c requirements.txt # katsdptelstate requests==2.31.0 # via sphinx -scipy==1.11.1 +scipy==1.11.4 # via # -c requirements.txt # katgpucbf (setup.cfg) @@ -331,37 +326,42 @@ spead2==4.1.1 # via # -c requirements.txt # katgpucbf (setup.cfg) -sphinx==6.2.1 +sphinx==7.2.6 # via # katgpucbf (setup.cfg) # sphinx-mathjax-offline # sphinx-rtd-theme + # sphinxcontrib-applehelp # sphinxcontrib-bibtex + # sphinxcontrib-devhelp + # sphinxcontrib-htmlhelp # sphinxcontrib-jquery + # sphinxcontrib-qthelp + # sphinxcontrib-serializinghtml # sphinxcontrib-tikz sphinx-mathjax-offline==0.0.2 # via katgpucbf (setup.cfg) -sphinx-rtd-theme==1.2.2 +sphinx-rtd-theme==2.0.0 # via katgpucbf (setup.cfg) -sphinxcontrib-applehelp==1.0.4 +sphinxcontrib-applehelp==1.0.7 # via sphinx -sphinxcontrib-bibtex==2.5.0 +sphinxcontrib-bibtex==2.6.1 # via katgpucbf (setup.cfg) -sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-devhelp==1.0.5 # via sphinx -sphinxcontrib-htmlhelp==2.0.1 +sphinxcontrib-htmlhelp==2.0.4 # via sphinx sphinxcontrib-jquery==4.1 # via sphinx-rtd-theme sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-qthelp==1.0.6 # via sphinx -sphinxcontrib-serializinghtml==1.1.5 +sphinxcontrib-serializinghtml==1.1.9 # via sphinx sphinxcontrib-tikz==0.4.16 # via katgpucbf (setup.cfg) -stack-data==0.6.2 +stack-data==0.6.3 # via ipython terminaltables==3.1.10 # via @@ -379,11 +379,11 @@ toolz==0.12.0 # -c requirements.txt # dask # partd -traitlets==5.9.0 +traitlets==5.14.0 # via # ipython # matplotlib-inline -typing-extensions==4.7.1 +typing-extensions==4.8.0 # via # -c requirements.txt # aiokatcp @@ -393,27 +393,27 @@ tzdata==2023.3 # via # -c requirements.txt # pandas -urllib3==2.0.7 +urllib3==2.1.0 # via requests -virtualenv==20.24.2 +virtualenv==20.25.0 # via pre-commit -wcwidth==0.2.6 +wcwidth==0.2.12 # via prompt-toolkit -wheel==0.41.0 +wheel==0.42.0 # via pip-tools -wrapt==1.15.0 +wrapt==1.16.0 # via # -c requirements.txt # prometheus-async -xarray==2023.7.0 +xarray==2023.12.0 # via # -c requirements.txt # katgpucbf (setup.cfg) -yarl==1.9.2 +yarl==1.9.4 # via # -c requirements.txt # aiohttp -zipp==3.16.2 +zipp==3.17.0 # via # -c requirements.txt # importlib-metadata diff --git a/requirements.txt b/requirements.txt index ab06b6c53..8bd96e24f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ # aioconsole==0.6.2 # via aiomonitor -aiohttp==3.8.6 +aiohttp==3.9.1 # via prometheus-async aiokatcp==1.8.0 # via katgpucbf (setup.cfg) @@ -18,22 +18,20 @@ appdirs==1.4.4 # via # katsdpsigproc # pycuda -async-timeout==4.0.2 +async-timeout==4.0.3 # via # aiohttp # aiokatcp # redis attrs==23.1.0 # via aiohttp -cffi==1.15.1 +cffi==1.16.0 # via vkgdr -charset-normalizer==3.2.0 - # via aiohttp -click==8.1.6 +click==8.1.7 # via dask -cloudpickle==2.2.1 +cloudpickle==3.0.0 # via dask -dask==2023.7.1 +dask==2023.12.0 # via katgpucbf (setup.cfg) decorator==5.1.1 # via @@ -43,13 +41,13 @@ frozenlist==1.4.0 # via # aiohttp # aiosignal -fsspec==2023.6.0 +fsspec==2023.12.1 # via dask hiredis==2.2.3 # via katsdptelstate -idna==3.4 +idna==3.6 # via yarl -importlib-metadata==6.8.0 +importlib-metadata==7.0.0 # via dask katsdpservices==1.2 # via katgpucbf (setup.cfg) @@ -57,17 +55,17 @@ katsdpsigproc==1.8.1 # via katgpucbf (setup.cfg) katsdptelstate==0.13 # via katgpucbf (setup.cfg) -llvmlite==0.39.1 +llvmlite==0.41.1 # via numba locket==1.0.0 # via partd -mako==1.2.4 +mako==1.3.0 # via # katsdpsigproc # pycuda markupsafe==2.1.3 # via mako -msgpack==1.0.5 +msgpack==1.0.7 # via katsdptelstate multidict==6.0.4 # via @@ -75,11 +73,11 @@ multidict==6.0.4 # yarl netifaces==0.11.0 # via katsdpservices -numba==0.56.4 +numba==0.58.1 # via # katgpucbf (setup.cfg) # katsdpsigproc -numpy==1.23.5 +numpy==1.26.2 # via # katgpucbf (setup.cfg) # katsdpsigproc @@ -89,29 +87,29 @@ numpy==1.23.5 # scipy # spead2 # xarray -packaging==23.1 +packaging==23.2 # via # dask # xarray -pandas==2.0.3 +pandas==2.1.4 # via # katsdpsigproc # xarray -partd==1.4.0 +partd==1.4.1 # via dask -platformdirs==3.10.0 +platformdirs==4.1.0 # via pytools prometheus-async==22.2.0 # via katgpucbf (setup.cfg) -prometheus-client==0.17.1 +prometheus-client==0.19.0 # via # katgpucbf (setup.cfg) # prometheus-async -psutil==5.9.5 +psutil==5.9.6 # via -r requirements.in pycparser==2.21 # via cffi -pycuda==2022.2.2 +pycuda==2023.1 # via katsdpsigproc pygelf==0.4.2 # via katsdpservices @@ -121,13 +119,13 @@ python-dateutil==2.8.2 # via pandas pytools==2023.1.1 # via pycuda -pytz==2023.3 +pytz==2023.3.post1 # via pandas pyyaml==6.0.1 # via dask -redis==4.6.0 +redis==5.0.1 # via katsdptelstate -scipy==1.11.1 +scipy==1.11.4 # via # katgpucbf (setup.cfg) # katsdpsigproc @@ -143,7 +141,7 @@ toolz==0.12.0 # via # dask # partd -typing-extensions==4.7.1 +typing-extensions==4.8.0 # via # aiokatcp # katsdpsigproc @@ -152,14 +150,11 @@ tzdata==2023.3 # via pandas vkgdr==0.1 # via katgpucbf (setup.cfg) -wrapt==1.15.0 +wrapt==1.16.0 # via prometheus-async -xarray==2023.7.0 +xarray==2023.12.0 # via katgpucbf (setup.cfg) -yarl==1.9.2 +yarl==1.9.4 # via aiohttp -zipp==3.16.2 +zipp==3.17.0 # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/scratch/benchmarks/requirements.txt b/scratch/benchmarks/requirements.txt index 8a7050cb3..97abebb17 100644 --- a/scratch/benchmarks/requirements.txt +++ b/scratch/benchmarks/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --config=pyproject.toml --output-file=scratch/benchmarks/requirements.txt scratch/benchmarks/requirements.in # -aiohttp==3.8.6 +aiohttp==3.9.1 # via # -c scratch/benchmarks/../../qualification/requirements.txt # -c scratch/benchmarks/../../requirements-dev.txt @@ -16,7 +16,7 @@ aiosignal==1.3.1 # -c scratch/benchmarks/../../requirements-dev.txt # -c scratch/benchmarks/../../requirements.txt # aiohttp -async-timeout==4.0.2 +async-timeout==4.0.3 # via # -c scratch/benchmarks/../../qualification/requirements.txt # -c scratch/benchmarks/../../requirements-dev.txt @@ -31,17 +31,11 @@ attrs==23.1.0 # -c scratch/benchmarks/../../requirements-dev.txt # -c scratch/benchmarks/../../requirements.txt # aiohttp -cffi==1.15.1 +cffi==1.16.0 # via # -c scratch/benchmarks/../../requirements.txt # cryptography -charset-normalizer==3.2.0 - # via - # -c scratch/benchmarks/../../qualification/requirements.txt - # -c scratch/benchmarks/../../requirements-dev.txt - # -c scratch/benchmarks/../../requirements.txt - # aiohttp -cryptography==41.0.4 +cryptography==41.0.7 # via asyncssh frozenlist==1.4.0 # via @@ -50,7 +44,7 @@ frozenlist==1.4.0 # -c scratch/benchmarks/../../requirements.txt # aiohttp # aiosignal -idna==3.4 +idna==3.6 # via # -c scratch/benchmarks/../../qualification/requirements.txt # -c scratch/benchmarks/../../requirements-dev.txt @@ -63,7 +57,7 @@ multidict==6.0.4 # -c scratch/benchmarks/../../requirements.txt # aiohttp # yarl -numpy==1.23.5 +numpy==1.26.2 # via # -c scratch/benchmarks/../../qualification/requirements.txt # -c scratch/benchmarks/../../requirements-dev.txt @@ -73,22 +67,22 @@ numpy==1.23.5 # patsy # scipy # statsmodels -packaging==23.1 +packaging==23.2 # via # -c scratch/benchmarks/../../qualification/requirements.txt # -c scratch/benchmarks/../../requirements-dev.txt # -c scratch/benchmarks/../../requirements.txt # statsmodels -pandas==2.0.3 +pandas==2.1.4 # via # -c scratch/benchmarks/../../qualification/requirements.txt # -c scratch/benchmarks/../../requirements-dev.txt # -c scratch/benchmarks/../../requirements.txt # -r scratch/benchmarks/requirements.in # statsmodels -patsy==0.5.3 +patsy==0.5.4 # via statsmodels -prometheus-client==0.17.1 +prometheus-client==0.19.0 # via # -c scratch/benchmarks/../../qualification/requirements.txt # -c scratch/benchmarks/../../requirements-dev.txt @@ -104,13 +98,13 @@ python-dateutil==2.8.2 # -c scratch/benchmarks/../../requirements-dev.txt # -c scratch/benchmarks/../../requirements.txt # pandas -pytz==2023.3 +pytz==2023.3.post1 # via # -c scratch/benchmarks/../../qualification/requirements.txt # -c scratch/benchmarks/../../requirements-dev.txt # -c scratch/benchmarks/../../requirements.txt # pandas -scipy==1.11.1 +scipy==1.11.4 # via # -c scratch/benchmarks/../../qualification/requirements.txt # -c scratch/benchmarks/../../requirements-dev.txt @@ -131,7 +125,7 @@ tomli==2.0.1 ; python_version < "3.11" # -c scratch/benchmarks/../../qualification/requirements.txt # -c scratch/benchmarks/../../requirements-dev.txt # -r scratch/benchmarks/requirements.in -typing-extensions==4.7.1 +typing-extensions==4.8.0 # via # -c scratch/benchmarks/../../qualification/requirements.txt # -c scratch/benchmarks/../../requirements-dev.txt @@ -143,7 +137,7 @@ tzdata==2023.3 # -c scratch/benchmarks/../../requirements-dev.txt # -c scratch/benchmarks/../../requirements.txt # pandas -yarl==1.9.2 +yarl==1.9.4 # via # -c scratch/benchmarks/../../qualification/requirements.txt # -c scratch/benchmarks/../../requirements-dev.txt diff --git a/src/katgpucbf/dsim/send.py b/src/katgpucbf/dsim/send.py index cc8ae903b..7f6eb85f1 100644 --- a/src/katgpucbf/dsim/send.py +++ b/src/katgpucbf/dsim/send.py @@ -65,9 +65,9 @@ class HeapSet: """ def __init__(self, data: xr.Dataset) -> None: - if data.dims["time"] < 2: + if data.sizes["time"] < 2: raise ValueError("time dimension must have at least 2 elements") - middle = data.dims["time"] // 2 + middle = data.sizes["time"] // 2 self.data = data self.parts = [data.isel(time=np.s_[:middle]), data.isel(time=np.s_[middle:])] for part in self.parts: @@ -297,7 +297,7 @@ async def run(self, first_timestamp: int, time_converter: TimeConverter) -> None self._next_timestamp = first_timestamp self.time_converter = time_converter # Prepare initial timestamps - first_end_timestamp = first_timestamp + self.heap_set.data.dims["time"] * self.heap_samples + first_end_timestamp = first_timestamp + self.heap_set.data.sizes["time"] * self.heap_samples self.heap_set.data["timestamps"][:] = np.arange( first_timestamp, first_end_timestamp, self.heap_samples, dtype=">u8" ) @@ -309,12 +309,12 @@ async def run(self, first_timestamp: int, time_converter: TimeConverter) -> None # set_heaps may have swapped heap_set out from under us during # the await, so re-initialise part. part = self.heap_set.parts[i] - part["timestamps"] += self.heap_set.data.dims["time"] * self.heap_samples + part["timestamps"] += self.heap_set.data.sizes["time"] * self.heap_samples send_future = self.stream.async_send_heaps( part.attrs["heap_reference_list"], spead2.send.GroupMode.SERIAL ) self._futures[i] = send_future - self._next_timestamp += part.dims["time"] * self.heap_samples + self._next_timestamp += part.sizes["time"] * self.heap_samples send_future.add_done_callback( functools.partial( self._update_metrics, self._next_timestamp, part["heaps"].size, part["payload"].nbytes diff --git a/src/katgpucbf/dsim/server.py b/src/katgpucbf/dsim/server.py index 128a76b79..97758ef59 100644 --- a/src/katgpucbf/dsim/server.py +++ b/src/katgpucbf/dsim/server.py @@ -221,7 +221,7 @@ async def request_signals(self, ctx, signals_str: str, period: int | None = None signals = parse_signals(signals_str) except (pp.ParseBaseException, TerminalError) as exc: raise aiokatcp.FailReply(str(exc)) - n_pol = self.spare.data.dims["pol"] + n_pol = self.spare.data.sizes["pol"] if len(signals) != n_pol: raise aiokatcp.FailReply(f"expected {n_pol} signals, received {len(signals)}") return await self.set_signals(signals, signals_str, period) diff --git a/src/katgpucbf/dsim/signal.py b/src/katgpucbf/dsim/signal.py index f1eb14578..b96df1311 100644 --- a/src/katgpucbf/dsim/signal.py +++ b/src/katgpucbf/dsim/signal.py @@ -209,10 +209,11 @@ class CW(Signal): frequency: float @staticmethod - def _sample_chunk(offset: np.int64, *, amplitude: np.float32, chunk_size: int, scale: float) -> np.ndarray: + def _sample_chunk(offset: np.ndarray, *, amplitude: np.float32, chunk_size: int, scale: float) -> np.ndarray: """Compute :math:`np.cos(np.arange(offset, n + offset) * scale) * amplitude` efficiently. - The return value is single precision. + The return value is single precision. `offset` must be an array with a + single element. """ # Compute the complex exponential. Because it is being regularly # sampled, it is possible to do this efficiently by repeated @@ -220,7 +221,7 @@ def _sample_chunk(offset: np.int64, *, amplitude: np.float32, chunk_size: int, s # computation in single precision without losing much precision # (experimentally the results seem to be off by less than 1e-6). out = np.empty(chunk_size, np.complex64) - out[0] = np.exp(offset * scale * 1j) * amplitude + out[0] = np.exp(offset[0] * scale * 1j) * amplitude valid = 1 while valid < chunk_size: # Rotate the segment [0, valid) by valid steps, giving the segment diff --git a/test/test_utils.py b/test/test_utils.py index 6d32a7c41..8c73c3d92 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -19,7 +19,7 @@ import asyncio import gc import weakref -from typing import AsyncGenerator, Generator +from typing import AsyncGenerator from unittest import mock import aiokatcp @@ -30,6 +30,11 @@ from katgpucbf.utils import DeviceStatusSensor, TimeConverter, TimeoutSensorStatusObserver +class AsyncSolipsismPolicy(asyncio.DefaultEventLoopPolicy): + def new_event_loop(self) -> async_solipsism.EventLoop: + return async_solipsism.EventLoop() + + class TestDeviceStatusSensor: """Test :class:`.DeviceStatusSensor`. @@ -75,11 +80,9 @@ class TestTimeoutSensorStatus: """Tests for :func:`katgpucbf.utils.timeout_sensor_status`.""" @pytest.fixture - def event_loop(self) -> Generator[async_solipsism.EventLoop, None, None]: + def event_loop_policy(self) -> asyncio.AbstractEventLoopPolicy: """Use async_solipsism event loop.""" - loop = async_solipsism.EventLoop() - yield loop - loop.close() + return AsyncSolipsismPolicy() @pytest.fixture def sensor(self) -> aiokatcp.Sensor: From 18abe67a0ffd67514011fea43f241c60f5e4e54c Mon Sep 17 00:00:00 2001 From: Bruce Merry Date: Sat, 9 Dec 2023 20:29:27 +0200 Subject: [PATCH 2/4] Update versions in .pre-commit-config.yaml This makes them match better to requirements.txt and friend. I'm hoping the update to pip-tools might also fix a weird glitch where sometimes it reports coverage as being a dependency of itself. --- .pre-commit-config.yaml | 29 +++++++++++++++-------------- requirements-dev.txt | 4 +++- scratch/benchmarks/remote.py | 2 +- scratch/benchmarks/requirements.txt | 2 +- 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5bcfe102e..ebde14b0d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: check-merge-conflict - id: check-shebang-scripts-are-executable @@ -14,11 +14,11 @@ repos: - id: no-commit-to-branch # without arguments, master/main will be protected. - id: trailing-whitespace - repo: https://github.com/psf/black - rev: 23.7.0 + rev: 23.11.0 hooks: - id: black - repo: https://github.com/pycqa/isort - rev: 5.12.0 + rev: 5.13.0 hooks: - id: isort name: isort @@ -38,7 +38,7 @@ repos: # check the benchmarks/ directory because that code is more complex. exclude: 'setup\.py|scratch/(?!benchmarks)' - repo: https://github.com/pre-commit/mirrors-mypy - rev: 'v1.4.1' + rev: 'v1.7.1' hooks: - id: mypy # Passing filenames to mypy can do odd things. See @@ -53,24 +53,25 @@ repos: additional_dependencies: [ 'aiokatcp==1.8.0', 'asyncssh==2.14.1', - 'dask==2023.7.1', + 'dask==2023.12.0', 'katsdpsigproc==1.8.1', 'katsdptelstate==0.13', - 'numpy==1.24.4', - # Note: actual pandas version is 2.0.3, but the stubs for it depend - # on a newer version of numpy. - 'pandas-stubs==2.0.2.230605', + 'numpy==1.26.2', + # Note: actual pandas version is 2.1.4, but pandas-stubs doesn't + # have a release for that at the time of writing. + 'pandas-stubs==2.1.1.230928', 'pyparsing==3.0.9', - 'pytest==7.4.0', + 'pytest==7.4.3', + 'redis==5.0.1', # Indirect dependency of katsdptelstate 'spead2==4.1.1', 'types-decorator==5.1.1', - 'types-docutils==0.18.1', - 'types-redis==4.6.0', # Indirectly needed by katsdptelstate - 'types-setuptools==68.2.0.0', # Indirectly needed via redis + # Note: actual docutils version is 0.20.1, but types-docutils + # doesn't have a release for that yet. + 'types-docutils==0.20.0.3', 'types-six==1.16.0', ] - repo: https://github.com/jazzband/pip-tools - rev: 7.1.0 + rev: 7.3.0 hooks: - id: pip-compile name: pip-compile requirements.txt diff --git a/requirements-dev.txt b/requirements-dev.txt index d9b11ffa1..30ab78cf9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -66,7 +66,9 @@ cloudpickle==3.0.0 # -c requirements.txt # dask coverage==7.3.2 - # via pytest-cov + # via + # coverage + # pytest-cov dask==2023.12.0 # via # -c requirements.txt diff --git a/scratch/benchmarks/remote.py b/scratch/benchmarks/remote.py index 0e0ea95e6..dddb5e79e 100644 --- a/scratch/benchmarks/remote.py +++ b/scratch/benchmarks/remote.py @@ -152,7 +152,7 @@ async def run_tasks( infiniband_devices.pop() server_info = ServerInfo(cpus=cpus, infiniband_devices=infiniband_devices) await conn.run(f"docker pull {image}", check=True) - procs = [] + procs: list[asyncssh.SSHClientProcess] = [] for i in range(n): command = factory(server, server_info, conn, i) procs.append( diff --git a/scratch/benchmarks/requirements.txt b/scratch/benchmarks/requirements.txt index 97abebb17..d37ec1e74 100644 --- a/scratch/benchmarks/requirements.txt +++ b/scratch/benchmarks/requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile --config=pyproject.toml --output-file=scratch/benchmarks/requirements.txt scratch/benchmarks/requirements.in +# pip-compile --output-file=scratch/benchmarks/requirements.txt scratch/benchmarks/requirements.in # aiohttp==3.9.1 # via From da4a42cad0fab4aaa80b7041b0a861de5c0bb761 Mon Sep 17 00:00:00 2001 From: Bruce Merry Date: Wed, 13 Dec 2023 08:41:45 +0200 Subject: [PATCH 3/4] Revert upgrade to pytest-asyncio 0.23.2 pytest-asyncio 0.23 changed the way event loops work and caused a number of issues, including pytest-dev/asyncio#705 and pytest-dev/pytest-asyncio#718. I wasn't able to find an easy way around these for the qualification test, so for now I've pinned pytest-asyncio to <0.22 (the 0.22 release was a half-way point to 0.23 and has been yanked). --- .../antenna_channelised_voltage/__init__.py | 4 ---- .../baseline_correlation_products/__init__.py | 5 ----- qualification/conftest.py | 8 ++++++++ qualification/requirements.txt | 2 +- requirements-dev.txt | 2 +- setup.cfg | 2 +- test/test_utils.py | 13 +++++-------- 7 files changed, 16 insertions(+), 20 deletions(-) diff --git a/qualification/antenna_channelised_voltage/__init__.py b/qualification/antenna_channelised_voltage/__init__.py index 72678f8f2..03aaab67a 100644 --- a/qualification/antenna_channelised_voltage/__init__.py +++ b/qualification/antenna_channelised_voltage/__init__.py @@ -23,7 +23,6 @@ import asyncio import numpy as np -import pytest from numpy.typing import ArrayLike from katgpucbf import DIG_SAMPLE_BITS, N_POLS @@ -31,9 +30,6 @@ from .. import BaselineCorrelationProductsReceiver, CorrelatorRemoteControl from ..reporter import Reporter -# Make the event_loop package-scoped -pytestmark = pytest.mark.asyncio(scope="package") - async def sample_tone_response_hdr( correlator: CorrelatorRemoteControl, diff --git a/qualification/baseline_correlation_products/__init__.py b/qualification/baseline_correlation_products/__init__.py index 87b7343fa..6e031999e 100644 --- a/qualification/baseline_correlation_products/__init__.py +++ b/qualification/baseline_correlation_products/__init__.py @@ -1,6 +1 @@ # noqa: D104 - -import pytest - -# Make the event_loop package-scoped -pytestmark = pytest.mark.asyncio(scope="package") diff --git a/qualification/conftest.py b/qualification/conftest.py index f0c95f8a0..7560d4cf4 100644 --- a/qualification/conftest.py +++ b/qualification/conftest.py @@ -171,6 +171,14 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: metafunc.parametrize("n_channels, narrowband_decimation", configs, indirect=True) +# Need to redefine this from pytest-asyncio to have it at package scope +@pytest.fixture(scope="package") +def event_loop(): # noqa: D103 + loop = asyncio.get_event_loop_policy().new_event_loop() + yield loop + loop.close() + + @pytest.fixture(scope="package") def n_antennas(request: pytest.FixtureRequest): # noqa: D401 """Number of antennas, i.e. size of the array.""" diff --git a/qualification/requirements.txt b/qualification/requirements.txt index 11087b115..de287937e 100644 --- a/qualification/requirements.txt +++ b/qualification/requirements.txt @@ -265,7 +265,7 @@ pytest==7.4.3 # pytest-check # pytest-custom-exit-code # pytest-reportlog -pytest-asyncio==0.23.2 +pytest-asyncio==0.21.1 # via # -c qualification/../requirements-dev.txt # katgpucbf (setup.cfg) diff --git a/requirements-dev.txt b/requirements-dev.txt index 30ab78cf9..0bc6cba72 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -275,7 +275,7 @@ pytest==7.4.3 # pytest-custom-exit-code # pytest-mock # pytest-xdist -pytest-asyncio==0.23.2 +pytest-asyncio==0.21.1 # via katgpucbf (setup.cfg) pytest-cov==4.1.0 # via -r requirements-dev.in diff --git a/setup.cfg b/setup.cfg index 3037bec52..4f339319a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -49,7 +49,7 @@ test = async-solipsism async-timeout pytest - pytest-asyncio>=0.18 + pytest-asyncio>=0.18,<0.22 pytest-custom_exit_code pytest-mock diff --git a/test/test_utils.py b/test/test_utils.py index 8c73c3d92..6d32a7c41 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -19,7 +19,7 @@ import asyncio import gc import weakref -from typing import AsyncGenerator +from typing import AsyncGenerator, Generator from unittest import mock import aiokatcp @@ -30,11 +30,6 @@ from katgpucbf.utils import DeviceStatusSensor, TimeConverter, TimeoutSensorStatusObserver -class AsyncSolipsismPolicy(asyncio.DefaultEventLoopPolicy): - def new_event_loop(self) -> async_solipsism.EventLoop: - return async_solipsism.EventLoop() - - class TestDeviceStatusSensor: """Test :class:`.DeviceStatusSensor`. @@ -80,9 +75,11 @@ class TestTimeoutSensorStatus: """Tests for :func:`katgpucbf.utils.timeout_sensor_status`.""" @pytest.fixture - def event_loop_policy(self) -> asyncio.AbstractEventLoopPolicy: + def event_loop(self) -> Generator[async_solipsism.EventLoop, None, None]: """Use async_solipsism event loop.""" - return AsyncSolipsismPolicy() + loop = async_solipsism.EventLoop() + yield loop + loop.close() @pytest.fixture def sensor(self) -> aiokatcp.Sensor: From 1ac2083d74750a2ba0fe274b17cfa439d6eb1139 Mon Sep 17 00:00:00 2001 From: Bruce Merry Date: Wed, 13 Dec 2023 09:11:19 +0200 Subject: [PATCH 4/4] Bump a copyright notice --- src/katgpucbf/dsim/send.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/katgpucbf/dsim/send.py b/src/katgpucbf/dsim/send.py index 7f6eb85f1..4a8d0f334 100644 --- a/src/katgpucbf/dsim/send.py +++ b/src/katgpucbf/dsim/send.py @@ -1,5 +1,5 @@ ################################################################################ -# Copyright (c) 2021-2022, National Research Foundation (SARAO) +# Copyright (c) 2021-2023, National Research Foundation (SARAO) # # Licensed under the BSD 3-Clause License (the "License"); you may not use # this file except in compliance with the License. You may obtain a copy