Diff of the two buildlogs: -- --- b1/build.log 2025-09-02 13:06:36.510319275 +0000 +++ b2/build.log 2025-09-02 13:41:52.116845340 +0000 @@ -1,6 +1,6 @@ I: pbuilder: network access will be disabled during build -I: Current time: Mon Oct 5 07:09:12 -12 2026 -I: pbuilder-time-stamp: 1791227352 +I: Current time: Wed Sep 3 03:06:39 +14 2025 +I: pbuilder-time-stamp: 1756818399 I: Building the build Environment I: extracting base tarball [/var/cache/pbuilder/forky-reproducible-base.tgz] I: copying local configuration @@ -44,53 +44,85 @@ dpkg-source: info: applying dask-matching-interpreter.patch I: Not using root during the build. I: Installing the build-deps -I: user script /srv/workspace/pbuilder/3739753/tmp/hooks/D02_print_environment starting +I: user script /srv/workspace/pbuilder/1909207/tmp/hooks/D01_modify_environment starting +debug: Running on codethink04-arm64. +I: Changing host+domainname to test build reproducibility +I: Adding a custom variable just for the fun of it... +I: Changing /bin/sh to bash +'/bin/sh' -> '/bin/bash' +lrwxrwxrwx 1 root root 9 Sep 2 13:06 /bin/sh -> /bin/bash +I: Setting pbuilder2's login shell to /bin/bash +I: Setting pbuilder2's GECOS to second user,second room,second work-phone,second home-phone,second other +I: user script /srv/workspace/pbuilder/1909207/tmp/hooks/D01_modify_environment finished +I: user script /srv/workspace/pbuilder/1909207/tmp/hooks/D02_print_environment starting I: set - BUILDDIR='/build/reproducible-path' - BUILDUSERGECOS='first user,first room,first work-phone,first home-phone,first other' - BUILDUSERNAME='pbuilder1' - BUILD_ARCH='arm64' - DEBIAN_FRONTEND='noninteractive' + BASH=/bin/sh + BASHOPTS=checkwinsize:cmdhist:complete_fullquote:extquote:force_fignore:globasciiranges:globskipdots:hostcomplete:interactive_comments:patsub_replacement:progcomp:promptvars:sourcepath + BASH_ALIASES=() + BASH_ARGC=() + BASH_ARGV=() + BASH_CMDS=() + BASH_LINENO=([0]="12" [1]="0") + BASH_LOADABLES_PATH=/usr/local/lib/bash:/usr/lib/bash:/opt/local/lib/bash:/usr/pkg/lib/bash:/opt/pkg/lib/bash:. + BASH_SOURCE=([0]="/tmp/hooks/D02_print_environment" [1]="/tmp/hooks/D02_print_environment") + BASH_VERSINFO=([0]="5" [1]="2" [2]="37" [3]="1" [4]="release" [5]="aarch64-unknown-linux-gnu") + BASH_VERSION='5.2.37(1)-release' + BUILDDIR=/build/reproducible-path + BUILDUSERGECOS='second user,second room,second work-phone,second home-phone,second other' + BUILDUSERNAME=pbuilder2 + BUILD_ARCH=arm64 + DEBIAN_FRONTEND=noninteractive DEB_BUILD_OPTIONS='buildinfo=+all reproducible=+all parallel=12 ' - DISTRIBUTION='forky' - HOME='/root' - HOST_ARCH='arm64' + DIRSTACK=() + DISTRIBUTION=forky + EUID=0 + FUNCNAME=([0]="Echo" [1]="main") + GROUPS=() + HOME=/root + HOSTNAME=i-capture-the-hostname + HOSTTYPE=aarch64 + HOST_ARCH=arm64 IFS=' ' - INVOCATION_ID='489385c8a313474d925d6226e98cd701' - LANG='C' - LANGUAGE='en_US:en' - LC_ALL='C' - MAIL='/var/mail/root' - OPTIND='1' - PATH='/usr/sbin:/usr/bin:/sbin:/bin:/usr/games' - PBCURRENTCOMMANDLINEOPERATION='build' - PBUILDER_OPERATION='build' - PBUILDER_PKGDATADIR='/usr/share/pbuilder' - PBUILDER_PKGLIBDIR='/usr/lib/pbuilder' - PBUILDER_SYSCONFDIR='/etc' - PPID='3739753' - PS1='# ' - PS2='> ' + INVOCATION_ID=e6f913cb7e0c469cacdd9d6fcce91ba1 + LANG=C + LANGUAGE=nl_BE:nl + LC_ALL=C + MACHTYPE=aarch64-unknown-linux-gnu + MAIL=/var/mail/root + OPTERR=1 + OPTIND=1 + OSTYPE=linux-gnu + PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path + PBCURRENTCOMMANDLINEOPERATION=build + PBUILDER_OPERATION=build + PBUILDER_PKGDATADIR=/usr/share/pbuilder + PBUILDER_PKGLIBDIR=/usr/lib/pbuilder + PBUILDER_SYSCONFDIR=/etc + PIPESTATUS=([0]="0") + POSIXLY_CORRECT=y + PPID=1909207 PS4='+ ' - PWD='/' - SHELL='/bin/bash' - SHLVL='2' - SUDO_COMMAND='/usr/bin/timeout -k 18.1h 18h /usr/bin/ionice -c 3 /usr/bin/nice /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/r-b-build.Fd0i0VrY/pbuilderrc_0qL1 --distribution forky --hookdir /etc/pbuilder/first-build-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/forky-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/r-b-build.Fd0i0VrY/b1 --logfile b1/build.log dask.distributed_2024.12.1+ds-1.dsc' - SUDO_GID='109' - SUDO_HOME='/var/lib/jenkins' - SUDO_UID='104' - SUDO_USER='jenkins' - TERM='unknown' - TZ='/usr/share/zoneinfo/Etc/GMT+12' - USER='root' - _='/usr/bin/systemd-run' - http_proxy='http://192.168.101.4:3128' + PWD=/ + SHELL=/bin/bash + SHELLOPTS=braceexpand:errexit:hashall:interactive-comments:posix + SHLVL=3 + SUDO_COMMAND='/usr/bin/timeout -k 24.1h 24h /usr/bin/ionice -c 3 /usr/bin/nice -n 11 /usr/bin/unshare --uts -- /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/r-b-build.Fd0i0VrY/pbuilderrc_qHR5 --distribution forky --hookdir /etc/pbuilder/rebuild-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/forky-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/r-b-build.Fd0i0VrY/b2 --logfile b2/build.log dask.distributed_2024.12.1+ds-1.dsc' + SUDO_GID=109 + SUDO_HOME=/var/lib/jenkins + SUDO_UID=104 + SUDO_USER=jenkins + TERM=unknown + TZ=/usr/share/zoneinfo/Etc/GMT-14 + UID=0 + USER=root + _='I: set' + http_proxy=http://192.168.101.4:3128 I: uname -a - Linux codethink03-arm64 6.12.41+deb13-cloud-arm64 #1 SMP Debian 6.12.41-1 (2025-08-12) aarch64 GNU/Linux + Linux i-capture-the-hostname 6.12.41+deb13-cloud-arm64 #1 SMP Debian 6.12.41-1 (2025-08-12) aarch64 GNU/Linux I: ls -l /bin - lrwxrwxrwx 1 root root 7 Aug 10 2025 /bin -> usr/bin -I: user script /srv/workspace/pbuilder/3739753/tmp/hooks/D02_print_environment finished + lrwxrwxrwx 1 root root 7 Aug 10 12:30 /bin -> usr/bin +I: user script /srv/workspace/pbuilder/1909207/tmp/hooks/D02_print_environment finished -> Attempting to satisfy build-dependencies -> Creating pbuilder-satisfydepends-dummy package Package: pbuilder-satisfydepends-dummy @@ -451,7 +483,7 @@ Get: 237 http://deb.debian.org/debian forky/main arm64 python3-zict all 3.0.0-4 [29.9 kB] Get: 238 http://deb.debian.org/debian forky/main arm64 tzdata-legacy all 2025b-5 [179 kB] Get: 239 http://deb.debian.org/debian forky/main arm64 uglifyjs all 3.17.4-2 [12.1 kB] -Fetched 137 MB in 2s (82.6 MB/s) +Fetched 137 MB in 1s (250 MB/s) Preconfiguring packages ... Selecting previously unselected package fonts-lato. (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 20011 files and directories currently installed.) @@ -1215,8 +1247,8 @@ Setting up tzdata (2025b-5) ... Current default time zone: 'Etc/UTC' -Local time is now: Mon Oct 5 19:09:58 UTC 2026. -Universal Time is now: Mon Oct 5 19:09:58 UTC 2026. +Local time is now: Tue Sep 2 13:07:14 UTC 2025. +Universal Time is now: Tue Sep 2 13:07:14 UTC 2025. Run 'dpkg-reconfigure tzdata' if you wish to change it. Setting up libpgm-5.3-0t64:arm64 (5.3.128~dfsg-2.1+b1) ... @@ -1447,7 +1479,11 @@ Building tag database... -> Finished parsing the build-deps I: Building the package -I: Running cd /build/reproducible-path/dask.distributed-2024.12.1+ds/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-buildpackage -us -uc -b && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-genchanges -S > ../dask.distributed_2024.12.1+ds-1_source.changes +I: user script /srv/workspace/pbuilder/1909207/tmp/hooks/A99_set_merged_usr starting +Not re-configuring usrmerge for forky +I: user script /srv/workspace/pbuilder/1909207/tmp/hooks/A99_set_merged_usr finished +hostname: Name or service not known +I: Running cd /build/reproducible-path/dask.distributed-2024.12.1+ds/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-buildpackage -us -uc -b && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-genchanges -S > ../dask.distributed_2024.12.1+ds-1_source.changes dpkg-buildpackage: info: source package dask.distributed dpkg-buildpackage: info: source version 2024.12.1+ds-1 dpkg-buildpackage: info: source distribution unstable @@ -1505,8 +1541,8 @@ ******************************************************************************** Please use a simple string containing a SPDX expression for `project.license`. You can also use `project.license-files`. (Both options available on setuptools>=77.0.0). - This deprecation is overdue, please update your project and remove deprecated - calls to avoid build errors in the future. + By 2026-Feb-18, you need to update your project and remove deprecated calls + or your builds will no longer be supported. See https://packaging.python.org/en/latest/guides/writing-pyproject-toml/#license for details. ******************************************************************************** @@ -2864,7 +2900,7 @@ Copying distributed.egg-info to build/bdist.linux-aarch64/wheel/./distributed-2024.12.1.egg-info running install_scripts creating build/bdist.linux-aarch64/wheel/distributed-2024.12.1.dist-info/WHEEL -creating '/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/.tmp-vz7g9cuf/distributed-2024.12.1-py3-none-any.whl' and adding 'build/bdist.linux-aarch64/wheel' to it +creating '/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/.tmp-ogn8hk6i/distributed-2024.12.1-py3-none-any.whl' and adding 'build/bdist.linux-aarch64/wheel' to it adding 'distributed/__init__.py' adding 'distributed/_async_taskgroup.py' adding 'distributed/_asyncio.py' @@ -5146,7 +5182,7 @@ distributed/tests/test_nanny.py::test_nanny_closes_cleanly PASSED [ 51%] distributed/tests/test_nanny.py::test_lifetime SKIPPED (need --runslow option to run) [ 51%] -distributed/tests/test_nanny.py::test_nanny_closes_cleanly_if_worker_is_terminated PASSED [ 51%] +distributed/tests/test_nanny.py::test_nanny_closes_cleanly_if_worker_is_terminated FAILED [ 51%] distributed/tests/test_nanny.py::test_config PASSED [ 51%] distributed/tests/test_nanny.py::test_nanny_port_range PASSED [ 51%] distributed/tests/test_nanny.py::test_nanny_closed_by_keyboard_interrupt[tcp] SKIPPED [ 51%] @@ -6019,7 +6055,7 @@ distributed/tests/test_steal.py::test_balance[but don't move too many] PASSED [ 78%] distributed/tests/test_steal.py::test_balance[no one clearly saturated] PASSED [ 78%] distributed/tests/test_steal.py::test_balance[balance multiple saturated workers] PASSED [ 78%] -distributed/tests/test_steal.py::test_restart PASSED [ 78%] +distributed/tests/test_steal.py::test_restart FAILED [ 78%] distributed/tests/test_steal.py::test_paused_workers_must_not_steal PASSED [ 78%] distributed/tests/test_steal.py::test_dont_steal_already_released PASSED [ 78%] distributed/tests/test_steal.py::test_dont_steal_long_running_tasks PASSED [ 78%] @@ -6098,13 +6134,13 @@ distributed/tests/test_tls_functional.py::test_client_submit PASSED [ 81%] distributed/tests/test_tls_functional.py::test_gather PASSED [ 81%] distributed/tests/test_tls_functional.py::test_scatter PASSED [ 81%] -distributed/tests/test_tls_functional.py::test_nanny PASSED [ 81%] +distributed/tests/test_tls_functional.py::test_nanny FAILED [ 81%] distributed/tests/test_tls_functional.py::test_rebalance PASSED [ 81%] distributed/tests/test_tls_functional.py::test_work_stealing PASSED [ 81%] distributed/tests/test_tls_functional.py::test_worker_client PASSED [ 81%] distributed/tests/test_tls_functional.py::test_worker_client_gather PASSED [ 81%] distributed/tests/test_tls_functional.py::test_worker_client_executor PASSED [ 81%] -distributed/tests/test_tls_functional.py::test_retire_workers PASSED [ 81%] +distributed/tests/test_tls_functional.py::test_retire_workers FAILED [ 81%] distributed/tests/test_tls_functional.py::test_security_dict_input PASSED [ 81%] distributed/tests/test_utils.py::test_All PASSED [ 81%] distributed/tests/test_utils.py::test_sync PASSED [ 81%] @@ -6710,27 +6746,3225 @@ distributed/tests/test_worker_state_machine.py::test_remove_worker_while_in_fetch PASSED [ 99%] distributed/tests/test_worker_state_machine.py::test_remove_worker_unknown PASSED [100%] +=================================== FAILURES =================================== +______________ test_nanny_closes_cleanly_if_worker_is_terminated _______________ + +c = +s = + + @gen_cluster(client=True, nthreads=[]) + async def test_nanny_closes_cleanly_if_worker_is_terminated(c, s): + async with Nanny(s.address) as n: + async with c.rpc(n.worker_address) as w: + IOLoop.current().add_callback(w.terminate) + start = time() + while n.status != Status.closed: + await asyncio.sleep(0.01) +> assert time() < start + 5 +E assert 1756819390.6073353 < (1756819383.7670298 + 5) +E + where 1756819390.6073353 = time() + +distributed/tests/test_nanny.py:520: AssertionError +----------------------------- Captured stderr call ----------------------------- +2025-09-03 03:23:01,197 - distributed.scheduler - INFO - State start +2025-09-03 03:23:01,206 - distributed.scheduler - INFO - Scheduler at: tcp://127.0.0.1:37553 +2025-09-03 03:23:01,207 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:38509/status +2025-09-03 03:23:01,212 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:23:01,251 - distributed.scheduler - INFO - Receive client connection: Client-f305e53c-87ff-11f0-beb6-4171fc750539 +2025-09-03 03:23:01,285 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:54108 +2025-09-03 03:23:01,299 - distributed.nanny - INFO - Start Nanny at: 'tcp://127.0.0.1:34445' +2025-09-03 03:23:02,354 - distributed.worker - INFO - Start worker at: tcp://127.0.0.1:33535 +2025-09-03 03:23:02,354 - distributed.worker - INFO - Listening to: tcp://127.0.0.1:33535 +2025-09-03 03:23:02,355 - distributed.worker - INFO - dashboard at: 127.0.0.1:42187 +2025-09-03 03:23:02,355 - distributed.worker - INFO - Waiting to connect to: tcp://127.0.0.1:37553 +2025-09-03 03:23:02,355 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:23:02,355 - distributed.worker - INFO - Threads: 12 +2025-09-03 03:23:02,355 - distributed.worker - INFO - Memory: 31.29 GiB +2025-09-03 03:23:02,355 - distributed.worker - INFO - Local Directory: /tmp/dask-scratch-space/worker-p7d59vb3 +2025-09-03 03:23:02,355 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:23:03,741 - distributed.scheduler - INFO - Register worker addr: tcp://127.0.0.1:33535 name: tcp://127.0.0.1:33535 +2025-09-03 03:23:03,754 - distributed.scheduler - INFO - Starting worker compute stream, tcp://127.0.0.1:33535 +2025-09-03 03:23:03,756 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:54150 +2025-09-03 03:23:03,757 - distributed.worker - INFO - Starting Worker plugin shuffle +2025-09-03 03:23:03,757 - distributed.worker - INFO - Registered to: tcp://127.0.0.1:37553 +2025-09-03 03:23:03,757 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:23:03,758 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:37553 +2025-09-03 03:23:03,769 - distributed.worker - INFO - Stopping worker at tcp://127.0.0.1:33535. Reason: worker-close +2025-09-03 03:23:03,771 - distributed.nanny - INFO - Closing Nanny gracefully at 'tcp://127.0.0.1:34445'. Reason: worker-close +2025-09-03 03:23:03,772 - distributed.worker - INFO - Removing Worker plugin shuffle +2025-09-03 03:23:03,773 - distributed.core - INFO - Connection to tcp://127.0.0.1:37553 has been closed. +2025-09-03 03:23:03,773 - distributed.core - INFO - Received 'close-stream' from tcp://127.0.0.1:54150; closing. +2025-09-03 03:23:03,774 - distributed.scheduler - INFO - Remove worker addr: tcp://127.0.0.1:33535 name: tcp://127.0.0.1:33535 (stimulus_id='handle-worker-cleanup-1756819383.7740333') +2025-09-03 03:23:03,775 - distributed.scheduler - INFO - Lost all workers +2025-09-03 03:23:03,777 - distributed.nanny - INFO - Worker closed +2025-09-03 03:23:03,778 - tornado.application - ERROR - Exception in callback functools.partial(>, .send_recv_from_rpc() done, defined at /build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py:1251> exception=CommClosedError('in : Stream is closed')>) +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 225, in read + frames_nosplit_nbytes_bin = await stream.read_bytes(fmt_size) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +tornado.iostream.StreamClosedError: Stream is closed + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/usr/lib/python3/dist-packages/tornado/ioloop.py", line 750, in _run_callback + ret = callback() + File "/usr/lib/python3/dist-packages/tornado/ioloop.py", line 774, in _discard_future_result + future.result() + ~~~~~~~~~~~~~^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1259, in send_recv_from_rpc + return await send_recv(comm=comm, op=key, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1018, in send_recv + response = await comm.read(deserializers=deserializers) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 236, in read + convert_stream_closed_error(self, e) + ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 142, in convert_stream_closed_error + raise CommClosedError(f"in {obj}: {exc}") from exc +distributed.comm.core.CommClosedError: in : Stream is closed +2025-09-03 03:23:10,607 - distributed.nanny - INFO - Closing Nanny at 'tcp://127.0.0.1:34445'. Reason: nanny-close +2025-09-03 03:23:10,607 - distributed.nanny - INFO - Nanny asking worker to close. Reason: nanny-close +2025-09-03 03:23:10,608 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 6.89s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:23:10,609 - distributed.core - INFO - Event loop was unresponsive in Nanny for 6.84s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:23:11,037 - distributed.nanny - INFO - Nanny at 'tcp://127.0.0.1:34445' closed. +2025-09-03 03:23:11,038 - distributed.scheduler - INFO - Remove client Client-f305e53c-87ff-11f0-beb6-4171fc750539 +2025-09-03 03:23:11,039 - distributed.core - INFO - Received 'close-stream' from tcp://127.0.0.1:54108; closing. +2025-09-03 03:23:11,039 - distributed.scheduler - INFO - Remove client Client-f305e53c-87ff-11f0-beb6-4171fc750539 +2025-09-03 03:23:11,045 - distributed.scheduler - INFO - Close client connection: Client-f305e53c-87ff-11f0-beb6-4171fc750539 +2025-09-03 03:23:11,047 - distributed.scheduler - INFO - Closing scheduler. Reason: unknown +2025-09-03 03:23:11,053 - distributed.scheduler - INFO - Scheduler closing all comms +_________________________________ test_restart _________________________________ + +args = (), kwds = {} + + @wraps(func) + def inner(*args, **kwds): + with self._recreate_cm(): +> return func(*args, **kwds) + +/usr/lib/python3.13/contextlib.py:85: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +/usr/lib/python3.13/contextlib.py:84: in inner + with self._recreate_cm(): +/usr/lib/python3.13/contextlib.py:148: in __exit__ + next(self.gen) +distributed/utils_test.py:1786: in clean + with check_instances() if instances else nullcontext(): +/usr/lib/python3.13/contextlib.py:148: in __exit__ + next(self.gen) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + + @contextmanager + def check_instances(): + Client._instances.clear() + Worker._instances.clear() + Scheduler._instances.clear() + SpecCluster._instances.clear() + Worker._initialized_clients.clear() + SchedulerTaskState._instances.clear() + WorkerTaskState._instances.clear() + Nanny._instances.clear() + _global_clients.clear() + Comm._instances.clear() + + yield + + start = time() + while set(_global_clients): + sleep(0.1) + assert time() < start + 10 + + _global_clients.clear() + + for w in Worker._instances: + with suppress(RuntimeError): # closed IOLoop + w.loop.add_callback(w.close, executor_wait=False) + if w.status in WORKER_ANY_RUNNING: + w.loop.add_callback(w.close) + Worker._instances.clear() + + start = time() + while any(c.status != "closed" for c in Worker._initialized_clients): + sleep(0.1) + assert time() < start + 10 + Worker._initialized_clients.clear() + + for _ in range(5): + if all(c.closed() for c in Comm._instances): + break + else: + sleep(0.1) + else: + L = [c for c in Comm._instances if not c.closed()] + Comm._instances.clear() + raise ValueError("Unclosed Comms", L) + +> assert all( + n.status in {Status.closed, Status.init, Status.failed} + for n in Nanny._instances + ), {n: n.status for n in Nanny._instances} +E AssertionError: {: , : } +E assert False +E + where False = all(. at 0xffff3d184ba0>) + +distributed/utils_test.py:1740: AssertionError +----------------------------- Captured stdout call ----------------------------- +Unclosed Comms: [] +----------------------------- Captured stderr call ----------------------------- +2025-09-03 03:30:23,477 - distributed.scheduler - INFO - State start +2025-09-03 03:30:23,490 - distributed.scheduler - INFO - Scheduler at: tcp://127.0.0.1:34055 +2025-09-03 03:30:23,504 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:43399/status +2025-09-03 03:30:23,506 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:30:23,550 - distributed.nanny - INFO - Start Nanny at: 'tcp://127.0.0.1:35571' +2025-09-03 03:30:23,573 - distributed.nanny - INFO - Start Nanny at: 'tcp://127.0.0.1:43073' +2025-09-03 03:30:25,528 - distributed.worker - INFO - Start worker at: tcp://127.0.0.1:37243 +2025-09-03 03:30:25,528 - distributed.worker - INFO - Listening to: tcp://127.0.0.1:37243 +2025-09-03 03:30:25,528 - distributed.worker - INFO - Worker name: 0 +2025-09-03 03:30:25,540 - distributed.worker - INFO - dashboard at: 127.0.0.1:35759 +2025-09-03 03:30:25,540 - distributed.worker - INFO - Waiting to connect to: tcp://127.0.0.1:34055 +2025-09-03 03:30:25,540 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:30:25,540 - distributed.worker - INFO - Threads: 1 +2025-09-03 03:30:25,540 - distributed.worker - INFO - Memory: 31.29 GiB +2025-09-03 03:30:25,540 - distributed.worker - INFO - Local Directory: /tmp/dask-scratch-space/worker-u1et9tql +2025-09-03 03:30:25,541 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:30:25,987 - distributed.worker - INFO - Start worker at: tcp://127.0.0.1:35975 +2025-09-03 03:30:25,988 - distributed.worker - INFO - Listening to: tcp://127.0.0.1:35975 +2025-09-03 03:30:25,988 - distributed.worker - INFO - Worker name: 1 +2025-09-03 03:30:25,988 - distributed.worker - INFO - dashboard at: 127.0.0.1:35375 +2025-09-03 03:30:25,988 - distributed.worker - INFO - Waiting to connect to: tcp://127.0.0.1:34055 +2025-09-03 03:30:25,992 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:30:25,992 - distributed.worker - INFO - Threads: 1 +2025-09-03 03:30:25,992 - distributed.worker - INFO - Memory: 31.29 GiB +2025-09-03 03:30:25,992 - distributed.worker - INFO - Local Directory: /tmp/dask-scratch-space/worker-31ucsc0m +2025-09-03 03:30:25,992 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:30:26,442 - distributed.scheduler - INFO - Register worker addr: tcp://127.0.0.1:37243 name: 0 +2025-09-03 03:30:26,466 - distributed.worker - INFO - Starting Worker plugin shuffle +2025-09-03 03:30:26,467 - distributed.worker - INFO - Registered to: tcp://127.0.0.1:34055 +2025-09-03 03:30:26,467 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:30:26,476 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:34055 +2025-09-03 03:30:26,468 - distributed.scheduler - INFO - Starting worker compute stream, tcp://127.0.0.1:37243 +2025-09-03 03:30:26,477 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:44160 +2025-09-03 03:30:26,963 - distributed.scheduler - INFO - Register worker addr: tcp://127.0.0.1:35975 name: 1 +2025-09-03 03:30:27,033 - distributed.worker - INFO - Starting Worker plugin shuffle +2025-09-03 03:30:27,034 - distributed.worker - INFO - Registered to: tcp://127.0.0.1:34055 +2025-09-03 03:30:27,034 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:30:27,037 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:34055 +2025-09-03 03:30:27,040 - distributed.scheduler - INFO - Starting worker compute stream, tcp://127.0.0.1:35975 +2025-09-03 03:30:27,042 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:44168 +2025-09-03 03:30:27,134 - distributed.scheduler - INFO - Receive client connection: Client-fcc3ad17-8800-11f0-beb6-4171fc750539 +2025-09-03 03:30:27,203 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:44172 +2025-09-03 03:30:30,078 - distributed.core - INFO - Event loop was unresponsive in Worker for 3.11s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:30:30,287 - distributed.scheduler - INFO - Restarting workers and releasing all keys (stimulus_id='client-restart-1756819830.285763') +2025-09-03 03:30:30,307 - distributed.scheduler - INFO - Restarting all workers (stimulus_id='client-restart-1756819830.285763' +2025-09-03 03:30:30,331 - distributed.nanny - INFO - Nanny asking worker to close. Reason: client-restart-1756819830.285763 +2025-09-03 03:30:30,331 - distributed.nanny - INFO - Nanny asking worker to close. Reason: client-restart-1756819830.285763 +2025-09-03 03:30:30,344 - distributed.worker - INFO - Stopping worker at tcp://127.0.0.1:37243. Reason: client-restart-1756819830.285763 +2025-09-03 03:30:30,345 - distributed.worker - INFO - Removing Worker plugin shuffle +2025-09-03 03:30:30,346 - distributed.core - INFO - Connection to tcp://127.0.0.1:34055 has been closed. +2025-09-03 03:30:30,348 - distributed.core - INFO - Received 'close-stream' from tcp://127.0.0.1:44160; closing. +2025-09-03 03:30:30,349 - distributed.nanny - INFO - Worker closed +2025-09-03 03:30:30,348 - distributed.scheduler - INFO - Remove worker addr: tcp://127.0.0.1:37243 name: 0 (stimulus_id='handle-worker-cleanup-1756819830.348812') +2025-09-03 03:30:30,378 - distributed.worker - INFO - Stopping worker at tcp://127.0.0.1:35975. Reason: client-restart-1756819830.285763 +2025-09-03 03:30:30,379 - distributed.core - INFO - Event loop was unresponsive in Worker for 3.35s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:30:30,385 - distributed.worker.state_machine - WARNING - Async instruction for > ended with CancelledError +2025-09-03 03:30:30,386 - distributed.worker - INFO - Removing Worker plugin shuffle +2025-09-03 03:30:30,387 - distributed.worker - ERROR - Failed to communicate with scheduler during heartbeat. +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 225, in read + frames_nosplit_nbytes_bin = await stream.read_bytes(fmt_size) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +tornado.iostream.StreamClosedError: Stream is closed + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/worker.py", line 1269, in heartbeat + response = await retry_operation( + ^^^^^^^^^^^^^^^^^^^^^^ + ...<14 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_comm.py", line 441, in retry_operation + return await retry( + ^^^^^^^^^^^^ + ...<5 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_comm.py", line 420, in retry + return await coro() + ^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1259, in send_recv_from_rpc + return await send_recv(comm=comm, op=key, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1018, in send_recv + response = await comm.read(deserializers=deserializers) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 236, in read + convert_stream_closed_error(self, e) + ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 142, in convert_stream_closed_error + raise CommClosedError(f"in {obj}: {exc}") from exc +distributed.comm.core.CommClosedError: in : Stream is closed +2025-09-03 03:30:30,394 - distributed.core - INFO - Connection to tcp://127.0.0.1:34055 has been closed. +2025-09-03 03:30:30,400 - distributed.core - INFO - Received 'close-stream' from tcp://127.0.0.1:44168; closing. +2025-09-03 03:30:30,401 - distributed.scheduler - INFO - Remove worker addr: tcp://127.0.0.1:35975 name: 1 (stimulus_id='handle-worker-cleanup-1756819830.4010656') +2025-09-03 03:30:30,403 - distributed.scheduler - INFO - Lost all workers +2025-09-03 03:30:30,413 - distributed.nanny - INFO - Worker closed +2025-09-03 03:30:31,335 - distributed.nanny - WARNING - Restarting worker +2025-09-03 03:30:31,431 - distributed.nanny - WARNING - Restarting worker +2025-09-03 03:30:33,020 - distributed.worker - INFO - Start worker at: tcp://127.0.0.1:41547 +2025-09-03 03:30:33,024 - distributed.worker - INFO - Listening to: tcp://127.0.0.1:41547 +2025-09-03 03:30:33,024 - distributed.worker - INFO - Worker name: 0 +2025-09-03 03:30:33,024 - distributed.worker - INFO - dashboard at: 127.0.0.1:37115 +2025-09-03 03:30:33,024 - distributed.worker - INFO - Waiting to connect to: tcp://127.0.0.1:34055 +2025-09-03 03:30:33,024 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:30:33,024 - distributed.worker - INFO - Threads: 1 +2025-09-03 03:30:33,025 - distributed.worker - INFO - Memory: 31.29 GiB +2025-09-03 03:30:33,025 - distributed.worker - INFO - Local Directory: /tmp/dask-scratch-space/worker-goez1j2b +2025-09-03 03:30:33,025 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:30:33,538 - distributed.worker - INFO - Start worker at: tcp://127.0.0.1:37673 +2025-09-03 03:30:33,538 - distributed.worker - INFO - Listening to: tcp://127.0.0.1:37673 +2025-09-03 03:30:33,538 - distributed.worker - INFO - Worker name: 1 +2025-09-03 03:30:33,538 - distributed.worker - INFO - dashboard at: 127.0.0.1:44121 +2025-09-03 03:30:33,538 - distributed.worker - INFO - Waiting to connect to: tcp://127.0.0.1:34055 +2025-09-03 03:30:33,538 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:30:33,538 - distributed.worker - INFO - Threads: 1 +2025-09-03 03:30:33,538 - distributed.worker - INFO - Memory: 31.29 GiB +2025-09-03 03:30:33,538 - distributed.worker - INFO - Local Directory: /tmp/dask-scratch-space/worker-lzevtmnd +2025-09-03 03:30:33,538 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:30:33,896 - distributed.scheduler - INFO - Register worker addr: tcp://127.0.0.1:41547 name: 0 +2025-09-03 03:30:33,961 - distributed.scheduler - INFO - Starting worker compute stream, tcp://127.0.0.1:41547 +2025-09-03 03:30:33,963 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:33790 +2025-09-03 03:30:33,969 - distributed.worker - INFO - Starting Worker plugin shuffle +2025-09-03 03:30:33,969 - distributed.worker - INFO - Registered to: tcp://127.0.0.1:34055 +2025-09-03 03:30:33,970 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:30:33,984 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:34055 +2025-09-03 03:30:35,069 - distributed.scheduler - INFO - Register worker addr: tcp://127.0.0.1:37673 name: 1 +2025-09-03 03:30:35,149 - distributed.worker - INFO - Starting Worker plugin shuffle +2025-09-03 03:30:35,150 - distributed.worker - INFO - Registered to: tcp://127.0.0.1:34055 +2025-09-03 03:30:35,150 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:30:35,150 - distributed.scheduler - INFO - Starting worker compute stream, tcp://127.0.0.1:37673 +2025-09-03 03:30:35,152 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:33796 +2025-09-03 03:30:35,152 - distributed.core - INFO - Starting established connection to tcp://127.0.0.1:34055 +2025-09-03 03:30:35,157 - distributed.scheduler - INFO - Workers restart finished (stimulus_id='client-restart-1756819830.285763' +2025-09-03 03:30:35,173 - distributed.scheduler - INFO - Remove client Client-fcc3ad17-8800-11f0-beb6-4171fc750539 +2025-09-03 03:30:35,175 - distributed.core - INFO - Received 'close-stream' from tcp://127.0.0.1:44172; closing. +2025-09-03 03:30:35,175 - distributed.scheduler - INFO - Remove client Client-fcc3ad17-8800-11f0-beb6-4171fc750539 +2025-09-03 03:30:35,190 - distributed.scheduler - INFO - Close client connection: Client-fcc3ad17-8800-11f0-beb6-4171fc750539 +2025-09-03 03:30:35,205 - distributed.nanny - INFO - Closing Nanny at 'tcp://127.0.0.1:35571'. Reason: nanny-close +2025-09-03 03:30:35,205 - distributed.nanny - INFO - Nanny asking worker to close. Reason: nanny-close +2025-09-03 03:30:35,205 - distributed.nanny - INFO - Closing Nanny at 'tcp://127.0.0.1:43073'. Reason: nanny-close +2025-09-03 03:30:35,206 - distributed.nanny - INFO - Nanny asking worker to close. Reason: nanny-close +2025-09-03 03:30:35,212 - distributed.worker - INFO - Stopping worker at tcp://127.0.0.1:41547. Reason: nanny-close +2025-09-03 03:30:35,213 - distributed.worker - INFO - Removing Worker plugin shuffle +2025-09-03 03:30:35,214 - distributed.core - INFO - Connection to tcp://127.0.0.1:34055 has been closed. +2025-09-03 03:30:35,215 - distributed.core - INFO - Received 'close-stream' from tcp://127.0.0.1:33790; closing. +2025-09-03 03:30:35,215 - distributed.scheduler - INFO - Remove worker addr: tcp://127.0.0.1:41547 name: 0 (stimulus_id='handle-worker-cleanup-1756819835.2154458') +2025-09-03 03:30:35,219 - distributed.worker - INFO - Stopping worker at tcp://127.0.0.1:37673. Reason: nanny-close +2025-09-03 03:30:35,219 - distributed.worker - INFO - Removing Worker plugin shuffle +2025-09-03 03:30:35,220 - distributed.core - INFO - Connection to tcp://127.0.0.1:34055 has been closed. +2025-09-03 03:30:35,229 - distributed.core - INFO - Received 'close-stream' from tcp://127.0.0.1:33796; closing. +2025-09-03 03:30:35,229 - distributed.scheduler - INFO - Remove worker addr: tcp://127.0.0.1:37673 name: 1 (stimulus_id='handle-worker-cleanup-1756819835.2294338') +2025-09-03 03:30:35,233 - distributed.nanny - INFO - Worker closed +2025-09-03 03:30:35,233 - distributed.nanny - INFO - Worker closed +2025-09-03 03:30:35,231 - distributed.scheduler - INFO - Lost all workers +2025-09-03 03:30:41,761 - distributed.core - INFO - Event loop was unresponsive in Nanny for 6.78s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:30:41,762 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 6.72s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:30:41,762 - distributed.core - INFO - Event loop was unresponsive in Nanny for 6.71s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:30:41,762 - distributed.nanny - WARNING - Worker process still alive after 4.0 seconds, killing +2025-09-03 03:30:41,763 - distributed.nanny - WARNING - Worker process still alive after 4.0 seconds, killing +2025-09-03 03:30:41,804 - distributed.nanny - INFO - Worker process 2265611 was killed by signal 9 +2025-09-03 03:30:42,212 - distributed.nanny - INFO - Worker process 2265638 was killed by signal 9 +2025-09-03 03:30:42,213 - distributed.nanny - INFO - Nanny at 'tcp://127.0.0.1:43073' closed. +2025-09-03 03:30:42,213 - distributed.scheduler - INFO - Closing scheduler. Reason: unknown +2025-09-03 03:30:42,216 - distributed.scheduler - INFO - Scheduler closing all comms +__________________________________ test_nanny __________________________________ + +fut = , timeout = 0 + + async def wait_for(fut: Awaitable[T], timeout: float) -> T: + async with asyncio.timeout(timeout): +> return await fut + +distributed/utils.py:1914: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +distributed/nanny.py:358: in start_unsafe + comm = await self.rpc.connect(saddr) +distributed/core.py:1485: in connect + return await self._connect(addr=addr, timeout=timeout) +distributed/core.py:1429: in _connect + comm = await connect( +distributed/comm/core.py:342: in connect + comm = await wait_for( +distributed/utils.py:1914: in wait_for + return await fut +distributed/comm/tcp.py:546: in connect + stream = await self.client.connect( +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +host = '127.0.0.1', port = 42791, af = +ssl_options = +max_buffer_size = 16798425088.0, source_ip = None, source_port = None +timeout = None + + async def connect( + self, + host: str, + port: int, + af: socket.AddressFamily = socket.AF_UNSPEC, + ssl_options: Optional[Union[Dict[str, Any], ssl.SSLContext]] = None, + max_buffer_size: Optional[int] = None, + source_ip: Optional[str] = None, + source_port: Optional[int] = None, + timeout: Optional[Union[float, datetime.timedelta]] = None, + ) -> IOStream: + """Connect to the given host and port. + + Asynchronously returns an `.IOStream` (or `.SSLIOStream` if + ``ssl_options`` is not None). + + Using the ``source_ip`` kwarg, one can specify the source + IP address to use when establishing the connection. + In case the user needs to resolve and + use a specific interface, it has to be handled outside + of Tornado as this depends very much on the platform. + + Raises `TimeoutError` if the input future does not complete before + ``timeout``, which may be specified in any form allowed by + `.IOLoop.add_timeout` (i.e. a `datetime.timedelta` or an absolute time + relative to `.IOLoop.time`) + + Similarly, when the user requires a certain source port, it can + be specified using the ``source_port`` arg. + + .. versionchanged:: 4.5 + Added the ``source_ip`` and ``source_port`` arguments. + + .. versionchanged:: 5.0 + Added the ``timeout`` argument. + """ + if timeout is not None: + if isinstance(timeout, numbers.Real): + timeout = IOLoop.current().time() + timeout + elif isinstance(timeout, datetime.timedelta): + timeout = IOLoop.current().time() + timeout.total_seconds() + else: + raise TypeError("Unsupported timeout %r" % timeout) + if timeout is not None: + addrinfo = await gen.with_timeout( + timeout, self.resolver.resolve(host, port, af) + ) + else: + addrinfo = await self.resolver.resolve(host, port, af) + connector = _Connector( + addrinfo, + functools.partial( + self._create_stream, + max_buffer_size, + source_ip=source_ip, + source_port=source_port, + ), + ) +> af, addr, stream = await connector.start(connect_timeout=timeout) +E asyncio.exceptions.CancelledError + +/usr/lib/python3/dist-packages/tornado/tcpclient.py:279: CancelledError + +The above exception was the direct cause of the following exception: + +self = + + @final + async def start(self): + async with self._startup_lock: + if self.status == Status.failed: + assert self.__startup_exc is not None + raise self.__startup_exc + elif self.status != Status.init: + return self + timeout = getattr(self, "death_timeout", None) + + async def _close_on_failure(exc: Exception) -> None: + await self.close(reason=f"failure-to-start-{str(type(exc))}") + self.status = Status.failed + self.__startup_exc = exc + + try: +> await wait_for(self.start_unsafe(), timeout=timeout) + +distributed/core.py:528: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +distributed/utils.py:1913: in wait_for + async with asyncio.timeout(timeout): +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +exc_type = +exc_val = CancelledError(), exc_tb = + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> Optional[bool]: + assert self._state in (_State.ENTERED, _State.EXPIRING) + + if self._timeout_handler is not None: + self._timeout_handler.cancel() + self._timeout_handler = None + + if self._state is _State.EXPIRING: + self._state = _State.EXPIRED + + if self._task.uncancel() <= self._cancelling and exc_type is not None: + # Since there are no new cancel requests, we're + # handling this. + if issubclass(exc_type, exceptions.CancelledError): +> raise TimeoutError from exc_val +E TimeoutError + +/usr/lib/python3.13/asyncio/timeouts.py:116: TimeoutError + +The above exception was the direct cause of the following exception: + + @contextlib.asynccontextmanager + async def _cluster_factory(): + workers = [] + s = None + try: + for _ in range(60): + try: +> s, ws = await start_cluster( + nthreads, + scheduler, + security=security, + Worker=Worker, + scheduler_kwargs=scheduler_kwargs, + worker_kwargs=merge( + {"death_timeout": min(15, int(deadline.remaining))}, + worker_kwargs, + ), + ) + +distributed/utils_test.py:974: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +distributed/utils_test.py:791: in start_cluster + await asyncio.gather(*workers) +/usr/lib/python3.13/asyncio/tasks.py:737: in _wrap_awaitable + return await awaitable +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = + + @final + async def start(self): + async with self._startup_lock: + if self.status == Status.failed: + assert self.__startup_exc is not None + raise self.__startup_exc + elif self.status != Status.init: + return self + timeout = getattr(self, "death_timeout", None) + + async def _close_on_failure(exc: Exception) -> None: + await self.close(reason=f"failure-to-start-{str(type(exc))}") + self.status = Status.failed + self.__startup_exc = exc + + try: + await wait_for(self.start_unsafe(), timeout=timeout) + except asyncio.TimeoutError as exc: + await _close_on_failure(exc) +> raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +E TimeoutError: Nanny start timed out after 0s. + +distributed/core.py:531: TimeoutError + +During handling of the above exception, another exception occurred: + +fut = ._..test_func..async_fn at 0xffff80727a00> +timeout = 60 + + async def wait_for(fut: Awaitable[T], timeout: float) -> T: + async with asyncio.timeout(timeout): +> return await fut + +distributed/utils.py:1914: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +distributed/utils_test.py:1007: in async_fn + _cluster_factory() as (s, workers), +/usr/lib/python3.13/contextlib.py:214: in __aenter__ + return await anext(self.gen) +distributed/utils_test.py:991: in _cluster_factory + await asyncio.sleep(1) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +delay = 1, result = None + + async def sleep(delay, result=None): + """Coroutine that completes after a given time (in seconds).""" + if delay <= 0: + await __sleep0() + return result + + if math.isnan(delay): + raise ValueError("Invalid delay: NaN (not a number)") + + loop = events.get_running_loop() + future = loop.create_future() + h = loop.call_later(delay, + futures._set_result_unless_cancelled, + future, result) + try: +> return await future +E asyncio.exceptions.CancelledError + +/usr/lib/python3.13/asyncio/tasks.py:718: CancelledError + +The above exception was the direct cause of the following exception: + +args = (), kwds = {} + + @wraps(func) + def inner(*args, **kwds): + with self._recreate_cm(): +> return func(*args, **kwds) + +/usr/lib/python3.13/contextlib.py:85: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +/usr/lib/python3.13/contextlib.py:85: in inner + return func(*args, **kwds) +distributed/utils_test.py:1090: in test_func + return _run_and_close_tornado(async_fn_outer) +distributed/utils_test.py:380: in _run_and_close_tornado + return asyncio_run(inner_fn(), loop_factory=get_loop_factory()) +/usr/lib/python3.13/asyncio/runners.py:195: in run + return runner.run(main) +/usr/lib/python3.13/asyncio/runners.py:118: in run + return self._loop.run_until_complete(task) +/usr/lib/python3.13/asyncio/base_events.py:725: in run_until_complete + return future.result() +distributed/utils_test.py:377: in inner_fn + return await async_fn(*args, **kwargs) +distributed/utils_test.py:1087: in async_fn_outer + return await utils_wait_for(async_fn(), timeout=timeout * 2) +distributed/utils.py:1913: in wait_for + async with asyncio.timeout(timeout): +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +exc_type = +exc_val = CancelledError(), exc_tb = + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> Optional[bool]: + assert self._state in (_State.ENTERED, _State.EXPIRING) + + if self._timeout_handler is not None: + self._timeout_handler.cancel() + self._timeout_handler = None + + if self._state is _State.EXPIRING: + self._state = _State.EXPIRED + + if self._task.uncancel() <= self._cancelling and exc_type is not None: + # Since there are no new cancel requests, we're + # handling this. + if issubclass(exc_type, exceptions.CancelledError): +> raise TimeoutError from exc_val +E TimeoutError + +/usr/lib/python3.13/asyncio/timeouts.py:116: TimeoutError +----------------------------- Captured stderr call ----------------------------- +2025-09-03 03:32:30,390 - distributed.scheduler - INFO - State start +2025-09-03 03:32:30,400 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:43613 +2025-09-03 03:32:30,413 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:38169/status +2025-09-03 03:32:30,415 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:32:30,497 - distributed.nanny - INFO - Start Nanny at: 'tls://127.0.0.1:43253' +2025-09-03 03:32:30,517 - distributed.nanny - INFO - Start Nanny at: 'tls://127.0.0.1:41233' +2025-09-03 03:32:32,611 - distributed.worker.memory - WARNING - Ignoring provided memory limit 33596850176 due to system memory limit of 31.29 GiB +2025-09-03 03:32:32,690 - distributed.worker - INFO - Start worker at: tls://127.0.0.1:43809 +2025-09-03 03:32:32,690 - distributed.worker - INFO - Listening to: tls://127.0.0.1:43809 +2025-09-03 03:32:32,690 - distributed.worker - INFO - Worker name: 1 +2025-09-03 03:32:32,690 - distributed.worker - INFO - dashboard at: 127.0.0.1:43585 +2025-09-03 03:32:32,690 - distributed.worker - INFO - Waiting to connect to: tls://127.0.0.1:43613 +2025-09-03 03:32:32,690 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:32:32,690 - distributed.worker - INFO - Threads: 2 +2025-09-03 03:32:32,691 - distributed.worker - INFO - Memory: 31.29 GiB +2025-09-03 03:32:32,691 - distributed.worker - INFO - Local Directory: /tmp/dask-scratch-space/worker-euwqvrcp +2025-09-03 03:32:32,691 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:32:32,905 - distributed.worker.memory - WARNING - Ignoring provided memory limit 33596850176 due to system memory limit of 31.29 GiB +2025-09-03 03:32:32,935 - distributed.worker - INFO - Start worker at: tls://127.0.0.1:43081 +2025-09-03 03:32:32,935 - distributed.worker - INFO - Listening to: tls://127.0.0.1:43081 +2025-09-03 03:32:32,935 - distributed.worker - INFO - Worker name: 0 +2025-09-03 03:32:32,935 - distributed.worker - INFO - dashboard at: 127.0.0.1:38095 +2025-09-03 03:32:32,935 - distributed.worker - INFO - Waiting to connect to: tls://127.0.0.1:43613 +2025-09-03 03:32:32,935 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:32:32,935 - distributed.worker - INFO - Threads: 1 +2025-09-03 03:32:32,935 - distributed.worker - INFO - Memory: 31.29 GiB +2025-09-03 03:32:32,935 - distributed.worker - INFO - Local Directory: /tmp/dask-scratch-space/worker-tq8qrur0 +2025-09-03 03:32:32,935 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:32:34,504 - distributed.scheduler - INFO - Register worker addr: tls://127.0.0.1:43809 name: 1 +2025-09-03 03:32:34,585 - distributed.worker - INFO - Starting Worker plugin shuffle +2025-09-03 03:32:34,585 - distributed.worker - INFO - Registered to: tls://127.0.0.1:43613 +2025-09-03 03:32:34,586 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:32:34,596 - distributed.scheduler - INFO - Starting worker compute stream, tls://127.0.0.1:43809 +2025-09-03 03:32:34,599 - distributed.core - INFO - Starting established connection to tls://127.0.0.1:54638 +2025-09-03 03:32:34,600 - distributed.core - INFO - Starting established connection to tls://127.0.0.1:43613 +2025-09-03 03:32:35,066 - distributed.scheduler - INFO - Register worker addr: tls://127.0.0.1:43081 name: 0 +2025-09-03 03:32:35,090 - distributed.worker - INFO - Starting Worker plugin shuffle +2025-09-03 03:32:35,091 - distributed.worker - INFO - Registered to: tls://127.0.0.1:43613 +2025-09-03 03:32:35,091 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:32:35,092 - distributed.core - INFO - Starting established connection to tls://127.0.0.1:43613 +2025-09-03 03:32:35,092 - distributed.scheduler - INFO - Starting worker compute stream, tls://127.0.0.1:43081 +2025-09-03 03:32:35,095 - distributed.core - INFO - Starting established connection to tls://127.0.0.1:54648 +2025-09-03 03:32:35,096 - distributed.core - INFO - Connection to tls://127.0.0.1:43613 has been closed. +2025-09-03 03:32:35,096 - distributed.worker - INFO - Stopping worker at tls://127.0.0.1:43081. Reason: worker-handle-scheduler-connection-broken +2025-09-03 03:32:35,099 - distributed.core - INFO - Connection to tls://127.0.0.1:54648 has been closed. +2025-09-03 03:32:35,099 - distributed.scheduler - INFO - Remove worker addr: tls://127.0.0.1:43081 name: 0 (stimulus_id='handle-worker-cleanup-1756819955.0996883') +2025-09-03 03:32:35,117 - distributed.nanny - INFO - Closing Nanny gracefully at 'tls://127.0.0.1:43253'. Reason: worker-handle-scheduler-connection-broken +2025-09-03 03:32:35,118 - distributed.worker - INFO - Removing Worker plugin shuffle +2025-09-03 03:32:35,121 - distributed.nanny - INFO - Worker closed +2025-09-03 03:32:37,126 - distributed.nanny - ERROR - Worker process died unexpectedly +2025-09-03 03:32:37,949 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:43253'. Reason: nanny-close-gracefully +2025-09-03 03:32:37,949 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:43253' closed. +2025-09-03 03:33:05,119 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:41233'. Reason: nanny-close +2025-09-03 03:33:05,119 - distributed.nanny - INFO - Nanny asking worker to close. Reason: nanny-close +2025-09-03 03:33:05,128 - distributed.worker - INFO - Stopping worker at tls://127.0.0.1:43809. Reason: nanny-close +2025-09-03 03:33:05,129 - distributed.worker - INFO - Removing Worker plugin shuffle +2025-09-03 03:33:05,130 - distributed.core - INFO - Connection to tls://127.0.0.1:43613 has been closed. +2025-09-03 03:33:05,132 - distributed.nanny - INFO - Worker closed +2025-09-03 03:33:05,133 - distributed.core - INFO - Received 'close-stream' from tls://127.0.0.1:54638; closing. +2025-09-03 03:33:05,133 - distributed.scheduler - INFO - Remove worker addr: tls://127.0.0.1:43809 name: 1 (stimulus_id='handle-worker-cleanup-1756819985.1337178') +2025-09-03 03:33:05,135 - distributed.scheduler - INFO - Lost all workers +2025-09-03 03:33:05,831 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:41233' closed. +2025-09-03 03:33:05,831 - distributed.scheduler - INFO - Closing scheduler. Reason: unknown +2025-09-03 03:33:05,834 - distributed.scheduler - INFO - Scheduler closing all comms +2025-09-03 03:33:05,844 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Cluster creation timeout; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 806, in start_cluster + raise TimeoutError("Cluster creation timeout") +TimeoutError: Cluster creation timeout +2025-09-03 03:33:06,854 - distributed.scheduler - INFO - State start +2025-09-03 03:33:06,871 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:34269 +2025-09-03 03:33:06,873 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:45097/status +2025-09-03 03:33:06,875 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:33:06,902 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:43589'. Reason: failure-to-start- +2025-09-03 03:33:06,902 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:43589' closed. +2025-09-03 03:33:06,902 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:44805'. Reason: failure-to-start- +2025-09-03 03:33:06,902 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:44805' closed. +2025-09-03 03:33:06,903 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:34269': TLS handshake failed with remote 'tls://127.0.0.1:43102': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:06,903 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:34269': TLS handshake failed with remote 'tls://127.0.0.1:43106': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:06,903 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:33:07,914 - distributed.scheduler - INFO - State start +2025-09-03 03:33:07,925 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:33969 +2025-09-03 03:33:07,930 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:42873/status +2025-09-03 03:33:07,932 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:33:07,948 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:45537'. Reason: failure-to-start- +2025-09-03 03:33:07,957 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:45537' closed. +2025-09-03 03:33:07,957 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:45819'. Reason: failure-to-start- +2025-09-03 03:33:07,957 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:45819' closed. +2025-09-03 03:33:07,957 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:33969': TLS handshake failed with remote 'tls://127.0.0.1:45740': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:07,957 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:33969': TLS handshake failed with remote 'tls://127.0.0.1:45750': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:07,957 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:33:08,968 - distributed.scheduler - INFO - State start +2025-09-03 03:33:08,995 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:38781 +2025-09-03 03:33:09,006 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:39087/status +2025-09-03 03:33:09,008 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:33:09,041 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:35995'. Reason: failure-to-start- +2025-09-03 03:33:09,041 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:35995' closed. +2025-09-03 03:33:09,042 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:45303'. Reason: failure-to-start- +2025-09-03 03:33:09,042 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:45303' closed. +2025-09-03 03:33:09,042 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:38781': TLS handshake failed with remote 'tls://127.0.0.1:44576': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:09,042 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:38781': TLS handshake failed with remote 'tls://127.0.0.1:44578': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:09,042 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:33:10,050 - distributed.scheduler - INFO - State start +2025-09-03 03:33:10,063 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:42343 +2025-09-03 03:33:10,070 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:36793/status +2025-09-03 03:33:10,072 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:33:10,095 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:40603'. Reason: failure-to-start- +2025-09-03 03:33:10,096 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:40603' closed. +2025-09-03 03:33:10,096 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:35063'. Reason: failure-to-start- +2025-09-03 03:33:10,096 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:35063' closed. +2025-09-03 03:33:10,096 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:42343': TLS handshake failed with remote 'tls://127.0.0.1:60034': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:10,097 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:42343': TLS handshake failed with remote 'tls://127.0.0.1:60046': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:10,097 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:33:11,107 - distributed.scheduler - INFO - State start +2025-09-03 03:33:11,127 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:39409 +2025-09-03 03:33:11,134 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:35427/status +2025-09-03 03:33:11,136 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:33:11,167 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:35475'. Reason: failure-to-start- +2025-09-03 03:33:11,167 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:35475' closed. +2025-09-03 03:33:11,167 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:35931'. Reason: failure-to-start- +2025-09-03 03:33:11,167 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:35931' closed. +2025-09-03 03:33:11,168 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:39409': TLS handshake failed with remote 'tls://127.0.0.1:56234': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:11,168 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:39409': TLS handshake failed with remote 'tls://127.0.0.1:56236': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:11,168 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:33:12,182 - distributed.scheduler - INFO - State start +2025-09-03 03:33:12,196 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:46555 +2025-09-03 03:33:12,204 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:45969/status +2025-09-03 03:33:12,211 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:33:12,222 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:33909'. Reason: failure-to-start- +2025-09-03 03:33:12,222 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:33909' closed. +2025-09-03 03:33:12,222 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:33099'. Reason: failure-to-start- +2025-09-03 03:33:12,222 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:33099' closed. +2025-09-03 03:33:12,223 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:46555': TLS handshake failed with remote 'tls://127.0.0.1:45370': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:12,223 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:46555': TLS handshake failed with remote 'tls://127.0.0.1:45380': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:12,223 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:33:13,228 - distributed.scheduler - INFO - State start +2025-09-03 03:33:13,260 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:41039 +2025-09-03 03:33:13,262 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:35255/status +2025-09-03 03:33:13,264 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:33:13,297 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:43589'. Reason: failure-to-start- +2025-09-03 03:33:13,297 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:43589' closed. +2025-09-03 03:33:13,297 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:40665'. Reason: failure-to-start- +2025-09-03 03:33:13,297 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:40665' closed. +2025-09-03 03:33:13,298 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:41039': TLS handshake failed with remote 'tls://127.0.0.1:47700': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:13,298 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:41039': TLS handshake failed with remote 'tls://127.0.0.1:47704': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:13,298 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:33:14,307 - distributed.scheduler - INFO - State start +2025-09-03 03:33:14,312 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:40667 +2025-09-03 03:33:14,315 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:41079/status +2025-09-03 03:33:14,317 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:33:14,353 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:35355'. Reason: failure-to-start- +2025-09-03 03:33:14,353 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:35355' closed. +2025-09-03 03:33:14,353 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:39797'. Reason: failure-to-start- +2025-09-03 03:33:14,353 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:39797' closed. +2025-09-03 03:33:14,354 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:40667': TLS handshake failed with remote 'tls://127.0.0.1:53380': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:14,354 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:40667': TLS handshake failed with remote 'tls://127.0.0.1:53396': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:14,354 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:33:18,584 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 4.31s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:33:18,585 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 4.25s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:33:18,586 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 4.20s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:33:18,586 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 4.14s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:33:18,587 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 4.07s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:33:18,588 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 4.01s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:33:18,588 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 3.95s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:33:18,589 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 3.87s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:33:18,591 - distributed.scheduler - INFO - State start +2025-09-03 03:33:20,164 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:34091 +2025-09-03 03:33:20,174 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:37883/status +2025-09-03 03:33:20,184 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:33:20,575 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:42269'. Reason: failure-to-start- +2025-09-03 03:33:20,576 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:42269' closed. +2025-09-03 03:33:20,576 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:33813'. Reason: failure-to-start- +2025-09-03 03:33:20,584 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:33813' closed. +2025-09-03 03:33:20,587 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:34091': TLS handshake failed with remote 'tls://127.0.0.1:58840': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:20,587 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:34091': TLS handshake failed with remote 'tls://127.0.0.1:58856': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:20,587 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:33:21,601 - distributed.scheduler - INFO - State start +2025-09-03 03:33:21,618 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:33151 +2025-09-03 03:33:21,624 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:43929/status +2025-09-03 03:33:21,626 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:33:21,662 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:39227'. Reason: failure-to-start- +2025-09-03 03:33:21,662 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:39227' closed. +2025-09-03 03:33:21,662 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:41191'. Reason: failure-to-start- +2025-09-03 03:33:21,663 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:41191' closed. +2025-09-03 03:33:21,663 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:33151': TLS handshake failed with remote 'tls://127.0.0.1:56414': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:21,663 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:33151': TLS handshake failed with remote 'tls://127.0.0.1:56430': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:21,663 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:33:22,683 - distributed.scheduler - INFO - State start +2025-09-03 03:33:22,705 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:42291 +2025-09-03 03:33:22,708 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:38179/status +2025-09-03 03:33:22,714 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:33:22,743 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:40185'. Reason: failure-to-start- +2025-09-03 03:33:22,743 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:40185' closed. +2025-09-03 03:33:22,743 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:40481'. Reason: failure-to-start- +2025-09-03 03:33:22,743 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:40481' closed. +2025-09-03 03:33:22,744 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:42291': TLS handshake failed with remote 'tls://127.0.0.1:37390': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:22,744 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:42291': TLS handshake failed with remote 'tls://127.0.0.1:37398': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:22,744 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:33:23,753 - distributed.scheduler - INFO - State start +2025-09-03 03:33:23,767 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:42947 +2025-09-03 03:33:23,773 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:46207/status +2025-09-03 03:33:23,775 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:33:23,815 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:33313'. Reason: failure-to-start- +2025-09-03 03:33:23,815 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:33313' closed. +2025-09-03 03:33:23,815 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:39345'. Reason: failure-to-start- +2025-09-03 03:33:23,815 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:39345' closed. +2025-09-03 03:33:23,816 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:42947': TLS handshake failed with remote 'tls://127.0.0.1:48438': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:23,820 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:42947': TLS handshake failed with remote 'tls://127.0.0.1:48452': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:23,820 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:33:24,825 - distributed.scheduler - INFO - State start +2025-09-03 03:33:24,834 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:44805 +2025-09-03 03:33:24,845 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:36079/status +2025-09-03 03:33:24,847 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:33:24,879 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:41053'. Reason: failure-to-start- +2025-09-03 03:33:24,879 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:41053' closed. +2025-09-03 03:33:24,879 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:34279'. Reason: failure-to-start- +2025-09-03 03:33:24,879 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:34279' closed. +2025-09-03 03:33:24,880 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:44805': TLS handshake failed with remote 'tls://127.0.0.1:56406': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:24,880 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:44805': TLS handshake failed with remote 'tls://127.0.0.1:56410': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:24,888 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:33:25,919 - distributed.scheduler - INFO - State start +2025-09-03 03:33:25,932 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:46137 +2025-09-03 03:33:25,935 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:42201/status +2025-09-03 03:33:25,941 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:33:25,968 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:42503'. Reason: failure-to-start- +2025-09-03 03:33:25,969 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:42503' closed. +2025-09-03 03:33:25,969 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:37861'. Reason: failure-to-start- +2025-09-03 03:33:25,969 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:37861' closed. +2025-09-03 03:33:25,970 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:46137': TLS handshake failed with remote 'tls://127.0.0.1:53922': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:25,970 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:46137': TLS handshake failed with remote 'tls://127.0.0.1:53928': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:25,970 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:33:26,975 - distributed.scheduler - INFO - State start +2025-09-03 03:33:27,002 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:41831 +2025-09-03 03:33:27,015 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:44281/status +2025-09-03 03:33:27,028 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:33:27,068 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:44609'. Reason: failure-to-start- +2025-09-03 03:33:27,069 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:44609' closed. +2025-09-03 03:33:27,069 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:35181'. Reason: failure-to-start- +2025-09-03 03:33:27,069 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:35181' closed. +2025-09-03 03:33:27,069 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:41831': TLS handshake failed with remote 'tls://127.0.0.1:33130': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:27,069 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:41831': TLS handshake failed with remote 'tls://127.0.0.1:33138': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:27,070 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:33:28,075 - distributed.scheduler - INFO - State start +2025-09-03 03:33:28,092 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:42167 +2025-09-03 03:33:28,095 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:41639/status +2025-09-03 03:33:28,101 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:33:28,125 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:40857'. Reason: failure-to-start- +2025-09-03 03:33:28,125 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:40857' closed. +2025-09-03 03:33:28,125 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:43595'. Reason: failure-to-start- +2025-09-03 03:33:28,125 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:43595' closed. +2025-09-03 03:33:28,126 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:42167': TLS handshake failed with remote 'tls://127.0.0.1:49828': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:28,126 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:42167': TLS handshake failed with remote 'tls://127.0.0.1:49834': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:28,126 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:33:29,130 - distributed.scheduler - INFO - State start +2025-09-03 03:33:29,145 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:40195 +2025-09-03 03:33:29,147 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:39223/status +2025-09-03 03:33:29,154 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:33:29,213 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:41947'. Reason: failure-to-start- +2025-09-03 03:33:29,214 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:41947' closed. +2025-09-03 03:33:29,214 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:46227'. Reason: failure-to-start- +2025-09-03 03:33:29,214 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:46227' closed. +2025-09-03 03:33:29,214 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:40195': TLS handshake failed with remote 'tls://127.0.0.1:42158': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:29,214 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:40195': TLS handshake failed with remote 'tls://127.0.0.1:42164': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:29,215 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:33:30,220 - distributed.scheduler - INFO - State start +2025-09-03 03:33:30,234 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:42791 +2025-09-03 03:33:30,243 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:36581/status +2025-09-03 03:33:30,247 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:33:30,271 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:46443'. Reason: failure-to-start- +2025-09-03 03:33:30,272 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:46443' closed. +2025-09-03 03:33:30,272 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:39325'. Reason: failure-to-start- +2025-09-03 03:33:30,272 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:39325' closed. +2025-09-03 03:33:30,277 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:42791': TLS handshake failed with remote 'tls://127.0.0.1:58596': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:30,277 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:42791': TLS handshake failed with remote 'tls://127.0.0.1:58608': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:33:30,277 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +_____________________________ test_retire_workers ______________________________ + +fut = ._..test_func..async_fn at 0xffff3d4aa180> +timeout = 60 + + async def wait_for(fut: Awaitable[T], timeout: float) -> T: + async with asyncio.timeout(timeout): +> return await fut + +distributed/utils.py:1914: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +distributed/utils_test.py:1007: in async_fn + _cluster_factory() as (s, workers), +/usr/lib/python3.13/contextlib.py:214: in __aenter__ + return await anext(self.gen) +distributed/utils_test.py:974: in _cluster_factory + s, ws = await start_cluster( +distributed/utils_test.py:768: in start_cluster + s = await Scheduler( +distributed/core.py:528: in start + await wait_for(self.start_unsafe(), timeout=timeout) +distributed/utils.py:1914: in wait_for + return await fut +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = + + async def start_unsafe(self) -> Self: + """Clear out old state and restart all running coroutines""" + await super().start_unsafe() + + enable_gc_diagnosis() + + self._clear_task_state() + + for addr in self._start_address: + await self.listen( + addr, + allow_offload=False, + handshake_overrides={"pickle-protocol": 4, "compression": None}, + **self.security.get_listen_args("scheduler"), + ) + self.ip = get_address_host(self.listen_address) + listen_ip = self.ip + + if listen_ip == "0.0.0.0": + listen_ip = "" + + if self.address.startswith("inproc://"): + listen_ip = "localhost" + + # Services listen on all addresses + self.start_services(listen_ip) + + for listener in self.listeners: + logger.info(" Scheduler at: %25s", listener.contact_address) + for name, server in self.services.items(): + if name == "dashboard": + addr = get_address_host(listener.contact_address) + try: + link = format_dashboard_link(addr, server.port) + # formatting dashboard link can fail if distributed.dashboard.link + # refers to non-existant env vars. + except KeyError as e: + logger.warning( + f"Failed to format dashboard link, unknown value: {e}" + ) + link = f":{server.port}" + else: + link = f"{listen_ip}:{server.port}" + logger.info("%11s at: %25s", name, link) + + if self.scheduler_file: + with open(self.scheduler_file, "w") as f: + json.dump(self.identity(), f, indent=2) + + fn = self.scheduler_file # remove file when we close the process + + def del_scheduler_file() -> None: + if os.path.exists(fn): + os.remove(fn) + + weakref.finalize(self, del_scheduler_file) + + await self.preloads.start() + + if self.jupyter: + # Allow insecure communications from local users + if self.address.startswith("tls://"): + await self.listen("tcp://localhost:0") + os.environ["DASK_SCHEDULER_ADDRESS"] = self.listeners[-1].contact_address + +> await asyncio.gather( + *[plugin.start(self) for plugin in list(self.plugins.values())] + ) +E asyncio.exceptions.CancelledError + +distributed/scheduler.py:4259: CancelledError + +The above exception was the direct cause of the following exception: + +args = (), kwds = {} + + @wraps(func) + def inner(*args, **kwds): + with self._recreate_cm(): +> return func(*args, **kwds) + +/usr/lib/python3.13/contextlib.py:85: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +/usr/lib/python3.13/contextlib.py:85: in inner + return func(*args, **kwds) +distributed/utils_test.py:1090: in test_func + return _run_and_close_tornado(async_fn_outer) +distributed/utils_test.py:380: in _run_and_close_tornado + return asyncio_run(inner_fn(), loop_factory=get_loop_factory()) +/usr/lib/python3.13/asyncio/runners.py:195: in run + return runner.run(main) +/usr/lib/python3.13/asyncio/runners.py:118: in run + return self._loop.run_until_complete(task) +/usr/lib/python3.13/asyncio/base_events.py:725: in run_until_complete + return future.result() +distributed/utils_test.py:377: in inner_fn + return await async_fn(*args, **kwargs) +distributed/utils_test.py:1087: in async_fn_outer + return await utils_wait_for(async_fn(), timeout=timeout * 2) +distributed/utils.py:1913: in wait_for + async with asyncio.timeout(timeout): +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +exc_type = +exc_val = CancelledError(), exc_tb = + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> Optional[bool]: + assert self._state in (_State.ENTERED, _State.EXPIRING) + + if self._timeout_handler is not None: + self._timeout_handler.cancel() + self._timeout_handler = None + + if self._state is _State.EXPIRING: + self._state = _State.EXPIRED + + if self._task.uncancel() <= self._cancelling and exc_type is not None: + # Since there are no new cancel requests, we're + # handling this. + if issubclass(exc_type, exceptions.CancelledError): +> raise TimeoutError from exc_val +E TimeoutError + +/usr/lib/python3.13/asyncio/timeouts.py:116: TimeoutError +----------------------------- Captured stderr call ----------------------------- +2025-09-03 03:33:36,646 - distributed.scheduler - INFO - State start +2025-09-03 03:33:36,655 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:45571 +2025-09-03 03:33:36,662 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:40931/status +2025-09-03 03:33:36,664 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:33:36,743 - distributed.nanny - INFO - Start Nanny at: 'tls://127.0.0.1:46325' +2025-09-03 03:33:36,752 - distributed.nanny - INFO - Start Nanny at: 'tls://127.0.0.1:45119' +2025-09-03 03:33:38,321 - distributed.worker.memory - WARNING - Ignoring provided memory limit 33596850176 due to system memory limit of 31.29 GiB +2025-09-03 03:33:38,337 - distributed.worker.memory - WARNING - Ignoring provided memory limit 33596850176 due to system memory limit of 31.29 GiB +2025-09-03 03:33:38,368 - distributed.worker - INFO - Start worker at: tls://127.0.0.1:34337 +2025-09-03 03:33:38,376 - distributed.worker - INFO - Listening to: tls://127.0.0.1:34337 +2025-09-03 03:33:38,376 - distributed.worker - INFO - Worker name: 1 +2025-09-03 03:33:38,376 - distributed.worker - INFO - dashboard at: 127.0.0.1:45735 +2025-09-03 03:33:38,376 - distributed.worker - INFO - Waiting to connect to: tls://127.0.0.1:45571 +2025-09-03 03:33:38,377 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:33:38,377 - distributed.worker - INFO - Threads: 2 +2025-09-03 03:33:38,377 - distributed.worker - INFO - Memory: 31.29 GiB +2025-09-03 03:33:38,377 - distributed.worker - INFO - Local Directory: /tmp/dask-scratch-space/worker-bl2281qi +2025-09-03 03:33:38,377 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:33:38,382 - distributed.worker - INFO - Start worker at: tls://127.0.0.1:43599 +2025-09-03 03:33:38,382 - distributed.worker - INFO - Listening to: tls://127.0.0.1:43599 +2025-09-03 03:33:38,382 - distributed.worker - INFO - Worker name: 0 +2025-09-03 03:33:38,382 - distributed.worker - INFO - dashboard at: 127.0.0.1:38535 +2025-09-03 03:33:38,382 - distributed.worker - INFO - Waiting to connect to: tls://127.0.0.1:45571 +2025-09-03 03:33:38,382 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:33:38,382 - distributed.worker - INFO - Threads: 1 +2025-09-03 03:33:38,382 - distributed.worker - INFO - Memory: 31.29 GiB +2025-09-03 03:33:38,382 - distributed.worker - INFO - Local Directory: /tmp/dask-scratch-space/worker-ua03bqy0 +2025-09-03 03:33:38,382 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:33:39,176 - distributed.scheduler - INFO - Register worker addr: tls://127.0.0.1:34337 name: 1 +2025-09-03 03:33:39,229 - distributed.worker - INFO - Starting Worker plugin shuffle +2025-09-03 03:33:39,230 - distributed.worker - INFO - Registered to: tls://127.0.0.1:45571 +2025-09-03 03:33:39,230 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:33:39,230 - distributed.scheduler - INFO - Starting worker compute stream, tls://127.0.0.1:34337 +2025-09-03 03:33:39,237 - distributed.core - INFO - Starting established connection to tls://127.0.0.1:57704 +2025-09-03 03:33:39,238 - distributed.core - INFO - Starting established connection to tls://127.0.0.1:45571 +2025-09-03 03:33:39,300 - distributed.scheduler - INFO - Register worker addr: tls://127.0.0.1:43599 name: 0 +2025-09-03 03:33:39,351 - distributed.worker - INFO - Starting Worker plugin shuffle +2025-09-03 03:33:39,352 - distributed.worker - INFO - Registered to: tls://127.0.0.1:45571 +2025-09-03 03:33:39,352 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:33:39,353 - distributed.core - INFO - Starting established connection to tls://127.0.0.1:45571 +2025-09-03 03:33:39,353 - distributed.scheduler - INFO - Starting worker compute stream, tls://127.0.0.1:43599 +2025-09-03 03:33:39,355 - distributed.core - INFO - Starting established connection to tls://127.0.0.1:57706 +2025-09-03 03:33:39,360 - distributed.core - INFO - Connection to tls://127.0.0.1:57706 has been closed. +2025-09-03 03:33:39,361 - distributed.scheduler - INFO - Remove worker addr: tls://127.0.0.1:43599 name: 0 (stimulus_id='handle-worker-cleanup-1756820019.361046') +2025-09-03 03:33:39,364 - distributed.core - INFO - Connection to tls://127.0.0.1:45571 has been closed. +2025-09-03 03:33:39,364 - distributed.worker - INFO - Stopping worker at tls://127.0.0.1:43599. Reason: worker-handle-scheduler-connection-broken +2025-09-03 03:33:39,399 - distributed.nanny - INFO - Closing Nanny gracefully at 'tls://127.0.0.1:46325'. Reason: worker-handle-scheduler-connection-broken +2025-09-03 03:33:39,400 - distributed.worker - INFO - Removing Worker plugin shuffle +2025-09-03 03:33:39,404 - distributed.nanny - INFO - Worker closed +2025-09-03 03:33:41,937 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:46325'. Reason: nanny-close-gracefully +2025-09-03 03:33:41,938 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:46325' closed. +2025-09-03 03:34:09,378 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:45119'. Reason: nanny-close +2025-09-03 03:34:09,379 - distributed.nanny - INFO - Nanny asking worker to close. Reason: nanny-close +2025-09-03 03:34:09,380 - distributed.worker - INFO - Stopping worker at tls://127.0.0.1:34337. Reason: nanny-close +2025-09-03 03:34:09,380 - distributed.worker - INFO - Removing Worker plugin shuffle +2025-09-03 03:34:09,383 - distributed.core - INFO - Received 'close-stream' from tls://127.0.0.1:57704; closing. +2025-09-03 03:34:09,383 - distributed.core - INFO - Connection to tls://127.0.0.1:45571 has been closed. +2025-09-03 03:34:09,385 - distributed.nanny - INFO - Worker closed +2025-09-03 03:34:09,383 - distributed.scheduler - INFO - Remove worker addr: tls://127.0.0.1:34337 name: 1 (stimulus_id='handle-worker-cleanup-1756820049.383546') +2025-09-03 03:34:09,387 - distributed.scheduler - INFO - Lost all workers +2025-09-03 03:34:09,843 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:45119' closed. +2025-09-03 03:34:09,844 - distributed.scheduler - INFO - Closing scheduler. Reason: unknown +2025-09-03 03:34:09,847 - distributed.scheduler - INFO - Scheduler closing all comms +2025-09-03 03:34:09,851 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Cluster creation timeout; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 806, in start_cluster + raise TimeoutError("Cluster creation timeout") +TimeoutError: Cluster creation timeout +2025-09-03 03:34:10,858 - distributed.scheduler - INFO - State start +2025-09-03 03:34:10,863 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:37695 +2025-09-03 03:34:10,866 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:39389/status +2025-09-03 03:34:10,867 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:34:10,874 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:36279'. Reason: failure-to-start- +2025-09-03 03:34:10,875 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:36279' closed. +2025-09-03 03:34:10,875 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:33983'. Reason: failure-to-start- +2025-09-03 03:34:10,875 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:33983' closed. +2025-09-03 03:34:14,919 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:37695': TLS handshake failed with remote 'tls://127.0.0.1:32824': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:34:14,919 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:37695': TLS handshake failed with remote 'tls://127.0.0.1:32832': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:34:14,919 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:34:14,925 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 4.06s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:34:15,930 - distributed.scheduler - INFO - State start +2025-09-03 03:34:15,945 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:36991 +2025-09-03 03:34:15,947 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:42963/status +2025-09-03 03:34:15,956 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:34:16,031 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:41061'. Reason: failure-to-start- +2025-09-03 03:34:16,032 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:41061' closed. +2025-09-03 03:34:16,032 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:43427'. Reason: failure-to-start- +2025-09-03 03:34:16,032 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:43427' closed. +2025-09-03 03:34:16,033 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:36991': TLS handshake failed with remote 'tls://127.0.0.1:59668': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:34:16,033 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:36991': TLS handshake failed with remote 'tls://127.0.0.1:59682': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:34:16,033 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:34:17,048 - distributed.scheduler - INFO - State start +2025-09-03 03:34:17,058 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:43185 +2025-09-03 03:34:17,062 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:46037/status +2025-09-03 03:34:17,065 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:34:17,082 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:42927'. Reason: failure-to-start- +2025-09-03 03:34:17,083 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:42927' closed. +2025-09-03 03:34:17,083 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:46803'. Reason: failure-to-start- +2025-09-03 03:34:17,083 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:46803' closed. +2025-09-03 03:34:17,084 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:43185': TLS handshake failed with remote 'tls://127.0.0.1:43084': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:34:17,084 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:43185': TLS handshake failed with remote 'tls://127.0.0.1:43098': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:34:17,084 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:34:18,343 - distributed.scheduler - INFO - State start +2025-09-03 03:34:18,352 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:41805 +2025-09-03 03:34:18,356 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:44121/status +2025-09-03 03:34:18,360 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:34:18,479 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:46323'. Reason: failure-to-start- +2025-09-03 03:34:18,480 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:46323' closed. +2025-09-03 03:34:18,480 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:33873'. Reason: failure-to-start- +2025-09-03 03:34:18,480 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:33873' closed. +2025-09-03 03:34:18,483 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:41805': TLS handshake failed with remote 'tls://127.0.0.1:45490': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:34:18,483 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:41805': TLS handshake failed with remote 'tls://127.0.0.1:45498': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:34:18,483 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:34:19,493 - distributed.scheduler - INFO - State start +2025-09-03 03:34:19,509 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:38865 +2025-09-03 03:34:19,513 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:40653/status +2025-09-03 03:34:19,517 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:34:19,534 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:44005'. Reason: failure-to-start- +2025-09-03 03:34:19,535 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:44005' closed. +2025-09-03 03:34:19,571 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:42897'. Reason: failure-to-start- +2025-09-03 03:34:19,571 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:42897' closed. +2025-09-03 03:34:19,572 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:38865': TLS handshake failed with remote 'tls://127.0.0.1:43668': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:34:19,572 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:38865': TLS handshake failed with remote 'tls://127.0.0.1:43678': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:34:19,573 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:34:20,581 - distributed.scheduler - INFO - State start +2025-09-03 03:34:20,588 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:36801 +2025-09-03 03:34:20,591 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:43847/status +2025-09-03 03:34:20,594 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:34:20,605 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:46445'. Reason: failure-to-start- +2025-09-03 03:34:20,606 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:46445' closed. +2025-09-03 03:34:20,606 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:43137'. Reason: failure-to-start- +2025-09-03 03:34:20,606 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:43137' closed. +2025-09-03 03:34:20,607 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:36801': TLS handshake failed with remote 'tls://127.0.0.1:48462': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:34:20,607 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:36801': TLS handshake failed with remote 'tls://127.0.0.1:48472': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:34:20,607 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:34:21,615 - distributed.scheduler - INFO - State start +2025-09-03 03:34:21,624 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:44837 +2025-09-03 03:34:21,626 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:44965/status +2025-09-03 03:34:21,628 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:34:21,635 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:34385'. Reason: failure-to-start- +2025-09-03 03:34:21,636 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:34385' closed. +2025-09-03 03:34:21,636 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:39397'. Reason: failure-to-start- +2025-09-03 03:34:21,636 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:39397' closed. +2025-09-03 03:34:21,636 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:44837': TLS handshake failed with remote 'tls://127.0.0.1:49502': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:34:21,636 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:44837': TLS handshake failed with remote 'tls://127.0.0.1:49514': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:34:21,636 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:34:22,762 - distributed.scheduler - INFO - State start +2025-09-03 03:34:22,772 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:36869 +2025-09-03 03:34:22,776 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:37485/status +2025-09-03 03:34:22,779 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:34:22,796 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:44273'. Reason: failure-to-start- +2025-09-03 03:34:22,797 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:44273' closed. +2025-09-03 03:34:22,797 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:37157'. Reason: failure-to-start- +2025-09-03 03:34:22,798 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:37157' closed. +2025-09-03 03:34:22,798 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:36869': TLS handshake failed with remote 'tls://127.0.0.1:47340': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:34:22,799 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:36869': TLS handshake failed with remote 'tls://127.0.0.1:47354': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:34:22,799 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:34:23,808 - distributed.scheduler - INFO - State start +2025-09-03 03:34:23,819 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:39215 +2025-09-03 03:34:23,823 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:42607/status +2025-09-03 03:34:23,828 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:34:23,861 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:38941'. Reason: failure-to-start- +2025-09-03 03:34:24,308 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:38941' closed. +2025-09-03 03:34:24,309 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:34225'. Reason: failure-to-start- +2025-09-03 03:34:24,309 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:34225' closed. +2025-09-03 03:34:24,310 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:39215': TLS handshake failed with remote 'tls://127.0.0.1:55300': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:34:24,311 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:39215': TLS handshake failed with remote 'tls://127.0.0.1:55316': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:34:24,311 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:34:25,321 - distributed.scheduler - INFO - State start +2025-09-03 03:34:25,346 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:37811 +2025-09-03 03:34:25,369 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:41249/status +2025-09-03 03:34:25,371 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:34:25,434 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:36065'. Reason: failure-to-start- +2025-09-03 03:34:25,434 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:36065' closed. +2025-09-03 03:34:25,434 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:41953'. Reason: failure-to-start- +2025-09-03 03:34:25,434 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:41953' closed. +2025-09-03 03:34:25,435 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:37811': TLS handshake failed with remote 'tls://127.0.0.1:46536': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:34:25,435 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:37811': TLS handshake failed with remote 'tls://127.0.0.1:46542': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:34:25,435 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:34:32,949 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 7.66s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:34:32,949 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 7.56s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:34:32,950 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 7.56s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:34:32,950 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 7.55s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:34:32,951 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 7.55s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:34:32,965 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 7.49s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:34:32,965 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 7.44s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:34:32,966 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 7.39s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:34:32,966 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 7.37s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:34:32,967 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 7.33s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:34:49,672 - distributed.scheduler - INFO - State start +2025-09-03 03:34:49,687 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:39035 +2025-09-03 03:34:49,697 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:37177/status +2025-09-03 03:34:51,020 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:34:51,024 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 18.06s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:34:51,025 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 18.06s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:34:51,025 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 18.06s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:34:51,026 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 18.06s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:34:51,026 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 18.08s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:34:51,027 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 18.08s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:35:02,022 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 29.07s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:35:02,023 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 29.07s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:35:02,023 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 29.07s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:35:02,024 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 29.06s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:35:02,062 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 11.04s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:35:02,063 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 11.04s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:35:02,063 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 11.04s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:35:02,064 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 11.04s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:35:02,064 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 11.04s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. +2025-09-03 03:35:02,065 - distributed.core - INFO - Event loop was unresponsive in Scheduler for 11.04s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. ============================= slowest 20 durations ============================= -12.62s call distributed/tests/test_gc.py::test_gc_diagnosis_cpu_time +85.43s call distributed/tests/test_tls_functional.py::test_retire_workers +60.00s call distributed/tests/test_tls_functional.py::test_nanny +26.05s call distributed/tests/test_gc.py::test_gc_diagnosis_cpu_time +18.76s call distributed/tests/test_steal.py::test_restart +14.52s call distributed/tests/test_stress.py::test_cancel_stress +11.62s call distributed/tests/test_stress.py::test_cancel_stress_sync +10.97s call distributed/tests/test_steal.py::test_trivial_workload_should_not_cause_work_stealing +10.58s call distributed/tests/test_nanny.py::test_num_fds 10.02s call distributed/tests/test_utils_test.py::test_popen_timeout -7.89s call distributed/tests/test_stress.py::test_cancel_stress -6.18s call distributed/tests/test_stress.py::test_cancel_stress_sync -6.15s call distributed/tests/test_failed_workers.py::test_worker_doesnt_await_task_completion -6.03s call distributed/tests/test_nanny.py::test_num_fds -5.22s call distributed/tests/test_steal.py::test_balance_with_longer_task -5.02s call distributed/tests/test_chaos.py::test_KillWorker[sys.exit] -4.56s call distributed/diagnostics/tests/test_progress.py::test_group_timing -4.37s call distributed/shuffle/tests/test_rechunk.py::test_rechunk_method -4.24s call distributed/shuffle/tests/test_rechunk.py::test_homogeneously_schedule_unpack -3.90s call distributed/diagnostics/tests/test_progress.py::test_AllProgress -3.82s call distributed/tests/test_failed_workers.py::test_restart_sync -3.77s call distributed/tests/test_tls_functional.py::test_retire_workers -3.72s call distributed/protocol/tests/test_serialize.py::test_large_pickled_object -3.66s call distributed/tests/test_nanny.py::test_environ_plugin -3.63s call distributed/tests/test_scheduler.py::test_balance_many_workers_2 -3.44s call distributed/shuffle/tests/test_rechunk.py::test_rechunk_with_partially_unknown_dimension[x5-chunks5] -3.24s call distributed/tests/test_nanny.py::test_nanny_closes_cleanly_if_worker_is_terminated -3.23s call distributed/tests/test_failed_workers.py::test_restart_during_computation +9.86s call distributed/tests/test_nanny.py::test_nanny_closes_cleanly_if_worker_is_terminated +7.75s call distributed/tests/test_nanny.py::test_environ_plugin +7.66s call distributed/tests/test_scheduler.py::test_balance_many_workers_2 +7.60s call distributed/tests/test_failed_workers.py::test_restart_sync +7.49s call distributed/tests/test_failed_workers.py::test_worker_doesnt_await_task_completion +6.51s call distributed/tests/test_nanny.py::test_nanny_restart +6.37s call distributed/shuffle/tests/test_rechunk.py::test_rechunk_with_partially_unknown_dimension[x5-chunks5] +6.19s call distributed/tests/test_nanny.py::test_nanny_restart_timeout +6.08s call distributed/tests/test_scheduler.py::test_task_prefix +6.06s call distributed/tests/test_stress.py::test_stress_scatter_death +5.93s call distributed/tests/test_failed_workers.py::test_restart_cleared =========================== short test summary info ============================ SKIPPED [1] distributed/cli/tests/test_dask_ssh.py:9: could not import 'paramiko': No module named 'paramiko' SKIPPED [1] distributed/comm/tests/test_ucx.py:15: could not import 'ucp': No module named 'ucp' @@ -6946,8 +10180,5198 @@ SKIPPED [1] distributed/tests/test_worker_memory.py:871: need --runslow option to run SKIPPED [2] distributed/tests/test_worker_memory.py:883: need --runslow option to run SKIPPED [1] distributed/tests/test_worker_memory.py:997: need --runslow option to run -= 2912 passed, 265 skipped, 222 deselected, 15 xfailed, 8 xpassed in 1093.71s (0:18:13) = -*** END OF RUN 1: ALL TESTS RUN HAVE NOW PASSED/XFAILED *** +FAILED distributed/tests/test_nanny.py::test_nanny_closes_cleanly_if_worker_is_terminated - assert 1756819390.6073353 < (1756819383.7670298 + 5) + + where 1756819390.6073353 = time() +FAILED distributed/tests/test_steal.py::test_restart - AssertionError: {: , : } +assert False + + where False = all(. at 0xffff3d184ba0>) +FAILED distributed/tests/test_tls_functional.py::test_nanny - TimeoutError +FAILED distributed/tests/test_tls_functional.py::test_retire_workers - TimeoutError += 4 failed, 2908 passed, 265 skipped, 222 deselected, 15 xfailed, 8 xpassed in 1839.09s (0:30:39) = +*** END OF RUN 1: NOT ALL TESTS HAVE YET PASSED/XFAILED *** +*** STARTING RUN 2: python3.13 -m pytest --pyargs distributed --verbose --color=no --timeout-method=signal --timeout=300 -m not avoid_ci -rfE --last-failed --last-failed-no-failures none --ignore=distributed/comm/tests/test_comms.py --ignore=distributed/comm/tests/test_ws.py --ignore=distributed/deploy/tests/test_adaptive.py --ignore=distributed/deploy/tests/test_local.py --ignore=distributed/deploy/tests/test_slow_adaptive.py --ignore=distributed/deploy/tests/test_spec_cluster.py --deselect=distributed/cli/tests/test_dask_scheduler.py::test_no_dashboard --deselect=distributed/deploy/tests/test_local.py::test_localcluster_get_client --deselect=distributed/deploy/tests/test_old_ssh.py::test_cluster --deselect=distributed/deploy/tests/test_old_ssh.py::test_old_ssh_nprocs_renamed_to_n_workers --deselect=distributed/deploy/tests/test_old_ssh.py::test_nprocs_attribute_is_deprecated --deselect=distributed/deploy/tests/test_ssh.py::test_nprocs_attribute_is_deprecated --deselect=distributed/http/tests/test_core.py::test_prometheus_api_doc --deselect=distributed/tests/test_init.py::test_git_revision --deselect=distributed/tests/test_jupyter.py::test_jupyter_idle_timeout --deselect=distributed/tests/test_jupyter.py::test_jupyter_idle_timeout_returned --deselect=distributed/tests/test_jupyter.py::test_jupyter_server --deselect=distributed/tests/test_jupyter.py::test_shutsdown_cleanly --deselect=distributed/tests/test_profile.py::test_stack_overflow --deselect=distributed/tests/test_pubsub.py::test_client_worker --deselect=distributed/tests/test_queues.py::test_queue_in_task --deselect=distributed/tests/test_spill.py::test_spillbuffer_oserror --deselect=distributed/tests/test_steal.py::test_steal_twice --deselect=distributed/tests/test_utils_test.py::test_cluster --deselect=distributed/tests/test_variable.py::test_variable_in_task --deselect=distributed/tests/test_worker.py::test_process_executor_kills_process --deselect=distributed/tests/test_worker_memory.py::test_fail_to_pickle_execute_1 --deselect=distributed/tests/test_worker_state_machine.py::test_task_state_instance_are_garbage_collected --deselect=distributed/protocol/tests/test_protocol.py::test_deeply_nested_structures --deselect=distributed/protocol/tests/test_serialize.py::test_deeply_nested_structures --deselect=distributed/cli/tests/test_dask_scheduler.py::test_defaults --deselect=distributed/cli/tests/test_dask_scheduler.py::test_hostport --deselect=distributed/cli/tests/test_dask_spec.py::test_errors --deselect=distributed/cli/tests/test_dask_worker.py::test_scheduler_file --deselect=distributed/cli/tests/test_dask_worker.py::test_scheduler_file --deselect=distributed/cli/tests/test_dask_worker.py::test_scheduler_file --deselect=distributed/dashboard/tests/test_scheduler_bokeh.py::test_counters --deselect=distributed/dashboard/tests/test_worker_bokeh.py::test_counters --deselect=distributed/deploy/tests/test_local.py::test_adapt_then_manual --deselect=distributed/deploy/tests/test_local.py::test_async_with --deselect=distributed/deploy/tests/test_local.py::test_close_twice --deselect=distributed/deploy/tests/test_local.py::test_cluster_info_sync --deselect=distributed/deploy/tests/test_local.py::test_local_tls --deselect=distributed/deploy/tests/test_local.py::test_no_dangling_asyncio_tasks --deselect=distributed/deploy/tests/test_local.py::test_only_local_access --deselect=distributed/deploy/tests/test_local.py::test_remote_access --deselect=distributed/diagnostics/tests/test_progress_widgets.py::test_serializers --deselect=distributed/diagnostics/tests/test_scheduler_plugin.py::test_lifecycle --deselect=distributed/http/scheduler/tests/test_missing_bokeh.py::test_missing_bokeh --deselect=distributed/http/scheduler/tests/test_scheduler_http.py::test_metrics_when_prometheus_client_not_installed --deselect=distributed/protocol/tests/test_serialize.py::test_errors --deselect=distributed/tests/test_batched.py::test_BatchedSend --deselect=distributed/tests/test_batched.py::test_close_closed --deselect=distributed/tests/test_batched.py::test_close_twice --deselect=distributed/tests/test_batched.py::test_send_after_stream_start --deselect=distributed/tests/test_batched.py::test_send_before_close --deselect=distributed/tests/test_batched.py::test_send_before_start --deselect=distributed/tests/test_batched.py::test_sending_traffic_jam --deselect=distributed/tests/test_batched.py::test_serializers --deselect=distributed/tests/test_client.py::TestClientSecurityLoader::test_security_loader --deselect=distributed/tests/test_client.py::TestClientSecurityLoader::test_security_loader_ignored_if_explicit_security_provided --deselect=distributed/tests/test_client.py::TestClientSecurityLoader::test_security_loader_ignored_if_returns_none --deselect=distributed/tests/test_client.py::test_async_with --deselect=distributed/tests/test_client.py::test_client_is_quiet_cluster_close --deselect=distributed/tests/test_client.py::test_dashboard_link_cluster --deselect=distributed/tests/test_client.py::test_dashboard_link_inproc --deselect=distributed/tests/test_client.py::test_file_descriptors_dont_leak --deselect=distributed/tests/test_client.py::test_mixing_clients_different_scheduler --deselect=distributed/tests/test_client.py::test_quiet_client_close --deselect=distributed/tests/test_client.py::test_rebalance_sync --deselect=distributed/tests/test_client.py::test_repr_localcluster --deselect=distributed/tests/test_client.py::test_security_loader --deselect=distributed/tests/test_client.py::test_security_loader_ignored_if_explicit_security_provided --deselect=distributed/tests/test_client.py::test_security_loader_ignored_if_returns_none --deselect=distributed/tests/test_client.py::test_shutdown --deselect=distributed/tests/test_client.py::test_shutdown_is_quiet_with_cluster --deselect=distributed/tests/test_client.py::test_shutdown_localcluster --deselect=distributed/tests/test_client.py::test_shutdown_stops_callbacks --deselect=distributed/tests/test_client_loop.py::test_close_loop_sync_start_new_loop --deselect=distributed/tests/test_client_loop.py::test_close_loop_sync_use_running_loop --deselect=distributed/tests/test_core.py::test_close_fast_without_active_handlers --deselect=distributed/tests/test_core.py::test_close_grace_period_for_handlers --deselect=distributed/tests/test_core.py::test_close_properly --deselect=distributed/tests/test_core.py::test_compression --deselect=distributed/tests/test_core.py::test_connection_pool --deselect=distributed/tests/test_core.py::test_connection_pool_close_while_connecting --deselect=distributed/tests/test_core.py::test_connection_pool_detects_remote_close --deselect=distributed/tests/test_core.py::test_connection_pool_outside_cancellation --deselect=distributed/tests/test_core.py::test_connection_pool_remove --deselect=distributed/tests/test_core.py::test_connection_pool_respects_limit --deselect=distributed/tests/test_core.py::test_connection_pool_tls --deselect=distributed/tests/test_core.py::test_counters --deselect=distributed/tests/test_core.py::test_deserialize_error --deselect=distributed/tests/test_core.py::test_errors --deselect=distributed/tests/test_core.py::test_identity_inproc --deselect=distributed/tests/test_core.py::test_identity_tcp --deselect=distributed/tests/test_core.py::test_large_packets_inproc --deselect=distributed/tests/test_core.py::test_messages_are_ordered_bsend --deselect=distributed/tests/test_core.py::test_messages_are_ordered_raw --deselect=distributed/tests/test_core.py::test_ports --deselect=distributed/tests/test_core.py::test_rpc_default --deselect=distributed/tests/test_core.py::test_rpc_inproc --deselect=distributed/tests/test_core.py::test_rpc_message_lifetime_default --deselect=distributed/tests/test_core.py::test_rpc_message_lifetime_inproc --deselect=distributed/tests/test_core.py::test_rpc_message_lifetime_tcp --deselect=distributed/tests/test_core.py::test_rpc_serialization --deselect=distributed/tests/test_core.py::test_rpc_tcp --deselect=distributed/tests/test_core.py::test_rpc_tls --deselect=distributed/tests/test_core.py::test_rpc_with_many_connections_inproc --deselect=distributed/tests/test_core.py::test_rpc_with_many_connections_tcp --deselect=distributed/tests/test_core.py::test_send_recv_args --deselect=distributed/tests/test_core.py::test_send_recv_cancelled --deselect=distributed/tests/test_core.py::test_server --deselect=distributed/tests/test_core.py::test_server_comms_mark_active_handlers --deselect=distributed/tests/test_core.py::test_server_raises_on_blocked_handlers --deselect=distributed/tests/test_jupyter.py::test_jupyter_idle_timeout --deselect=distributed/tests/test_jupyter.py::test_jupyter_server --deselect=distributed/tests/test_locks.py::test_errors --deselect=distributed/tests/test_nanny.py::test_scheduler_file --deselect=distributed/tests/test_nanny.py::test_scheduler_file --deselect=distributed/tests/test_nanny.py::test_scheduler_file --deselect=distributed/tests/test_nanny.py::test_worker_uses_same_host_as_nanny --deselect=distributed/tests/test_preload.py::test_failure_doesnt_crash_scheduler --deselect=distributed/tests/test_preload.py::test_preload_import_time --deselect=distributed/tests/test_preload.py::test_preload_manager_sequence --deselect=distributed/tests/test_preload.py::test_worker_preload_text --deselect=distributed/tests/test_scheduler.py::test_allowed_failures_config --deselect=distributed/tests/test_scheduler.py::test_async_context_manager --deselect=distributed/tests/test_scheduler.py::test_dashboard_host --deselect=distributed/tests/test_scheduler.py::test_file_descriptors_dont_leak --deselect=distributed/tests/test_scheduler.py::test_finished --deselect=distributed/tests/test_scheduler.py::test_multiple_listeners --deselect=distributed/tests/test_scheduler.py::test_no_dangling_asyncio_tasks --deselect=distributed/tests/test_scheduler.py::test_scheduler_file --deselect=distributed/tests/test_scheduler.py::test_scheduler_file --deselect=distributed/tests/test_scheduler.py::test_scheduler_file --deselect=distributed/tests/test_security.py::test_require_encryption --deselect=distributed/tests/test_security.py::test_tls_listen_connect --deselect=distributed/tests/test_security.py::test_tls_temporary_credentials_functional --deselect=distributed/tests/test_semaphore.py::test_threadpoolworkers_pick_correct_ioloop --deselect=distributed/tests/test_tls_functional.py::test_security_dict_input_no_security --deselect=distributed/tests/test_utils_test.py::test_ensure_no_new_clients --deselect=distributed/tests/test_utils_test.py::test_freeze_batched_send --deselect=distributed/tests/test_utils_test.py::test_locked_comm_drop_in_replacement --deselect=distributed/tests/test_utils_test.py::test_locked_comm_intercept_read --deselect=distributed/tests/test_utils_test.py::test_locked_comm_intercept_write --deselect=distributed/tests/test_worker.py::test_host_uses_scheduler_protocol --deselect=distributed/tests/test_worker.py::test_plugin_exception --deselect=distributed/tests/test_worker.py::test_plugin_internal_exception --deselect=distributed/tests/test_worker.py::test_plugin_multiple_exceptions --deselect=distributed/tests/test_worker.py::test_scheduler_file --deselect=distributed/tests/test_worker.py::test_scheduler_file --deselect=distributed/tests/test_worker.py::test_scheduler_file --deselect=distributed/tests/test_worker_client.py::test_dont_override_default_get --deselect=distributed/cli/tests/test_dask_scheduler.py::test_dashboard --deselect=distributed/cli/tests/test_dask_scheduler.py::test_dashboard_allowlist --deselect=distributed/cli/tests/test_dask_scheduler.py::test_dashboard_non_standard_ports --deselect=distributed/cli/tests/test_dask_scheduler.py::test_dashboard_port_zero --deselect=distributed/cli/tests/test_dask_scheduler.py::test_defaults --deselect=distributed/cli/tests/test_dask_scheduler.py::test_hostport --deselect=distributed/cli/tests/test_dask_scheduler.py::test_multiple_protocols --deselect=distributed/cli/tests/test_dask_scheduler.py::test_multiple_workers --deselect=distributed/cli/tests/test_dask_scheduler.py::test_multiple_workers_2 --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_command --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_command_default --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_config --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_file --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_module --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_remote_module --deselect=distributed/cli/tests/test_dask_scheduler.py::test_scheduler_port_zero --deselect=distributed/cli/tests/test_dask_scheduler.py::test_single_executable_deprecated --deselect=distributed/cli/tests/test_dask_worker.py::test_contact_listen_address --deselect=distributed/cli/tests/test_dask_worker.py::test_dashboard_non_standard_ports --deselect=distributed/cli/tests/test_dask_worker.py::test_error_during_startup --deselect=distributed/cli/tests/test_dask_worker.py::test_integer_names --deselect=distributed/cli/tests/test_dask_worker.py::test_listen_address_ipv6 --deselect=distributed/cli/tests/test_dask_worker.py::test_local_directory --deselect=distributed/cli/tests/test_dask_worker.py::test_memory_limit --deselect=distributed/cli/tests/test_dask_worker.py::test_nanny_worker_port_range --deselect=distributed/cli/tests/test_dask_worker.py::test_nanny_worker_port_range_too_many_workers_raises --deselect=distributed/cli/tests/test_dask_worker.py::test_nanny_worker_ports --deselect=distributed/cli/tests/test_dask_worker.py::test_no_nanny --deselect=distributed/cli/tests/test_dask_worker.py::test_nworkers_auto --deselect=distributed/cli/tests/test_dask_worker.py::test_nworkers_expands_name --deselect=distributed/cli/tests/test_dask_worker.py::test_nworkers_negative --deselect=distributed/cli/tests/test_dask_worker.py::test_nworkers_requires_nanny --deselect=distributed/cli/tests/test_dask_worker.py::test_preload_config --deselect=distributed/cli/tests/test_dask_worker.py::test_resources --deselect=distributed/cli/tests/test_dask_worker.py::test_respect_host_listen_address --deselect=distributed/cli/tests/test_dask_worker.py::test_scheduler_address_env --deselect=distributed/cli/tests/test_dask_worker.py::test_scheduler_file --deselect=distributed/cli/tests/test_dask_worker.py::test_set_lifetime_restart_via_env_var --deselect=distributed/cli/tests/test_dask_worker.py::test_set_lifetime_stagger_via_env_var --deselect=distributed/cli/tests/test_dask_worker.py::test_signal_handling --deselect=distributed/cli/tests/test_dask_worker.py::test_single_executable_deprecated --deselect=distributed/cli/tests/test_dask_worker.py::test_single_executable_works --deselect=distributed/cli/tests/test_dask_worker.py::test_timeout --deselect=distributed/cli/tests/test_dask_worker.py::test_worker_class --deselect=distributed/tests/test_config.py::test_logging_extended --deselect=distributed/tests/test_config.py::test_logging_file_config --deselect=distributed/tests/test_config.py::test_logging_mutual_exclusive --deselect=distributed/tests/test_config.py::test_logging_simple --deselect=distributed/tests/test_config.py::test_logging_simple_under_distributed --deselect=distributed/tests/test_queues.py::test_queue_in_task --deselect=distributed/tests/test_variable.py::test_variable_in_task +============================= test session starts ============================== +platform linux -- Python 3.13.7, pytest-8.3.5, pluggy-1.6.0 -- /usr/bin/python3.13 +cachedir: .pytest_cache +rootdir: /build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build +configfile: pyproject.toml +plugins: timeout-2.3.1, rerunfailures-15.1, typeguard-4.4.4, anyio-4.8.0 +timeout: 300.0s +timeout method: signal +timeout func_only: False +collecting ... collected 4 items +run-last-failure: rerun previous 4 failures (skipped 146 files) + +distributed/tests/test_nanny.py::test_nanny_closes_cleanly_if_worker_is_terminated PASSED [ 25%] +distributed/tests/test_steal.py::test_restart PASSED [ 50%] +distributed/tests/test_tls_functional.py::test_nanny PASSED [ 75%] +distributed/tests/test_tls_functional.py::test_retire_workers FAILED [100%] + +=================================== FAILURES =================================== +_____________________________ test_retire_workers ______________________________ + +fut = , timeout = 0 + + async def wait_for(fut: Awaitable[T], timeout: float) -> T: + async with asyncio.timeout(timeout): +> return await fut + +distributed/utils.py:1914: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +distributed/nanny.py:358: in start_unsafe + comm = await self.rpc.connect(saddr) +distributed/core.py:1485: in connect + return await self._connect(addr=addr, timeout=timeout) +distributed/core.py:1429: in _connect + comm = await connect( +distributed/comm/core.py:342: in connect + comm = await wait_for( +distributed/utils.py:1914: in wait_for + return await fut +distributed/comm/tcp.py:546: in connect + stream = await self.client.connect( +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +host = '127.0.0.1', port = 36035, af = +ssl_options = +max_buffer_size = 16798425088.0, source_ip = None, source_port = None +timeout = None + + async def connect( + self, + host: str, + port: int, + af: socket.AddressFamily = socket.AF_UNSPEC, + ssl_options: Optional[Union[Dict[str, Any], ssl.SSLContext]] = None, + max_buffer_size: Optional[int] = None, + source_ip: Optional[str] = None, + source_port: Optional[int] = None, + timeout: Optional[Union[float, datetime.timedelta]] = None, + ) -> IOStream: + """Connect to the given host and port. + + Asynchronously returns an `.IOStream` (or `.SSLIOStream` if + ``ssl_options`` is not None). + + Using the ``source_ip`` kwarg, one can specify the source + IP address to use when establishing the connection. + In case the user needs to resolve and + use a specific interface, it has to be handled outside + of Tornado as this depends very much on the platform. + + Raises `TimeoutError` if the input future does not complete before + ``timeout``, which may be specified in any form allowed by + `.IOLoop.add_timeout` (i.e. a `datetime.timedelta` or an absolute time + relative to `.IOLoop.time`) + + Similarly, when the user requires a certain source port, it can + be specified using the ``source_port`` arg. + + .. versionchanged:: 4.5 + Added the ``source_ip`` and ``source_port`` arguments. + + .. versionchanged:: 5.0 + Added the ``timeout`` argument. + """ + if timeout is not None: + if isinstance(timeout, numbers.Real): + timeout = IOLoop.current().time() + timeout + elif isinstance(timeout, datetime.timedelta): + timeout = IOLoop.current().time() + timeout.total_seconds() + else: + raise TypeError("Unsupported timeout %r" % timeout) + if timeout is not None: + addrinfo = await gen.with_timeout( + timeout, self.resolver.resolve(host, port, af) + ) + else: + addrinfo = await self.resolver.resolve(host, port, af) + connector = _Connector( + addrinfo, + functools.partial( + self._create_stream, + max_buffer_size, + source_ip=source_ip, + source_port=source_port, + ), + ) +> af, addr, stream = await connector.start(connect_timeout=timeout) +E asyncio.exceptions.CancelledError + +/usr/lib/python3/dist-packages/tornado/tcpclient.py:279: CancelledError + +The above exception was the direct cause of the following exception: + +self = + + @final + async def start(self): + async with self._startup_lock: + if self.status == Status.failed: + assert self.__startup_exc is not None + raise self.__startup_exc + elif self.status != Status.init: + return self + timeout = getattr(self, "death_timeout", None) + + async def _close_on_failure(exc: Exception) -> None: + await self.close(reason=f"failure-to-start-{str(type(exc))}") + self.status = Status.failed + self.__startup_exc = exc + + try: +> await wait_for(self.start_unsafe(), timeout=timeout) + +distributed/core.py:528: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +distributed/utils.py:1913: in wait_for + async with asyncio.timeout(timeout): +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +exc_type = +exc_val = CancelledError(), exc_tb = + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> Optional[bool]: + assert self._state in (_State.ENTERED, _State.EXPIRING) + + if self._timeout_handler is not None: + self._timeout_handler.cancel() + self._timeout_handler = None + + if self._state is _State.EXPIRING: + self._state = _State.EXPIRED + + if self._task.uncancel() <= self._cancelling and exc_type is not None: + # Since there are no new cancel requests, we're + # handling this. + if issubclass(exc_type, exceptions.CancelledError): +> raise TimeoutError from exc_val +E TimeoutError + +/usr/lib/python3.13/asyncio/timeouts.py:116: TimeoutError + +The above exception was the direct cause of the following exception: + + @contextlib.asynccontextmanager + async def _cluster_factory(): + workers = [] + s = None + try: + for _ in range(60): + try: +> s, ws = await start_cluster( + nthreads, + scheduler, + security=security, + Worker=Worker, + scheduler_kwargs=scheduler_kwargs, + worker_kwargs=merge( + {"death_timeout": min(15, int(deadline.remaining))}, + worker_kwargs, + ), + ) + +distributed/utils_test.py:974: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +distributed/utils_test.py:791: in start_cluster + await asyncio.gather(*workers) +/usr/lib/python3.13/asyncio/tasks.py:737: in _wrap_awaitable + return await awaitable +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = + + @final + async def start(self): + async with self._startup_lock: + if self.status == Status.failed: + assert self.__startup_exc is not None + raise self.__startup_exc + elif self.status != Status.init: + return self + timeout = getattr(self, "death_timeout", None) + + async def _close_on_failure(exc: Exception) -> None: + await self.close(reason=f"failure-to-start-{str(type(exc))}") + self.status = Status.failed + self.__startup_exc = exc + + try: + await wait_for(self.start_unsafe(), timeout=timeout) + except asyncio.TimeoutError as exc: + await _close_on_failure(exc) +> raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +E TimeoutError: Nanny start timed out after 0s. + +distributed/core.py:531: TimeoutError + +During handling of the above exception, another exception occurred: + +fut = ._..test_func..async_fn at 0xffffb02a7680> +timeout = 60 + + async def wait_for(fut: Awaitable[T], timeout: float) -> T: + async with asyncio.timeout(timeout): +> return await fut + +distributed/utils.py:1914: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +distributed/utils_test.py:1007: in async_fn + _cluster_factory() as (s, workers), +/usr/lib/python3.13/contextlib.py:214: in __aenter__ + return await anext(self.gen) +distributed/utils_test.py:991: in _cluster_factory + await asyncio.sleep(1) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +delay = 1, result = None + + async def sleep(delay, result=None): + """Coroutine that completes after a given time (in seconds).""" + if delay <= 0: + await __sleep0() + return result + + if math.isnan(delay): + raise ValueError("Invalid delay: NaN (not a number)") + + loop = events.get_running_loop() + future = loop.create_future() + h = loop.call_later(delay, + futures._set_result_unless_cancelled, + future, result) + try: +> return await future +E asyncio.exceptions.CancelledError + +/usr/lib/python3.13/asyncio/tasks.py:718: CancelledError + +The above exception was the direct cause of the following exception: + +args = (), kwds = {} + + @wraps(func) + def inner(*args, **kwds): + with self._recreate_cm(): +> return func(*args, **kwds) + +/usr/lib/python3.13/contextlib.py:85: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +/usr/lib/python3.13/contextlib.py:85: in inner + return func(*args, **kwds) +distributed/utils_test.py:1090: in test_func + return _run_and_close_tornado(async_fn_outer) +distributed/utils_test.py:380: in _run_and_close_tornado + return asyncio_run(inner_fn(), loop_factory=get_loop_factory()) +/usr/lib/python3.13/asyncio/runners.py:195: in run + return runner.run(main) +/usr/lib/python3.13/asyncio/runners.py:118: in run + return self._loop.run_until_complete(task) +/usr/lib/python3.13/asyncio/base_events.py:725: in run_until_complete + return future.result() +distributed/utils_test.py:377: in inner_fn + return await async_fn(*args, **kwargs) +distributed/utils_test.py:1087: in async_fn_outer + return await utils_wait_for(async_fn(), timeout=timeout * 2) +distributed/utils.py:1913: in wait_for + async with asyncio.timeout(timeout): +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +exc_type = +exc_val = CancelledError(), exc_tb = + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> Optional[bool]: + assert self._state in (_State.ENTERED, _State.EXPIRING) + + if self._timeout_handler is not None: + self._timeout_handler.cancel() + self._timeout_handler = None + + if self._state is _State.EXPIRING: + self._state = _State.EXPIRED + + if self._task.uncancel() <= self._cancelling and exc_type is not None: + # Since there are no new cancel requests, we're + # handling this. + if issubclass(exc_type, exceptions.CancelledError): +> raise TimeoutError from exc_val +E TimeoutError + +/usr/lib/python3.13/asyncio/timeouts.py:116: TimeoutError +----------------------------- Captured stderr call ----------------------------- +2025-09-03 03:39:27,620 - distributed.scheduler - INFO - State start +2025-09-03 03:39:27,623 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:39889 +2025-09-03 03:39:27,623 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:44201/status +2025-09-03 03:39:27,623 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:39:27,663 - distributed.nanny - INFO - Start Nanny at: 'tls://127.0.0.1:42489' +2025-09-03 03:39:27,665 - distributed.nanny - INFO - Start Nanny at: 'tls://127.0.0.1:33455' +2025-09-03 03:39:28,227 - distributed.worker - INFO - Start worker at: tls://127.0.0.1:37049 +2025-09-03 03:39:28,227 - distributed.worker - INFO - Listening to: tls://127.0.0.1:37049 +2025-09-03 03:39:28,227 - distributed.worker - INFO - Worker name: 0 +2025-09-03 03:39:28,228 - distributed.worker - INFO - dashboard at: 127.0.0.1:34101 +2025-09-03 03:39:28,228 - distributed.worker - INFO - Waiting to connect to: tls://127.0.0.1:39889 +2025-09-03 03:39:28,228 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:39:28,228 - distributed.worker - INFO - Threads: 1 +2025-09-03 03:39:28,228 - distributed.worker - INFO - Memory: 31.29 GiB +2025-09-03 03:39:28,228 - distributed.worker - INFO - Local Directory: /tmp/dask-scratch-space/worker-hjp6n3hq +2025-09-03 03:39:28,228 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:39:28,266 - distributed.worker - INFO - Start worker at: tls://127.0.0.1:36257 +2025-09-03 03:39:28,266 - distributed.worker - INFO - Listening to: tls://127.0.0.1:36257 +2025-09-03 03:39:28,267 - distributed.worker - INFO - Worker name: 1 +2025-09-03 03:39:28,267 - distributed.worker - INFO - dashboard at: 127.0.0.1:44815 +2025-09-03 03:39:28,267 - distributed.worker - INFO - Waiting to connect to: tls://127.0.0.1:39889 +2025-09-03 03:39:28,267 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:39:28,267 - distributed.worker - INFO - Threads: 2 +2025-09-03 03:39:28,267 - distributed.worker - INFO - Memory: 31.29 GiB +2025-09-03 03:39:28,267 - distributed.worker - INFO - Local Directory: /tmp/dask-scratch-space/worker-a22n_th8 +2025-09-03 03:39:28,267 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:39:28,521 - distributed.scheduler - INFO - Register worker addr: tls://127.0.0.1:37049 name: 0 +2025-09-03 03:39:28,523 - distributed.worker - INFO - Starting Worker plugin shuffle +2025-09-03 03:39:28,524 - distributed.worker - INFO - Registered to: tls://127.0.0.1:39889 +2025-09-03 03:39:28,524 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:39:28,524 - distributed.scheduler - INFO - Starting worker compute stream, tls://127.0.0.1:37049 +2025-09-03 03:39:28,524 - distributed.core - INFO - Starting established connection to tls://127.0.0.1:43690 +2025-09-03 03:39:28,525 - distributed.core - INFO - Starting established connection to tls://127.0.0.1:39889 +2025-09-03 03:39:28,555 - distributed.scheduler - INFO - Register worker addr: tls://127.0.0.1:36257 name: 1 +2025-09-03 03:39:28,557 - distributed.worker - INFO - Starting Worker plugin shuffle +2025-09-03 03:39:28,558 - distributed.worker - INFO - Registered to: tls://127.0.0.1:39889 +2025-09-03 03:39:28,558 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:39:28,558 - distributed.scheduler - INFO - Starting worker compute stream, tls://127.0.0.1:36257 +2025-09-03 03:39:28,558 - distributed.core - INFO - Starting established connection to tls://127.0.0.1:43694 +2025-09-03 03:39:28,559 - distributed.core - INFO - Connection to tls://127.0.0.1:43694 has been closed. +2025-09-03 03:39:28,559 - distributed.scheduler - INFO - Remove worker addr: tls://127.0.0.1:36257 name: 1 (stimulus_id='handle-worker-cleanup-1756820368.5595162') +2025-09-03 03:39:28,560 - distributed.core - INFO - Starting established connection to tls://127.0.0.1:39889 +2025-09-03 03:39:28,560 - distributed.core - INFO - Connection to tls://127.0.0.1:39889 has been closed. +2025-09-03 03:39:28,561 - distributed.worker - INFO - Stopping worker at tls://127.0.0.1:36257. Reason: worker-handle-scheduler-connection-broken +2025-09-03 03:39:28,596 - distributed.nanny - INFO - Closing Nanny gracefully at 'tls://127.0.0.1:33455'. Reason: worker-handle-scheduler-connection-broken +2025-09-03 03:39:28,597 - distributed.worker - INFO - Removing Worker plugin shuffle +2025-09-03 03:39:28,599 - distributed.nanny - INFO - Worker closed +2025-09-03 03:39:30,886 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:33455'. Reason: nanny-close-gracefully +2025-09-03 03:39:30,886 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:33455' closed. +2025-09-03 03:39:58,602 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:42489'. Reason: nanny-close +2025-09-03 03:39:58,602 - distributed.nanny - INFO - Nanny asking worker to close. Reason: nanny-close +2025-09-03 03:39:58,603 - distributed.worker - INFO - Stopping worker at tls://127.0.0.1:37049. Reason: nanny-close +2025-09-03 03:39:58,603 - distributed.worker - INFO - Removing Worker plugin shuffle +2025-09-03 03:39:58,605 - distributed.core - INFO - Connection to tls://127.0.0.1:39889 has been closed. +2025-09-03 03:39:58,606 - distributed.core - INFO - Received 'close-stream' from tls://127.0.0.1:43690; closing. +2025-09-03 03:39:58,607 - distributed.scheduler - INFO - Remove worker addr: tls://127.0.0.1:37049 name: 0 (stimulus_id='handle-worker-cleanup-1756820398.6070464') +2025-09-03 03:39:58,607 - distributed.scheduler - INFO - Lost all workers +2025-09-03 03:39:58,608 - distributed.nanny - INFO - Worker closed +2025-09-03 03:39:58,904 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:42489' closed. +2025-09-03 03:39:58,904 - distributed.scheduler - INFO - Closing scheduler. Reason: unknown +2025-09-03 03:39:58,904 - distributed.scheduler - INFO - Scheduler closing all comms +2025-09-03 03:39:58,905 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Cluster creation timeout; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 806, in start_cluster + raise TimeoutError("Cluster creation timeout") +TimeoutError: Cluster creation timeout +2025-09-03 03:39:59,910 - distributed.scheduler - INFO - State start +2025-09-03 03:39:59,914 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:36187 +2025-09-03 03:39:59,914 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:45313/status +2025-09-03 03:39:59,914 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:39:59,926 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:44771'. Reason: failure-to-start- +2025-09-03 03:39:59,927 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:44771' closed. +2025-09-03 03:39:59,927 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:40671'. Reason: failure-to-start- +2025-09-03 03:39:59,927 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:40671' closed. +2025-09-03 03:39:59,927 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:36187': TLS handshake failed with remote 'tls://127.0.0.1:40650': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:39:59,927 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:36187': TLS handshake failed with remote 'tls://127.0.0.1:40654': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:39:59,927 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:40:00,936 - distributed.scheduler - INFO - State start +2025-09-03 03:40:00,939 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:40139 +2025-09-03 03:40:00,939 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:35785/status +2025-09-03 03:40:00,939 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:40:00,947 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:35355'. Reason: failure-to-start- +2025-09-03 03:40:00,947 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:35355' closed. +2025-09-03 03:40:00,947 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:45777'. Reason: failure-to-start- +2025-09-03 03:40:00,947 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:45777' closed. +2025-09-03 03:40:00,948 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:40139': TLS handshake failed with remote 'tls://127.0.0.1:41294': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:00,948 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:40139': TLS handshake failed with remote 'tls://127.0.0.1:41304': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:00,948 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:40:01,954 - distributed.scheduler - INFO - State start +2025-09-03 03:40:01,957 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:37859 +2025-09-03 03:40:01,957 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:45567/status +2025-09-03 03:40:01,958 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:40:01,969 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:37601'. Reason: failure-to-start- +2025-09-03 03:40:01,970 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:37601' closed. +2025-09-03 03:40:01,970 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:33041'. Reason: failure-to-start- +2025-09-03 03:40:01,970 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:33041' closed. +2025-09-03 03:40:01,970 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:37859': TLS handshake failed with remote 'tls://127.0.0.1:33368': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:01,970 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:37859': TLS handshake failed with remote 'tls://127.0.0.1:33374': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:01,970 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:40:02,975 - distributed.scheduler - INFO - State start +2025-09-03 03:40:02,979 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:42601 +2025-09-03 03:40:02,979 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:36063/status +2025-09-03 03:40:02,979 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:40:02,991 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:44379'. Reason: failure-to-start- +2025-09-03 03:40:02,991 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:44379' closed. +2025-09-03 03:40:02,991 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:45615'. Reason: failure-to-start- +2025-09-03 03:40:02,991 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:45615' closed. +2025-09-03 03:40:02,992 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:42601': TLS handshake failed with remote 'tls://127.0.0.1:52178': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:02,992 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:42601': TLS handshake failed with remote 'tls://127.0.0.1:52188': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:02,992 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:40:03,997 - distributed.scheduler - INFO - State start +2025-09-03 03:40:04,001 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:42657 +2025-09-03 03:40:04,001 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:46801/status +2025-09-03 03:40:04,001 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:40:04,013 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:45371'. Reason: failure-to-start- +2025-09-03 03:40:04,013 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:45371' closed. +2025-09-03 03:40:04,013 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:42489'. Reason: failure-to-start- +2025-09-03 03:40:04,014 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:42489' closed. +2025-09-03 03:40:04,014 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:42657': TLS handshake failed with remote 'tls://127.0.0.1:48456': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:04,014 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:42657': TLS handshake failed with remote 'tls://127.0.0.1:48468': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:04,014 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:40:05,018 - distributed.scheduler - INFO - State start +2025-09-03 03:40:05,025 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:37651 +2025-09-03 03:40:05,026 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:38333/status +2025-09-03 03:40:05,026 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:40:05,034 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:46573'. Reason: failure-to-start- +2025-09-03 03:40:05,034 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:46573' closed. +2025-09-03 03:40:05,034 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:40675'. Reason: failure-to-start- +2025-09-03 03:40:05,034 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:40675' closed. +2025-09-03 03:40:05,034 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:37651': TLS handshake failed with remote 'tls://127.0.0.1:51170': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:05,035 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:37651': TLS handshake failed with remote 'tls://127.0.0.1:51178': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:05,035 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:40:06,043 - distributed.scheduler - INFO - State start +2025-09-03 03:40:06,060 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:35015 +2025-09-03 03:40:06,060 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:43579/status +2025-09-03 03:40:06,064 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:40:06,077 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:34155'. Reason: failure-to-start- +2025-09-03 03:40:06,077 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:34155' closed. +2025-09-03 03:40:06,077 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:42719'. Reason: failure-to-start- +2025-09-03 03:40:06,077 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:42719' closed. +2025-09-03 03:40:06,078 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:35015': TLS handshake failed with remote 'tls://127.0.0.1:47820': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:06,078 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:35015': TLS handshake failed with remote 'tls://127.0.0.1:47826': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:06,078 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:40:07,082 - distributed.scheduler - INFO - State start +2025-09-03 03:40:07,090 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:36855 +2025-09-03 03:40:07,090 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:38747/status +2025-09-03 03:40:07,090 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:40:07,102 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:33741'. Reason: failure-to-start- +2025-09-03 03:40:07,102 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:33741' closed. +2025-09-03 03:40:07,102 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:36621'. Reason: failure-to-start- +2025-09-03 03:40:07,102 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:36621' closed. +2025-09-03 03:40:07,103 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:36855': TLS handshake failed with remote 'tls://127.0.0.1:59864': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:07,103 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:36855': TLS handshake failed with remote 'tls://127.0.0.1:59876': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:07,103 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:40:08,107 - distributed.scheduler - INFO - State start +2025-09-03 03:40:08,112 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:39141 +2025-09-03 03:40:08,120 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:33749/status +2025-09-03 03:40:08,121 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:40:08,132 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:43395'. Reason: failure-to-start- +2025-09-03 03:40:08,132 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:43395' closed. +2025-09-03 03:40:08,133 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:41905'. Reason: failure-to-start- +2025-09-03 03:40:08,133 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:41905' closed. +2025-09-03 03:40:08,133 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:39141': TLS handshake failed with remote 'tls://127.0.0.1:54848': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:08,133 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:39141': TLS handshake failed with remote 'tls://127.0.0.1:54854': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:08,133 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:40:09,138 - distributed.scheduler - INFO - State start +2025-09-03 03:40:09,145 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:41597 +2025-09-03 03:40:09,146 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:34487/status +2025-09-03 03:40:09,146 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:40:09,158 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:43209'. Reason: failure-to-start- +2025-09-03 03:40:09,158 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:43209' closed. +2025-09-03 03:40:09,158 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:44225'. Reason: failure-to-start- +2025-09-03 03:40:09,158 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:44225' closed. +2025-09-03 03:40:09,158 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:41597': TLS handshake failed with remote 'tls://127.0.0.1:34402': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:09,159 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:41597': TLS handshake failed with remote 'tls://127.0.0.1:34404': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:09,159 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:40:10,163 - distributed.scheduler - INFO - State start +2025-09-03 03:40:10,167 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:34573 +2025-09-03 03:40:10,167 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:45431/status +2025-09-03 03:40:10,167 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:40:10,176 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:34995'. Reason: failure-to-start- +2025-09-03 03:40:10,176 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:34995' closed. +2025-09-03 03:40:10,176 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:45751'. Reason: failure-to-start- +2025-09-03 03:40:10,176 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:45751' closed. +2025-09-03 03:40:10,177 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:34573': TLS handshake failed with remote 'tls://127.0.0.1:53748': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:10,177 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:34573': TLS handshake failed with remote 'tls://127.0.0.1:53762': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:10,177 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:40:11,182 - distributed.scheduler - INFO - State start +2025-09-03 03:40:11,186 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:40013 +2025-09-03 03:40:11,186 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:39769/status +2025-09-03 03:40:11,186 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:40:11,194 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:44123'. Reason: failure-to-start- +2025-09-03 03:40:11,194 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:44123' closed. +2025-09-03 03:40:11,194 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:40437'. Reason: failure-to-start- +2025-09-03 03:40:11,194 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:40437' closed. +2025-09-03 03:40:11,195 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:40013': TLS handshake failed with remote 'tls://127.0.0.1:57910': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:11,195 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:40013': TLS handshake failed with remote 'tls://127.0.0.1:57924': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:11,195 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:40:12,199 - distributed.scheduler - INFO - State start +2025-09-03 03:40:12,202 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:37121 +2025-09-03 03:40:12,203 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:34109/status +2025-09-03 03:40:12,203 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:40:12,212 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:33551'. Reason: failure-to-start- +2025-09-03 03:40:12,212 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:33551' closed. +2025-09-03 03:40:12,212 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:44319'. Reason: failure-to-start- +2025-09-03 03:40:12,213 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:44319' closed. +2025-09-03 03:40:12,213 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:37121': TLS handshake failed with remote 'tls://127.0.0.1:33820': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:12,213 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:37121': TLS handshake failed with remote 'tls://127.0.0.1:33828': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:12,213 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:40:13,217 - distributed.scheduler - INFO - State start +2025-09-03 03:40:13,220 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:40323 +2025-09-03 03:40:13,221 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:33711/status +2025-09-03 03:40:13,221 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:40:13,228 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:37209'. Reason: failure-to-start- +2025-09-03 03:40:13,228 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:37209' closed. +2025-09-03 03:40:13,228 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:38897'. Reason: failure-to-start- +2025-09-03 03:40:13,229 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:38897' closed. +2025-09-03 03:40:13,229 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:40323': TLS handshake failed with remote 'tls://127.0.0.1:40366': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:13,229 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:40323': TLS handshake failed with remote 'tls://127.0.0.1:40372': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:13,229 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:40:14,234 - distributed.scheduler - INFO - State start +2025-09-03 03:40:14,241 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:43687 +2025-09-03 03:40:14,241 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:46011/status +2025-09-03 03:40:14,241 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:40:14,250 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:46819'. Reason: failure-to-start- +2025-09-03 03:40:14,250 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:46819' closed. +2025-09-03 03:40:14,250 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:43133'. Reason: failure-to-start- +2025-09-03 03:40:14,250 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:43133' closed. +2025-09-03 03:40:14,251 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:43687': TLS handshake failed with remote 'tls://127.0.0.1:57424': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:14,251 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:43687': TLS handshake failed with remote 'tls://127.0.0.1:57426': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:14,251 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:40:15,255 - distributed.scheduler - INFO - State start +2025-09-03 03:40:15,258 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:36843 +2025-09-03 03:40:15,259 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:44609/status +2025-09-03 03:40:15,259 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:40:15,275 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:45819'. Reason: failure-to-start- +2025-09-03 03:40:15,275 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:45819' closed. +2025-09-03 03:40:15,275 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:32983'. Reason: failure-to-start- +2025-09-03 03:40:15,275 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:32983' closed. +2025-09-03 03:40:15,276 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:36843': TLS handshake failed with remote 'tls://127.0.0.1:57624': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:15,276 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:36843': TLS handshake failed with remote 'tls://127.0.0.1:57632': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:15,276 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:40:16,285 - distributed.scheduler - INFO - State start +2025-09-03 03:40:16,288 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:33825 +2025-09-03 03:40:16,288 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:46685/status +2025-09-03 03:40:16,288 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:40:16,297 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:37451'. Reason: failure-to-start- +2025-09-03 03:40:16,297 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:37451' closed. +2025-09-03 03:40:16,297 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:38947'. Reason: failure-to-start- +2025-09-03 03:40:16,297 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:38947' closed. +2025-09-03 03:40:16,298 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:33825': TLS handshake failed with remote 'tls://127.0.0.1:38812': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:16,298 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:33825': TLS handshake failed with remote 'tls://127.0.0.1:38814': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:16,298 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:40:17,303 - distributed.scheduler - INFO - State start +2025-09-03 03:40:17,306 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:45105 +2025-09-03 03:40:17,307 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:40231/status +2025-09-03 03:40:17,307 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:40:17,315 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:44373'. Reason: failure-to-start- +2025-09-03 03:40:17,315 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:44373' closed. +2025-09-03 03:40:17,315 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:35217'. Reason: failure-to-start- +2025-09-03 03:40:17,315 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:35217' closed. +2025-09-03 03:40:17,316 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:45105': TLS handshake failed with remote 'tls://127.0.0.1:36086': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:17,316 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:45105': TLS handshake failed with remote 'tls://127.0.0.1:36090': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:17,316 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:40:18,320 - distributed.scheduler - INFO - State start +2025-09-03 03:40:18,324 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:34905 +2025-09-03 03:40:18,324 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:39201/status +2025-09-03 03:40:18,324 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:40:18,337 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:36465'. Reason: failure-to-start- +2025-09-03 03:40:18,337 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:36465' closed. +2025-09-03 03:40:18,337 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:41981'. Reason: failure-to-start- +2025-09-03 03:40:18,337 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:41981' closed. +2025-09-03 03:40:18,338 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:34905': TLS handshake failed with remote 'tls://127.0.0.1:48110': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:18,338 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:34905': TLS handshake failed with remote 'tls://127.0.0.1:48126': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:18,338 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:40:19,342 - distributed.scheduler - INFO - State start +2025-09-03 03:40:19,349 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:34257 +2025-09-03 03:40:19,350 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:36655/status +2025-09-03 03:40:19,350 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:40:19,357 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:33351'. Reason: failure-to-start- +2025-09-03 03:40:19,358 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:33351' closed. +2025-09-03 03:40:19,358 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:42383'. Reason: failure-to-start- +2025-09-03 03:40:19,358 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:42383' closed. +2025-09-03 03:40:19,358 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:34257': TLS handshake failed with remote 'tls://127.0.0.1:57812': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:19,358 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:34257': TLS handshake failed with remote 'tls://127.0.0.1:57820': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:19,358 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:40:20,363 - distributed.scheduler - INFO - State start +2025-09-03 03:40:20,366 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:37957 +2025-09-03 03:40:20,366 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:44631/status +2025-09-03 03:40:20,366 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:40:20,379 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:33983'. Reason: failure-to-start- +2025-09-03 03:40:20,379 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:33983' closed. +2025-09-03 03:40:20,379 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:44311'. Reason: failure-to-start- +2025-09-03 03:40:20,379 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:44311' closed. +2025-09-03 03:40:20,380 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:37957': TLS handshake failed with remote 'tls://127.0.0.1:38594': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:20,380 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:37957': TLS handshake failed with remote 'tls://127.0.0.1:38604': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:20,380 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:40:21,385 - distributed.scheduler - INFO - State start +2025-09-03 03:40:21,388 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:45897 +2025-09-03 03:40:21,388 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:34315/status +2025-09-03 03:40:21,388 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:40:21,396 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:45807'. Reason: failure-to-start- +2025-09-03 03:40:21,396 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:45807' closed. +2025-09-03 03:40:21,396 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:33225'. Reason: failure-to-start- +2025-09-03 03:40:21,396 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:33225' closed. +2025-09-03 03:40:21,397 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:45897': TLS handshake failed with remote 'tls://127.0.0.1:40094': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:21,397 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:45897': TLS handshake failed with remote 'tls://127.0.0.1:40096': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:21,397 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:40:22,402 - distributed.scheduler - INFO - State start +2025-09-03 03:40:22,406 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:40895 +2025-09-03 03:40:22,406 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:33759/status +2025-09-03 03:40:22,407 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:40:22,420 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:44109'. Reason: failure-to-start- +2025-09-03 03:40:22,420 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:44109' closed. +2025-09-03 03:40:22,420 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:36283'. Reason: failure-to-start- +2025-09-03 03:40:22,420 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:36283' closed. +2025-09-03 03:40:22,421 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:40895': TLS handshake failed with remote 'tls://127.0.0.1:47110': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:22,421 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:40895': TLS handshake failed with remote 'tls://127.0.0.1:47112': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:22,421 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:40:23,425 - distributed.scheduler - INFO - State start +2025-09-03 03:40:23,428 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:34747 +2025-09-03 03:40:23,429 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:37929/status +2025-09-03 03:40:23,429 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:40:23,441 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:36337'. Reason: failure-to-start- +2025-09-03 03:40:23,441 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:36337' closed. +2025-09-03 03:40:23,441 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:46555'. Reason: failure-to-start- +2025-09-03 03:40:23,442 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:46555' closed. +2025-09-03 03:40:23,443 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:34747': TLS handshake failed with remote 'tls://127.0.0.1:56080': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:23,443 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:34747': TLS handshake failed with remote 'tls://127.0.0.1:56086': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:23,443 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:40:24,447 - distributed.scheduler - INFO - State start +2025-09-03 03:40:24,456 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:46821 +2025-09-03 03:40:24,456 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:46539/status +2025-09-03 03:40:24,457 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:40:24,465 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:39401'. Reason: failure-to-start- +2025-09-03 03:40:24,466 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:39401' closed. +2025-09-03 03:40:24,466 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:36857'. Reason: failure-to-start- +2025-09-03 03:40:24,466 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:36857' closed. +2025-09-03 03:40:24,466 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:46821': TLS handshake failed with remote 'tls://127.0.0.1:49470': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:24,466 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:46821': TLS handshake failed with remote 'tls://127.0.0.1:49472': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:24,467 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:40:25,471 - distributed.scheduler - INFO - State start +2025-09-03 03:40:25,477 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:45311 +2025-09-03 03:40:25,477 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:38275/status +2025-09-03 03:40:25,477 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:40:25,493 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:44275'. Reason: failure-to-start- +2025-09-03 03:40:25,493 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:44275' closed. +2025-09-03 03:40:25,493 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:36539'. Reason: failure-to-start- +2025-09-03 03:40:25,493 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:36539' closed. +2025-09-03 03:40:25,494 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:45311': TLS handshake failed with remote 'tls://127.0.0.1:33056': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:25,494 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:45311': TLS handshake failed with remote 'tls://127.0.0.1:33058': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:25,494 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:40:26,498 - distributed.scheduler - INFO - State start +2025-09-03 03:40:26,507 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:39227 +2025-09-03 03:40:26,507 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:42145/status +2025-09-03 03:40:26,508 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:40:26,516 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:41773'. Reason: failure-to-start- +2025-09-03 03:40:26,517 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:41773' closed. +2025-09-03 03:40:26,517 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:44803'. Reason: failure-to-start- +2025-09-03 03:40:26,517 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:44803' closed. +2025-09-03 03:40:26,517 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:39227': TLS handshake failed with remote 'tls://127.0.0.1:47464': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:26,517 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:39227': TLS handshake failed with remote 'tls://127.0.0.1:47474': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:26,517 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:40:27,522 - distributed.scheduler - INFO - State start +2025-09-03 03:40:27,525 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:36035 +2025-09-03 03:40:27,526 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:40029/status +2025-09-03 03:40:27,526 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:40:27,539 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:40015'. Reason: failure-to-start- +2025-09-03 03:40:27,539 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:40015' closed. +2025-09-03 03:40:27,539 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:37987'. Reason: failure-to-start- +2025-09-03 03:40:27,539 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:37987' closed. +2025-09-03 03:40:27,540 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:36035': TLS handshake failed with remote 'tls://127.0.0.1:37092': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:27,540 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:36035': TLS handshake failed with remote 'tls://127.0.0.1:37096': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:40:27,541 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +============================= slowest 20 durations ============================= +60.00s call distributed/tests/test_tls_functional.py::test_retire_workers +3.43s call distributed/tests/test_nanny.py::test_nanny_closes_cleanly_if_worker_is_terminated +2.88s call distributed/tests/test_steal.py::test_restart +1.67s call distributed/tests/test_tls_functional.py::test_nanny +0.00s setup distributed/tests/test_steal.py::test_restart +0.00s setup distributed/tests/test_nanny.py::test_nanny_closes_cleanly_if_worker_is_terminated +0.00s teardown distributed/tests/test_tls_functional.py::test_retire_workers +0.00s teardown distributed/tests/test_tls_functional.py::test_nanny +0.00s teardown distributed/tests/test_steal.py::test_restart +0.00s teardown distributed/tests/test_nanny.py::test_nanny_closes_cleanly_if_worker_is_terminated +0.00s setup distributed/tests/test_tls_functional.py::test_retire_workers +0.00s setup distributed/tests/test_tls_functional.py::test_nanny +=========================== short test summary info ============================ +FAILED distributed/tests/test_tls_functional.py::test_retire_workers - TimeoutError +==================== 1 failed, 3 passed in 68.67s (0:01:08) ==================== +*** END OF RUN 2: NOT ALL TESTS HAVE YET PASSED/XFAILED *** +*** STARTING RUN 3: python3.13 -m pytest --pyargs distributed --verbose --color=no --timeout-method=signal --timeout=300 -m not avoid_ci -rfE --last-failed --last-failed-no-failures none --ignore=distributed/comm/tests/test_comms.py --ignore=distributed/comm/tests/test_ws.py --ignore=distributed/deploy/tests/test_adaptive.py --ignore=distributed/deploy/tests/test_local.py --ignore=distributed/deploy/tests/test_slow_adaptive.py --ignore=distributed/deploy/tests/test_spec_cluster.py --deselect=distributed/cli/tests/test_dask_scheduler.py::test_no_dashboard --deselect=distributed/deploy/tests/test_local.py::test_localcluster_get_client --deselect=distributed/deploy/tests/test_old_ssh.py::test_cluster --deselect=distributed/deploy/tests/test_old_ssh.py::test_old_ssh_nprocs_renamed_to_n_workers --deselect=distributed/deploy/tests/test_old_ssh.py::test_nprocs_attribute_is_deprecated --deselect=distributed/deploy/tests/test_ssh.py::test_nprocs_attribute_is_deprecated --deselect=distributed/http/tests/test_core.py::test_prometheus_api_doc --deselect=distributed/tests/test_init.py::test_git_revision --deselect=distributed/tests/test_jupyter.py::test_jupyter_idle_timeout --deselect=distributed/tests/test_jupyter.py::test_jupyter_idle_timeout_returned --deselect=distributed/tests/test_jupyter.py::test_jupyter_server --deselect=distributed/tests/test_jupyter.py::test_shutsdown_cleanly --deselect=distributed/tests/test_profile.py::test_stack_overflow --deselect=distributed/tests/test_pubsub.py::test_client_worker --deselect=distributed/tests/test_queues.py::test_queue_in_task --deselect=distributed/tests/test_spill.py::test_spillbuffer_oserror --deselect=distributed/tests/test_steal.py::test_steal_twice --deselect=distributed/tests/test_utils_test.py::test_cluster --deselect=distributed/tests/test_variable.py::test_variable_in_task --deselect=distributed/tests/test_worker.py::test_process_executor_kills_process --deselect=distributed/tests/test_worker_memory.py::test_fail_to_pickle_execute_1 --deselect=distributed/tests/test_worker_state_machine.py::test_task_state_instance_are_garbage_collected --deselect=distributed/protocol/tests/test_protocol.py::test_deeply_nested_structures --deselect=distributed/protocol/tests/test_serialize.py::test_deeply_nested_structures --deselect=distributed/cli/tests/test_dask_scheduler.py::test_defaults --deselect=distributed/cli/tests/test_dask_scheduler.py::test_hostport --deselect=distributed/cli/tests/test_dask_spec.py::test_errors --deselect=distributed/cli/tests/test_dask_worker.py::test_scheduler_file --deselect=distributed/cli/tests/test_dask_worker.py::test_scheduler_file --deselect=distributed/cli/tests/test_dask_worker.py::test_scheduler_file --deselect=distributed/dashboard/tests/test_scheduler_bokeh.py::test_counters --deselect=distributed/dashboard/tests/test_worker_bokeh.py::test_counters --deselect=distributed/deploy/tests/test_local.py::test_adapt_then_manual --deselect=distributed/deploy/tests/test_local.py::test_async_with --deselect=distributed/deploy/tests/test_local.py::test_close_twice --deselect=distributed/deploy/tests/test_local.py::test_cluster_info_sync --deselect=distributed/deploy/tests/test_local.py::test_local_tls --deselect=distributed/deploy/tests/test_local.py::test_no_dangling_asyncio_tasks --deselect=distributed/deploy/tests/test_local.py::test_only_local_access --deselect=distributed/deploy/tests/test_local.py::test_remote_access --deselect=distributed/diagnostics/tests/test_progress_widgets.py::test_serializers --deselect=distributed/diagnostics/tests/test_scheduler_plugin.py::test_lifecycle --deselect=distributed/http/scheduler/tests/test_missing_bokeh.py::test_missing_bokeh --deselect=distributed/http/scheduler/tests/test_scheduler_http.py::test_metrics_when_prometheus_client_not_installed --deselect=distributed/protocol/tests/test_serialize.py::test_errors --deselect=distributed/tests/test_batched.py::test_BatchedSend --deselect=distributed/tests/test_batched.py::test_close_closed --deselect=distributed/tests/test_batched.py::test_close_twice --deselect=distributed/tests/test_batched.py::test_send_after_stream_start --deselect=distributed/tests/test_batched.py::test_send_before_close --deselect=distributed/tests/test_batched.py::test_send_before_start --deselect=distributed/tests/test_batched.py::test_sending_traffic_jam --deselect=distributed/tests/test_batched.py::test_serializers --deselect=distributed/tests/test_client.py::TestClientSecurityLoader::test_security_loader --deselect=distributed/tests/test_client.py::TestClientSecurityLoader::test_security_loader_ignored_if_explicit_security_provided --deselect=distributed/tests/test_client.py::TestClientSecurityLoader::test_security_loader_ignored_if_returns_none --deselect=distributed/tests/test_client.py::test_async_with --deselect=distributed/tests/test_client.py::test_client_is_quiet_cluster_close --deselect=distributed/tests/test_client.py::test_dashboard_link_cluster --deselect=distributed/tests/test_client.py::test_dashboard_link_inproc --deselect=distributed/tests/test_client.py::test_file_descriptors_dont_leak --deselect=distributed/tests/test_client.py::test_mixing_clients_different_scheduler --deselect=distributed/tests/test_client.py::test_quiet_client_close --deselect=distributed/tests/test_client.py::test_rebalance_sync --deselect=distributed/tests/test_client.py::test_repr_localcluster --deselect=distributed/tests/test_client.py::test_security_loader --deselect=distributed/tests/test_client.py::test_security_loader_ignored_if_explicit_security_provided --deselect=distributed/tests/test_client.py::test_security_loader_ignored_if_returns_none --deselect=distributed/tests/test_client.py::test_shutdown --deselect=distributed/tests/test_client.py::test_shutdown_is_quiet_with_cluster --deselect=distributed/tests/test_client.py::test_shutdown_localcluster --deselect=distributed/tests/test_client.py::test_shutdown_stops_callbacks --deselect=distributed/tests/test_client_loop.py::test_close_loop_sync_start_new_loop --deselect=distributed/tests/test_client_loop.py::test_close_loop_sync_use_running_loop --deselect=distributed/tests/test_core.py::test_close_fast_without_active_handlers --deselect=distributed/tests/test_core.py::test_close_grace_period_for_handlers --deselect=distributed/tests/test_core.py::test_close_properly --deselect=distributed/tests/test_core.py::test_compression --deselect=distributed/tests/test_core.py::test_connection_pool --deselect=distributed/tests/test_core.py::test_connection_pool_close_while_connecting --deselect=distributed/tests/test_core.py::test_connection_pool_detects_remote_close --deselect=distributed/tests/test_core.py::test_connection_pool_outside_cancellation --deselect=distributed/tests/test_core.py::test_connection_pool_remove --deselect=distributed/tests/test_core.py::test_connection_pool_respects_limit --deselect=distributed/tests/test_core.py::test_connection_pool_tls --deselect=distributed/tests/test_core.py::test_counters --deselect=distributed/tests/test_core.py::test_deserialize_error --deselect=distributed/tests/test_core.py::test_errors --deselect=distributed/tests/test_core.py::test_identity_inproc --deselect=distributed/tests/test_core.py::test_identity_tcp --deselect=distributed/tests/test_core.py::test_large_packets_inproc --deselect=distributed/tests/test_core.py::test_messages_are_ordered_bsend --deselect=distributed/tests/test_core.py::test_messages_are_ordered_raw --deselect=distributed/tests/test_core.py::test_ports --deselect=distributed/tests/test_core.py::test_rpc_default --deselect=distributed/tests/test_core.py::test_rpc_inproc --deselect=distributed/tests/test_core.py::test_rpc_message_lifetime_default --deselect=distributed/tests/test_core.py::test_rpc_message_lifetime_inproc --deselect=distributed/tests/test_core.py::test_rpc_message_lifetime_tcp --deselect=distributed/tests/test_core.py::test_rpc_serialization --deselect=distributed/tests/test_core.py::test_rpc_tcp --deselect=distributed/tests/test_core.py::test_rpc_tls --deselect=distributed/tests/test_core.py::test_rpc_with_many_connections_inproc --deselect=distributed/tests/test_core.py::test_rpc_with_many_connections_tcp --deselect=distributed/tests/test_core.py::test_send_recv_args --deselect=distributed/tests/test_core.py::test_send_recv_cancelled --deselect=distributed/tests/test_core.py::test_server --deselect=distributed/tests/test_core.py::test_server_comms_mark_active_handlers --deselect=distributed/tests/test_core.py::test_server_raises_on_blocked_handlers --deselect=distributed/tests/test_jupyter.py::test_jupyter_idle_timeout --deselect=distributed/tests/test_jupyter.py::test_jupyter_server --deselect=distributed/tests/test_locks.py::test_errors --deselect=distributed/tests/test_nanny.py::test_scheduler_file --deselect=distributed/tests/test_nanny.py::test_scheduler_file --deselect=distributed/tests/test_nanny.py::test_scheduler_file --deselect=distributed/tests/test_nanny.py::test_worker_uses_same_host_as_nanny --deselect=distributed/tests/test_preload.py::test_failure_doesnt_crash_scheduler --deselect=distributed/tests/test_preload.py::test_preload_import_time --deselect=distributed/tests/test_preload.py::test_preload_manager_sequence --deselect=distributed/tests/test_preload.py::test_worker_preload_text --deselect=distributed/tests/test_scheduler.py::test_allowed_failures_config --deselect=distributed/tests/test_scheduler.py::test_async_context_manager --deselect=distributed/tests/test_scheduler.py::test_dashboard_host --deselect=distributed/tests/test_scheduler.py::test_file_descriptors_dont_leak --deselect=distributed/tests/test_scheduler.py::test_finished --deselect=distributed/tests/test_scheduler.py::test_multiple_listeners --deselect=distributed/tests/test_scheduler.py::test_no_dangling_asyncio_tasks --deselect=distributed/tests/test_scheduler.py::test_scheduler_file --deselect=distributed/tests/test_scheduler.py::test_scheduler_file --deselect=distributed/tests/test_scheduler.py::test_scheduler_file --deselect=distributed/tests/test_security.py::test_require_encryption --deselect=distributed/tests/test_security.py::test_tls_listen_connect --deselect=distributed/tests/test_security.py::test_tls_temporary_credentials_functional --deselect=distributed/tests/test_semaphore.py::test_threadpoolworkers_pick_correct_ioloop --deselect=distributed/tests/test_tls_functional.py::test_security_dict_input_no_security --deselect=distributed/tests/test_utils_test.py::test_ensure_no_new_clients --deselect=distributed/tests/test_utils_test.py::test_freeze_batched_send --deselect=distributed/tests/test_utils_test.py::test_locked_comm_drop_in_replacement --deselect=distributed/tests/test_utils_test.py::test_locked_comm_intercept_read --deselect=distributed/tests/test_utils_test.py::test_locked_comm_intercept_write --deselect=distributed/tests/test_worker.py::test_host_uses_scheduler_protocol --deselect=distributed/tests/test_worker.py::test_plugin_exception --deselect=distributed/tests/test_worker.py::test_plugin_internal_exception --deselect=distributed/tests/test_worker.py::test_plugin_multiple_exceptions --deselect=distributed/tests/test_worker.py::test_scheduler_file --deselect=distributed/tests/test_worker.py::test_scheduler_file --deselect=distributed/tests/test_worker.py::test_scheduler_file --deselect=distributed/tests/test_worker_client.py::test_dont_override_default_get --deselect=distributed/cli/tests/test_dask_scheduler.py::test_dashboard --deselect=distributed/cli/tests/test_dask_scheduler.py::test_dashboard_allowlist --deselect=distributed/cli/tests/test_dask_scheduler.py::test_dashboard_non_standard_ports --deselect=distributed/cli/tests/test_dask_scheduler.py::test_dashboard_port_zero --deselect=distributed/cli/tests/test_dask_scheduler.py::test_defaults --deselect=distributed/cli/tests/test_dask_scheduler.py::test_hostport --deselect=distributed/cli/tests/test_dask_scheduler.py::test_multiple_protocols --deselect=distributed/cli/tests/test_dask_scheduler.py::test_multiple_workers --deselect=distributed/cli/tests/test_dask_scheduler.py::test_multiple_workers_2 --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_command --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_command_default --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_config --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_file --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_module --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_remote_module --deselect=distributed/cli/tests/test_dask_scheduler.py::test_scheduler_port_zero --deselect=distributed/cli/tests/test_dask_scheduler.py::test_single_executable_deprecated --deselect=distributed/cli/tests/test_dask_worker.py::test_contact_listen_address --deselect=distributed/cli/tests/test_dask_worker.py::test_dashboard_non_standard_ports --deselect=distributed/cli/tests/test_dask_worker.py::test_error_during_startup --deselect=distributed/cli/tests/test_dask_worker.py::test_integer_names --deselect=distributed/cli/tests/test_dask_worker.py::test_listen_address_ipv6 --deselect=distributed/cli/tests/test_dask_worker.py::test_local_directory --deselect=distributed/cli/tests/test_dask_worker.py::test_memory_limit --deselect=distributed/cli/tests/test_dask_worker.py::test_nanny_worker_port_range --deselect=distributed/cli/tests/test_dask_worker.py::test_nanny_worker_port_range_too_many_workers_raises --deselect=distributed/cli/tests/test_dask_worker.py::test_nanny_worker_ports --deselect=distributed/cli/tests/test_dask_worker.py::test_no_nanny --deselect=distributed/cli/tests/test_dask_worker.py::test_nworkers_auto --deselect=distributed/cli/tests/test_dask_worker.py::test_nworkers_expands_name --deselect=distributed/cli/tests/test_dask_worker.py::test_nworkers_negative --deselect=distributed/cli/tests/test_dask_worker.py::test_nworkers_requires_nanny --deselect=distributed/cli/tests/test_dask_worker.py::test_preload_config --deselect=distributed/cli/tests/test_dask_worker.py::test_resources --deselect=distributed/cli/tests/test_dask_worker.py::test_respect_host_listen_address --deselect=distributed/cli/tests/test_dask_worker.py::test_scheduler_address_env --deselect=distributed/cli/tests/test_dask_worker.py::test_scheduler_file --deselect=distributed/cli/tests/test_dask_worker.py::test_set_lifetime_restart_via_env_var --deselect=distributed/cli/tests/test_dask_worker.py::test_set_lifetime_stagger_via_env_var --deselect=distributed/cli/tests/test_dask_worker.py::test_signal_handling --deselect=distributed/cli/tests/test_dask_worker.py::test_single_executable_deprecated --deselect=distributed/cli/tests/test_dask_worker.py::test_single_executable_works --deselect=distributed/cli/tests/test_dask_worker.py::test_timeout --deselect=distributed/cli/tests/test_dask_worker.py::test_worker_class --deselect=distributed/tests/test_config.py::test_logging_extended --deselect=distributed/tests/test_config.py::test_logging_file_config --deselect=distributed/tests/test_config.py::test_logging_mutual_exclusive --deselect=distributed/tests/test_config.py::test_logging_simple --deselect=distributed/tests/test_config.py::test_logging_simple_under_distributed --deselect=distributed/tests/test_queues.py::test_queue_in_task --deselect=distributed/tests/test_variable.py::test_variable_in_task +============================= test session starts ============================== +platform linux -- Python 3.13.7, pytest-8.3.5, pluggy-1.6.0 -- /usr/bin/python3.13 +cachedir: .pytest_cache +rootdir: /build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build +configfile: pyproject.toml +plugins: timeout-2.3.1, rerunfailures-15.1, typeguard-4.4.4, anyio-4.8.0 +timeout: 300.0s +timeout method: signal +timeout func_only: False +collecting ... collected 1 item +run-last-failure: rerun previous 1 failure (skipped 148 files) + +distributed/tests/test_tls_functional.py::test_retire_workers FAILED [100%] + +=================================== FAILURES =================================== +_____________________________ test_retire_workers ______________________________ + +fut = , timeout = 0 + + async def wait_for(fut: Awaitable[T], timeout: float) -> T: + async with asyncio.timeout(timeout): +> return await fut + +distributed/utils.py:1914: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +distributed/nanny.py:358: in start_unsafe + comm = await self.rpc.connect(saddr) +distributed/core.py:1485: in connect + return await self._connect(addr=addr, timeout=timeout) +distributed/core.py:1429: in _connect + comm = await connect( +distributed/comm/core.py:342: in connect + comm = await wait_for( +distributed/utils.py:1914: in wait_for + return await fut +distributed/comm/tcp.py:546: in connect + stream = await self.client.connect( +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +host = '127.0.0.1', port = 45857, af = +ssl_options = +max_buffer_size = 16798425088.0, source_ip = None, source_port = None +timeout = None + + async def connect( + self, + host: str, + port: int, + af: socket.AddressFamily = socket.AF_UNSPEC, + ssl_options: Optional[Union[Dict[str, Any], ssl.SSLContext]] = None, + max_buffer_size: Optional[int] = None, + source_ip: Optional[str] = None, + source_port: Optional[int] = None, + timeout: Optional[Union[float, datetime.timedelta]] = None, + ) -> IOStream: + """Connect to the given host and port. + + Asynchronously returns an `.IOStream` (or `.SSLIOStream` if + ``ssl_options`` is not None). + + Using the ``source_ip`` kwarg, one can specify the source + IP address to use when establishing the connection. + In case the user needs to resolve and + use a specific interface, it has to be handled outside + of Tornado as this depends very much on the platform. + + Raises `TimeoutError` if the input future does not complete before + ``timeout``, which may be specified in any form allowed by + `.IOLoop.add_timeout` (i.e. a `datetime.timedelta` or an absolute time + relative to `.IOLoop.time`) + + Similarly, when the user requires a certain source port, it can + be specified using the ``source_port`` arg. + + .. versionchanged:: 4.5 + Added the ``source_ip`` and ``source_port`` arguments. + + .. versionchanged:: 5.0 + Added the ``timeout`` argument. + """ + if timeout is not None: + if isinstance(timeout, numbers.Real): + timeout = IOLoop.current().time() + timeout + elif isinstance(timeout, datetime.timedelta): + timeout = IOLoop.current().time() + timeout.total_seconds() + else: + raise TypeError("Unsupported timeout %r" % timeout) + if timeout is not None: + addrinfo = await gen.with_timeout( + timeout, self.resolver.resolve(host, port, af) + ) + else: + addrinfo = await self.resolver.resolve(host, port, af) + connector = _Connector( + addrinfo, + functools.partial( + self._create_stream, + max_buffer_size, + source_ip=source_ip, + source_port=source_port, + ), + ) +> af, addr, stream = await connector.start(connect_timeout=timeout) +E asyncio.exceptions.CancelledError + +/usr/lib/python3/dist-packages/tornado/tcpclient.py:279: CancelledError + +The above exception was the direct cause of the following exception: + +self = + + @final + async def start(self): + async with self._startup_lock: + if self.status == Status.failed: + assert self.__startup_exc is not None + raise self.__startup_exc + elif self.status != Status.init: + return self + timeout = getattr(self, "death_timeout", None) + + async def _close_on_failure(exc: Exception) -> None: + await self.close(reason=f"failure-to-start-{str(type(exc))}") + self.status = Status.failed + self.__startup_exc = exc + + try: +> await wait_for(self.start_unsafe(), timeout=timeout) + +distributed/core.py:528: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +distributed/utils.py:1913: in wait_for + async with asyncio.timeout(timeout): +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +exc_type = +exc_val = CancelledError(), exc_tb = + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> Optional[bool]: + assert self._state in (_State.ENTERED, _State.EXPIRING) + + if self._timeout_handler is not None: + self._timeout_handler.cancel() + self._timeout_handler = None + + if self._state is _State.EXPIRING: + self._state = _State.EXPIRED + + if self._task.uncancel() <= self._cancelling and exc_type is not None: + # Since there are no new cancel requests, we're + # handling this. + if issubclass(exc_type, exceptions.CancelledError): +> raise TimeoutError from exc_val +E TimeoutError + +/usr/lib/python3.13/asyncio/timeouts.py:116: TimeoutError + +The above exception was the direct cause of the following exception: + + @contextlib.asynccontextmanager + async def _cluster_factory(): + workers = [] + s = None + try: + for _ in range(60): + try: +> s, ws = await start_cluster( + nthreads, + scheduler, + security=security, + Worker=Worker, + scheduler_kwargs=scheduler_kwargs, + worker_kwargs=merge( + {"death_timeout": min(15, int(deadline.remaining))}, + worker_kwargs, + ), + ) + +distributed/utils_test.py:974: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +distributed/utils_test.py:791: in start_cluster + await asyncio.gather(*workers) +/usr/lib/python3.13/asyncio/tasks.py:737: in _wrap_awaitable + return await awaitable +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = + + @final + async def start(self): + async with self._startup_lock: + if self.status == Status.failed: + assert self.__startup_exc is not None + raise self.__startup_exc + elif self.status != Status.init: + return self + timeout = getattr(self, "death_timeout", None) + + async def _close_on_failure(exc: Exception) -> None: + await self.close(reason=f"failure-to-start-{str(type(exc))}") + self.status = Status.failed + self.__startup_exc = exc + + try: + await wait_for(self.start_unsafe(), timeout=timeout) + except asyncio.TimeoutError as exc: + await _close_on_failure(exc) +> raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +E TimeoutError: Nanny start timed out after 0s. + +distributed/core.py:531: TimeoutError + +During handling of the above exception, another exception occurred: + +fut = ._..test_func..async_fn at 0xffff9b2a7140> +timeout = 60 + + async def wait_for(fut: Awaitable[T], timeout: float) -> T: + async with asyncio.timeout(timeout): +> return await fut + +distributed/utils.py:1914: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +distributed/utils_test.py:1007: in async_fn + _cluster_factory() as (s, workers), +/usr/lib/python3.13/contextlib.py:214: in __aenter__ + return await anext(self.gen) +distributed/utils_test.py:991: in _cluster_factory + await asyncio.sleep(1) +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +delay = 1, result = None + + async def sleep(delay, result=None): + """Coroutine that completes after a given time (in seconds).""" + if delay <= 0: + await __sleep0() + return result + + if math.isnan(delay): + raise ValueError("Invalid delay: NaN (not a number)") + + loop = events.get_running_loop() + future = loop.create_future() + h = loop.call_later(delay, + futures._set_result_unless_cancelled, + future, result) + try: +> return await future +E asyncio.exceptions.CancelledError + +/usr/lib/python3.13/asyncio/tasks.py:718: CancelledError + +The above exception was the direct cause of the following exception: + +args = (), kwds = {} + + @wraps(func) + def inner(*args, **kwds): + with self._recreate_cm(): +> return func(*args, **kwds) + +/usr/lib/python3.13/contextlib.py:85: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +/usr/lib/python3.13/contextlib.py:85: in inner + return func(*args, **kwds) +distributed/utils_test.py:1090: in test_func + return _run_and_close_tornado(async_fn_outer) +distributed/utils_test.py:380: in _run_and_close_tornado + return asyncio_run(inner_fn(), loop_factory=get_loop_factory()) +/usr/lib/python3.13/asyncio/runners.py:195: in run + return runner.run(main) +/usr/lib/python3.13/asyncio/runners.py:118: in run + return self._loop.run_until_complete(task) +/usr/lib/python3.13/asyncio/base_events.py:725: in run_until_complete + return future.result() +distributed/utils_test.py:377: in inner_fn + return await async_fn(*args, **kwargs) +distributed/utils_test.py:1087: in async_fn_outer + return await utils_wait_for(async_fn(), timeout=timeout * 2) +distributed/utils.py:1913: in wait_for + async with asyncio.timeout(timeout): +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +exc_type = +exc_val = CancelledError(), exc_tb = + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> Optional[bool]: + assert self._state in (_State.ENTERED, _State.EXPIRING) + + if self._timeout_handler is not None: + self._timeout_handler.cancel() + self._timeout_handler = None + + if self._state is _State.EXPIRING: + self._state = _State.EXPIRED + + if self._task.uncancel() <= self._cancelling and exc_type is not None: + # Since there are no new cancel requests, we're + # handling this. + if issubclass(exc_type, exceptions.CancelledError): +> raise TimeoutError from exc_val +E TimeoutError + +/usr/lib/python3.13/asyncio/timeouts.py:116: TimeoutError +----------------------------- Captured stderr call ----------------------------- +2025-09-03 03:40:29,421 - distributed.http.proxy - INFO - To route to workers diagnostics web server please install jupyter-server-proxy: python -m pip install jupyter-server-proxy +2025-09-03 03:40:29,422 - distributed.scheduler - INFO - State start +2025-09-03 03:40:29,426 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:39049 +2025-09-03 03:40:29,426 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:44127/status +2025-09-03 03:40:29,426 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:40:29,459 - distributed.nanny - INFO - Start Nanny at: 'tls://127.0.0.1:40435' +2025-09-03 03:40:29,466 - distributed.nanny - INFO - Start Nanny at: 'tls://127.0.0.1:41309' +2025-09-03 03:40:30,019 - distributed.worker - INFO - Start worker at: tls://127.0.0.1:45377 +2025-09-03 03:40:30,019 - distributed.worker - INFO - Listening to: tls://127.0.0.1:45377 +2025-09-03 03:40:30,019 - distributed.worker - INFO - Worker name: 0 +2025-09-03 03:40:30,019 - distributed.worker - INFO - dashboard at: 127.0.0.1:35235 +2025-09-03 03:40:30,019 - distributed.worker - INFO - Waiting to connect to: tls://127.0.0.1:39049 +2025-09-03 03:40:30,019 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:40:30,019 - distributed.worker - INFO - Threads: 1 +2025-09-03 03:40:30,019 - distributed.worker - INFO - Memory: 31.29 GiB +2025-09-03 03:40:30,019 - distributed.worker - INFO - Local Directory: /tmp/dask-scratch-space/worker-wzrbhkp1 +2025-09-03 03:40:30,019 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:40:30,026 - distributed.worker - INFO - Start worker at: tls://127.0.0.1:45817 +2025-09-03 03:40:30,026 - distributed.worker - INFO - Listening to: tls://127.0.0.1:45817 +2025-09-03 03:40:30,027 - distributed.worker - INFO - Worker name: 1 +2025-09-03 03:40:30,027 - distributed.worker - INFO - dashboard at: 127.0.0.1:36921 +2025-09-03 03:40:30,027 - distributed.worker - INFO - Waiting to connect to: tls://127.0.0.1:39049 +2025-09-03 03:40:30,027 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:40:30,027 - distributed.worker - INFO - Threads: 2 +2025-09-03 03:40:30,027 - distributed.worker - INFO - Memory: 31.29 GiB +2025-09-03 03:40:30,027 - distributed.worker - INFO - Local Directory: /tmp/dask-scratch-space/worker-7ypsjpcx +2025-09-03 03:40:30,027 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:40:30,271 - distributed.scheduler - INFO - Register worker addr: tls://127.0.0.1:45377 name: 0 +2025-09-03 03:40:30,553 - distributed.worker - INFO - Starting Worker plugin shuffle +2025-09-03 03:40:30,554 - distributed.worker - INFO - Registered to: tls://127.0.0.1:39049 +2025-09-03 03:40:30,554 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:40:30,554 - distributed.scheduler - INFO - Starting worker compute stream, tls://127.0.0.1:45377 +2025-09-03 03:40:30,554 - distributed.core - INFO - Starting established connection to tls://127.0.0.1:57758 +2025-09-03 03:40:30,556 - distributed.scheduler - INFO - Register worker addr: tls://127.0.0.1:45817 name: 1 +2025-09-03 03:40:30,556 - distributed.core - INFO - Starting established connection to tls://127.0.0.1:39049 +2025-09-03 03:40:30,557 - distributed.scheduler - INFO - Starting worker compute stream, tls://127.0.0.1:45817 +2025-09-03 03:40:30,557 - distributed.core - INFO - Starting established connection to tls://127.0.0.1:57762 +2025-09-03 03:40:30,559 - distributed.worker - INFO - Starting Worker plugin shuffle +2025-09-03 03:40:30,560 - distributed.worker - INFO - Registered to: tls://127.0.0.1:39049 +2025-09-03 03:40:30,560 - distributed.worker - INFO - ------------------------------------------------- +2025-09-03 03:40:30,562 - distributed.core - INFO - Connection to tls://127.0.0.1:57762 has been closed. +2025-09-03 03:40:30,562 - distributed.scheduler - INFO - Remove worker addr: tls://127.0.0.1:45817 name: 1 (stimulus_id='handle-worker-cleanup-1756820430.562626') +2025-09-03 03:40:30,564 - distributed.core - INFO - Starting established connection to tls://127.0.0.1:39049 +2025-09-03 03:40:30,564 - distributed.core - INFO - Connection to tls://127.0.0.1:39049 has been closed. +2025-09-03 03:40:30,564 - distributed.worker - INFO - Stopping worker at tls://127.0.0.1:45817. Reason: worker-handle-scheduler-connection-broken +2025-09-03 03:40:30,609 - distributed.nanny - INFO - Closing Nanny gracefully at 'tls://127.0.0.1:41309'. Reason: worker-handle-scheduler-connection-broken +2025-09-03 03:40:30,610 - distributed.worker - INFO - Removing Worker plugin shuffle +2025-09-03 03:40:30,613 - distributed.nanny - INFO - Worker closed +2025-09-03 03:40:32,889 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:41309'. Reason: nanny-close-gracefully +2025-09-03 03:40:32,890 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:41309' closed. +2025-09-03 03:41:00,610 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:40435'. Reason: nanny-close +2025-09-03 03:41:00,610 - distributed.nanny - INFO - Nanny asking worker to close. Reason: nanny-close +2025-09-03 03:41:00,611 - distributed.worker - INFO - Stopping worker at tls://127.0.0.1:45377. Reason: nanny-close +2025-09-03 03:41:00,611 - distributed.worker - INFO - Removing Worker plugin shuffle +2025-09-03 03:41:00,613 - distributed.core - INFO - Connection to tls://127.0.0.1:39049 has been closed. +2025-09-03 03:41:00,614 - distributed.core - INFO - Received 'close-stream' from tls://127.0.0.1:57758; closing. +2025-09-03 03:41:00,615 - distributed.scheduler - INFO - Remove worker addr: tls://127.0.0.1:45377 name: 0 (stimulus_id='handle-worker-cleanup-1756820460.615184') +2025-09-03 03:41:00,615 - distributed.scheduler - INFO - Lost all workers +2025-09-03 03:41:00,617 - distributed.nanny - INFO - Worker closed +2025-09-03 03:41:00,895 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:40435' closed. +2025-09-03 03:41:00,895 - distributed.scheduler - INFO - Closing scheduler. Reason: unknown +2025-09-03 03:41:00,896 - distributed.scheduler - INFO - Scheduler closing all comms +2025-09-03 03:41:00,896 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Cluster creation timeout; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 806, in start_cluster + raise TimeoutError("Cluster creation timeout") +TimeoutError: Cluster creation timeout +2025-09-03 03:41:01,902 - distributed.scheduler - INFO - State start +2025-09-03 03:41:01,905 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:36675 +2025-09-03 03:41:01,905 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:40639/status +2025-09-03 03:41:01,905 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:41:01,912 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:35833'. Reason: failure-to-start- +2025-09-03 03:41:01,913 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:35833' closed. +2025-09-03 03:41:01,913 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:46421'. Reason: failure-to-start- +2025-09-03 03:41:01,913 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:46421' closed. +2025-09-03 03:41:01,913 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:36675': TLS handshake failed with remote 'tls://127.0.0.1:43968': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:01,913 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:36675': TLS handshake failed with remote 'tls://127.0.0.1:43974': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:01,913 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:41:02,920 - distributed.scheduler - INFO - State start +2025-09-03 03:41:02,923 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:43335 +2025-09-03 03:41:02,923 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:37795/status +2025-09-03 03:41:02,923 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:41:02,936 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:35147'. Reason: failure-to-start- +2025-09-03 03:41:02,936 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:35147' closed. +2025-09-03 03:41:02,937 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:43193'. Reason: failure-to-start- +2025-09-03 03:41:02,937 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:43193' closed. +2025-09-03 03:41:02,937 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:43335': TLS handshake failed with remote 'tls://127.0.0.1:55118': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:02,937 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:43335': TLS handshake failed with remote 'tls://127.0.0.1:55126': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:02,937 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:41:03,941 - distributed.scheduler - INFO - State start +2025-09-03 03:41:03,944 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:37561 +2025-09-03 03:41:03,944 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:37675/status +2025-09-03 03:41:03,944 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:41:03,952 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:35025'. Reason: failure-to-start- +2025-09-03 03:41:03,952 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:35025' closed. +2025-09-03 03:41:03,952 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:42957'. Reason: failure-to-start- +2025-09-03 03:41:03,952 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:42957' closed. +2025-09-03 03:41:03,952 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:37561': TLS handshake failed with remote 'tls://127.0.0.1:38130': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:03,952 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:37561': TLS handshake failed with remote 'tls://127.0.0.1:38144': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:03,952 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:41:04,958 - distributed.scheduler - INFO - State start +2025-09-03 03:41:04,960 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:35111 +2025-09-03 03:41:04,961 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:44353/status +2025-09-03 03:41:04,961 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:41:04,969 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:41311'. Reason: failure-to-start- +2025-09-03 03:41:04,969 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:41311' closed. +2025-09-03 03:41:04,969 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:32801'. Reason: failure-to-start- +2025-09-03 03:41:04,969 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:32801' closed. +2025-09-03 03:41:04,969 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:35111': TLS handshake failed with remote 'tls://127.0.0.1:48404': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:04,969 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:35111': TLS handshake failed with remote 'tls://127.0.0.1:48418': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:04,970 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:41:05,974 - distributed.scheduler - INFO - State start +2025-09-03 03:41:05,977 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:37433 +2025-09-03 03:41:05,977 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:34745/status +2025-09-03 03:41:05,977 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:41:05,988 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:45883'. Reason: failure-to-start- +2025-09-03 03:41:05,989 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:45883' closed. +2025-09-03 03:41:05,989 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:35457'. Reason: failure-to-start- +2025-09-03 03:41:05,989 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:35457' closed. +2025-09-03 03:41:05,989 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:37433': TLS handshake failed with remote 'tls://127.0.0.1:51074': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:05,989 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:37433': TLS handshake failed with remote 'tls://127.0.0.1:51088': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:05,989 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:41:06,993 - distributed.scheduler - INFO - State start +2025-09-03 03:41:06,998 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:33193 +2025-09-03 03:41:06,998 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:38529/status +2025-09-03 03:41:06,999 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:41:07,011 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:44057'. Reason: failure-to-start- +2025-09-03 03:41:07,011 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:44057' closed. +2025-09-03 03:41:07,011 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:36535'. Reason: failure-to-start- +2025-09-03 03:41:07,012 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:36535' closed. +2025-09-03 03:41:07,012 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:33193': TLS handshake failed with remote 'tls://127.0.0.1:36148': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:07,012 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:33193': TLS handshake failed with remote 'tls://127.0.0.1:36164': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:07,012 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:41:08,016 - distributed.scheduler - INFO - State start +2025-09-03 03:41:08,020 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:39643 +2025-09-03 03:41:08,020 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:35055/status +2025-09-03 03:41:08,020 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:41:08,028 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:33021'. Reason: failure-to-start- +2025-09-03 03:41:08,028 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:33021' closed. +2025-09-03 03:41:08,028 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:37107'. Reason: failure-to-start- +2025-09-03 03:41:08,028 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:37107' closed. +2025-09-03 03:41:08,029 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:39643': TLS handshake failed with remote 'tls://127.0.0.1:40422': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:08,029 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:39643': TLS handshake failed with remote 'tls://127.0.0.1:40436': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:08,029 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:41:09,034 - distributed.scheduler - INFO - State start +2025-09-03 03:41:09,041 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:37789 +2025-09-03 03:41:09,041 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:38205/status +2025-09-03 03:41:09,041 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:41:09,050 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:44731'. Reason: failure-to-start- +2025-09-03 03:41:09,050 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:44731' closed. +2025-09-03 03:41:09,050 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:34963'. Reason: failure-to-start- +2025-09-03 03:41:09,050 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:34963' closed. +2025-09-03 03:41:09,050 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:37789': TLS handshake failed with remote 'tls://127.0.0.1:44774': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:09,050 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:37789': TLS handshake failed with remote 'tls://127.0.0.1:44786': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:09,050 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:41:10,057 - distributed.scheduler - INFO - State start +2025-09-03 03:41:10,060 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:40053 +2025-09-03 03:41:10,060 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:34331/status +2025-09-03 03:41:10,064 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:41:10,071 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:44145'. Reason: failure-to-start- +2025-09-03 03:41:10,072 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:44145' closed. +2025-09-03 03:41:10,072 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:34337'. Reason: failure-to-start- +2025-09-03 03:41:10,072 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:34337' closed. +2025-09-03 03:41:10,072 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:40053': TLS handshake failed with remote 'tls://127.0.0.1:49634': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:10,072 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:40053': TLS handshake failed with remote 'tls://127.0.0.1:49644': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:10,072 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:41:11,077 - distributed.scheduler - INFO - State start +2025-09-03 03:41:11,080 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:40459 +2025-09-03 03:41:11,080 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:34771/status +2025-09-03 03:41:11,080 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:41:11,092 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:42205'. Reason: failure-to-start- +2025-09-03 03:41:11,092 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:42205' closed. +2025-09-03 03:41:11,092 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:44135'. Reason: failure-to-start- +2025-09-03 03:41:11,092 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:44135' closed. +2025-09-03 03:41:11,093 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:40459': TLS handshake failed with remote 'tls://127.0.0.1:40308': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:11,093 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:40459': TLS handshake failed with remote 'tls://127.0.0.1:40324': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:11,093 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:41:12,098 - distributed.scheduler - INFO - State start +2025-09-03 03:41:12,101 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:34617 +2025-09-03 03:41:12,101 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:40351/status +2025-09-03 03:41:12,101 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:41:12,108 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:41123'. Reason: failure-to-start- +2025-09-03 03:41:12,108 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:41123' closed. +2025-09-03 03:41:12,109 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:33587'. Reason: failure-to-start- +2025-09-03 03:41:12,109 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:33587' closed. +2025-09-03 03:41:12,109 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:34617': TLS handshake failed with remote 'tls://127.0.0.1:34296': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:12,109 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:34617': TLS handshake failed with remote 'tls://127.0.0.1:34310': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:12,109 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:41:13,114 - distributed.scheduler - INFO - State start +2025-09-03 03:41:13,116 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:39453 +2025-09-03 03:41:13,117 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:36125/status +2025-09-03 03:41:13,117 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:41:13,124 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:43337'. Reason: failure-to-start- +2025-09-03 03:41:13,125 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:43337' closed. +2025-09-03 03:41:13,125 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:39225'. Reason: failure-to-start- +2025-09-03 03:41:13,125 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:39225' closed. +2025-09-03 03:41:13,125 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:39453': TLS handshake failed with remote 'tls://127.0.0.1:36610': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:13,125 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:39453': TLS handshake failed with remote 'tls://127.0.0.1:36624': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:13,125 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:41:14,130 - distributed.scheduler - INFO - State start +2025-09-03 03:41:14,137 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:44165 +2025-09-03 03:41:14,137 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:34437/status +2025-09-03 03:41:14,137 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:41:14,144 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:37445'. Reason: failure-to-start- +2025-09-03 03:41:14,144 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:37445' closed. +2025-09-03 03:41:14,144 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:45509'. Reason: failure-to-start- +2025-09-03 03:41:14,145 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:45509' closed. +2025-09-03 03:41:14,145 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:44165': TLS handshake failed with remote 'tls://127.0.0.1:39578': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:14,145 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:44165': TLS handshake failed with remote 'tls://127.0.0.1:39582': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:14,145 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:41:15,149 - distributed.scheduler - INFO - State start +2025-09-03 03:41:15,153 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:36185 +2025-09-03 03:41:15,153 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:37009/status +2025-09-03 03:41:15,153 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:41:15,160 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:42759'. Reason: failure-to-start- +2025-09-03 03:41:15,160 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:42759' closed. +2025-09-03 03:41:15,161 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:35683'. Reason: failure-to-start- +2025-09-03 03:41:15,161 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:35683' closed. +2025-09-03 03:41:15,161 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:36185': TLS handshake failed with remote 'tls://127.0.0.1:58750': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:15,161 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:36185': TLS handshake failed with remote 'tls://127.0.0.1:58754': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:15,161 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:41:16,165 - distributed.scheduler - INFO - State start +2025-09-03 03:41:16,168 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:39085 +2025-09-03 03:41:16,172 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:38969/status +2025-09-03 03:41:16,172 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:41:16,179 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:39903'. Reason: failure-to-start- +2025-09-03 03:41:16,180 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:39903' closed. +2025-09-03 03:41:16,180 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:45491'. Reason: failure-to-start- +2025-09-03 03:41:16,180 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:45491' closed. +2025-09-03 03:41:16,180 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:39085': TLS handshake failed with remote 'tls://127.0.0.1:41012': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:16,180 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:39085': TLS handshake failed with remote 'tls://127.0.0.1:41028': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:16,180 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:41:17,185 - distributed.scheduler - INFO - State start +2025-09-03 03:41:17,189 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:33753 +2025-09-03 03:41:17,189 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:43173/status +2025-09-03 03:41:17,189 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:41:17,196 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:46105'. Reason: failure-to-start- +2025-09-03 03:41:17,196 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:46105' closed. +2025-09-03 03:41:17,196 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:36231'. Reason: failure-to-start- +2025-09-03 03:41:17,196 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:36231' closed. +2025-09-03 03:41:17,197 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:33753': TLS handshake failed with remote 'tls://127.0.0.1:53700': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:17,197 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:33753': TLS handshake failed with remote 'tls://127.0.0.1:53716': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:17,197 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:41:18,201 - distributed.scheduler - INFO - State start +2025-09-03 03:41:18,204 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:36745 +2025-09-03 03:41:18,204 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:34407/status +2025-09-03 03:41:18,205 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:41:18,212 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:42835'. Reason: failure-to-start- +2025-09-03 03:41:18,212 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:42835' closed. +2025-09-03 03:41:18,212 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:44469'. Reason: failure-to-start- +2025-09-03 03:41:18,212 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:44469' closed. +2025-09-03 03:41:18,212 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:36745': TLS handshake failed with remote 'tls://127.0.0.1:58332': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:18,213 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:36745': TLS handshake failed with remote 'tls://127.0.0.1:58334': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:18,213 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:41:19,217 - distributed.scheduler - INFO - State start +2025-09-03 03:41:19,221 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:32985 +2025-09-03 03:41:19,221 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:46743/status +2025-09-03 03:41:19,221 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:41:19,228 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:41689'. Reason: failure-to-start- +2025-09-03 03:41:19,228 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:41689' closed. +2025-09-03 03:41:19,228 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:45943'. Reason: failure-to-start- +2025-09-03 03:41:19,229 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:45943' closed. +2025-09-03 03:41:19,229 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:32985': TLS handshake failed with remote 'tls://127.0.0.1:33288': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:19,229 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:32985': TLS handshake failed with remote 'tls://127.0.0.1:33296': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:19,229 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:41:20,233 - distributed.scheduler - INFO - State start +2025-09-03 03:41:20,236 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:43211 +2025-09-03 03:41:20,236 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:43275/status +2025-09-03 03:41:20,236 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:41:20,247 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:34967'. Reason: failure-to-start- +2025-09-03 03:41:20,247 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:34967' closed. +2025-09-03 03:41:20,248 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:42489'. Reason: failure-to-start- +2025-09-03 03:41:20,248 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:42489' closed. +2025-09-03 03:41:20,252 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:43211': TLS handshake failed with remote 'tls://127.0.0.1:50674': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:20,253 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:43211': TLS handshake failed with remote 'tls://127.0.0.1:50686': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:20,253 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:41:21,256 - distributed.scheduler - INFO - State start +2025-09-03 03:41:21,261 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:42715 +2025-09-03 03:41:21,261 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:42627/status +2025-09-03 03:41:21,261 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:41:21,272 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:46273'. Reason: failure-to-start- +2025-09-03 03:41:21,272 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:46273' closed. +2025-09-03 03:41:21,272 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:41107'. Reason: failure-to-start- +2025-09-03 03:41:21,273 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:41107' closed. +2025-09-03 03:41:21,273 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:42715': TLS handshake failed with remote 'tls://127.0.0.1:49912': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:21,273 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:42715': TLS handshake failed with remote 'tls://127.0.0.1:49916': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:21,273 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:41:22,277 - distributed.scheduler - INFO - State start +2025-09-03 03:41:22,280 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:44061 +2025-09-03 03:41:22,281 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:36485/status +2025-09-03 03:41:22,282 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:41:22,290 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:36687'. Reason: failure-to-start- +2025-09-03 03:41:22,290 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:36687' closed. +2025-09-03 03:41:22,291 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:43453'. Reason: failure-to-start- +2025-09-03 03:41:22,291 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:43453' closed. +2025-09-03 03:41:22,291 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:44061': TLS handshake failed with remote 'tls://127.0.0.1:59722': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:22,291 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:44061': TLS handshake failed with remote 'tls://127.0.0.1:59734': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:22,291 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:41:23,296 - distributed.scheduler - INFO - State start +2025-09-03 03:41:23,300 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:34187 +2025-09-03 03:41:23,304 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:37663/status +2025-09-03 03:41:23,305 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:41:23,311 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:35691'. Reason: failure-to-start- +2025-09-03 03:41:23,312 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:35691' closed. +2025-09-03 03:41:23,312 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:34337'. Reason: failure-to-start- +2025-09-03 03:41:23,312 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:34337' closed. +2025-09-03 03:41:23,312 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:34187': TLS handshake failed with remote 'tls://127.0.0.1:53660': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:23,312 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:34187': TLS handshake failed with remote 'tls://127.0.0.1:53668': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:23,312 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:41:24,318 - distributed.scheduler - INFO - State start +2025-09-03 03:41:24,325 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:41905 +2025-09-03 03:41:24,325 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:34243/status +2025-09-03 03:41:24,325 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:41:24,332 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:42521'. Reason: failure-to-start- +2025-09-03 03:41:24,332 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:42521' closed. +2025-09-03 03:41:24,332 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:44579'. Reason: failure-to-start- +2025-09-03 03:41:24,332 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:44579' closed. +2025-09-03 03:41:24,333 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:41905': TLS handshake failed with remote 'tls://127.0.0.1:59184': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:24,333 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:41905': TLS handshake failed with remote 'tls://127.0.0.1:59186': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:24,333 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:41:25,344 - distributed.scheduler - INFO - State start +2025-09-03 03:41:25,347 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:34601 +2025-09-03 03:41:25,347 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:36983/status +2025-09-03 03:41:25,348 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:41:25,359 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:39825'. Reason: failure-to-start- +2025-09-03 03:41:25,359 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:39825' closed. +2025-09-03 03:41:25,359 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:38203'. Reason: failure-to-start- +2025-09-03 03:41:25,359 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:38203' closed. +2025-09-03 03:41:25,359 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:34601': TLS handshake failed with remote 'tls://127.0.0.1:55622': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:25,360 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:34601': TLS handshake failed with remote 'tls://127.0.0.1:55626': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:25,360 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:41:26,363 - distributed.scheduler - INFO - State start +2025-09-03 03:41:26,366 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:37319 +2025-09-03 03:41:26,366 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:41921/status +2025-09-03 03:41:26,367 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:41:26,373 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:41783'. Reason: failure-to-start- +2025-09-03 03:41:26,374 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:41783' closed. +2025-09-03 03:41:26,374 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:39225'. Reason: failure-to-start- +2025-09-03 03:41:26,374 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:39225' closed. +2025-09-03 03:41:26,375 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:37319': TLS handshake failed with remote 'tls://127.0.0.1:55972': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:26,376 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:37319': TLS handshake failed with remote 'tls://127.0.0.1:55978': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:26,376 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:41:27,381 - distributed.scheduler - INFO - State start +2025-09-03 03:41:27,384 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:43321 +2025-09-03 03:41:27,384 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:44669/status +2025-09-03 03:41:27,385 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:41:27,396 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:43819'. Reason: failure-to-start- +2025-09-03 03:41:27,396 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:43819' closed. +2025-09-03 03:41:27,396 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:36903'. Reason: failure-to-start- +2025-09-03 03:41:27,396 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:36903' closed. +2025-09-03 03:41:27,396 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:43321': TLS handshake failed with remote 'tls://127.0.0.1:53710': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:27,396 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:43321': TLS handshake failed with remote 'tls://127.0.0.1:53716': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:27,396 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +2025-09-03 03:41:28,402 - distributed.scheduler - INFO - State start +2025-09-03 03:41:28,405 - distributed.scheduler - INFO - Scheduler at: tls://127.0.0.1:45857 +2025-09-03 03:41:28,405 - distributed.scheduler - INFO - dashboard at: http://127.0.0.1:39829/status +2025-09-03 03:41:28,405 - distributed.scheduler - INFO - Registering Worker plugin shuffle +2025-09-03 03:41:28,417 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:39637'. Reason: failure-to-start- +2025-09-03 03:41:28,417 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:39637' closed. +2025-09-03 03:41:28,417 - distributed.nanny - INFO - Closing Nanny at 'tls://127.0.0.1:34225'. Reason: failure-to-start- +2025-09-03 03:41:28,418 - distributed.nanny - INFO - Nanny at 'tls://127.0.0.1:34225' closed. +2025-09-03 03:41:28,418 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:45857': TLS handshake failed with remote 'tls://127.0.0.1:38106': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:28,418 - distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:45857': TLS handshake failed with remote 'tls://127.0.0.1:38110': [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1033) +2025-09-03 03:41:28,418 - distributed.utils_test - ERROR - Failed to start gen_cluster: TimeoutError: Nanny start timed out after 0s.; retrying +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/nanny.py", line 358, in start_unsafe + comm = await self.rpc.connect(saddr) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1485, in connect + return await self._connect(addr=addr, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 1429, in _connect + comm = await connect( + ^^^^^^^^^^^^^^ + ...<4 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/core.py", line 342, in connect + comm = await wait_for( + ^^^^^^^^^^^^^^^ + ...<2 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1914, in wait_for + return await fut + ^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/comm/tcp.py", line 546, in connect + stream = await self.client.connect( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ) + ^ + File "/usr/lib/python3/dist-packages/tornado/tcpclient.py", line 279, in connect + af, addr, stream = await connector.start(connect_timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 528, in start + await wait_for(self.start_unsafe(), timeout=timeout) + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils.py", line 1913, in wait_for + async with asyncio.timeout(timeout): + ~~~~~~~~~~~~~~~^^^^^^^^^ + File "/usr/lib/python3.13/asyncio/timeouts.py", line 116, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 974, in _cluster_factory + s, ws = await start_cluster( + ^^^^^^^^^^^^^^^^^^^^ + ...<9 lines>... + ) + ^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/utils_test.py", line 791, in start_cluster + await asyncio.gather(*workers) + File "/usr/lib/python3.13/asyncio/tasks.py", line 737, in _wrap_awaitable + return await awaitable + ^^^^^^^^^^^^^^^ + File "/build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build/distributed/core.py", line 531, in start + raise asyncio.TimeoutError( + f"{type(self).__name__} start timed out after {timeout}s." + ) from exc +TimeoutError: Nanny start timed out after 0s. +============================= slowest 20 durations ============================= +60.00s call distributed/tests/test_tls_functional.py::test_retire_workers +0.00s setup distributed/tests/test_tls_functional.py::test_retire_workers +0.00s teardown distributed/tests/test_tls_functional.py::test_retire_workers +=========================== short test summary info ============================ +FAILED distributed/tests/test_tls_functional.py::test_retire_workers - TimeoutError +========================= 1 failed in 60.47s (0:01:00) ========================= +*** END OF RUN 3: NOT ALL TESTS HAVE YET PASSED/XFAILED *** +*** STARTING RUN 4: python3.13 -m pytest --pyargs distributed --verbose --color=no --timeout-method=signal --timeout=300 -m not avoid_ci -rfE --last-failed --last-failed-no-failures none --ignore=distributed/comm/tests/test_comms.py --ignore=distributed/comm/tests/test_ws.py --ignore=distributed/deploy/tests/test_adaptive.py --ignore=distributed/deploy/tests/test_local.py --ignore=distributed/deploy/tests/test_slow_adaptive.py --ignore=distributed/deploy/tests/test_spec_cluster.py --deselect=distributed/cli/tests/test_dask_scheduler.py::test_no_dashboard --deselect=distributed/deploy/tests/test_local.py::test_localcluster_get_client --deselect=distributed/deploy/tests/test_old_ssh.py::test_cluster --deselect=distributed/deploy/tests/test_old_ssh.py::test_old_ssh_nprocs_renamed_to_n_workers --deselect=distributed/deploy/tests/test_old_ssh.py::test_nprocs_attribute_is_deprecated --deselect=distributed/deploy/tests/test_ssh.py::test_nprocs_attribute_is_deprecated --deselect=distributed/http/tests/test_core.py::test_prometheus_api_doc --deselect=distributed/tests/test_init.py::test_git_revision --deselect=distributed/tests/test_jupyter.py::test_jupyter_idle_timeout --deselect=distributed/tests/test_jupyter.py::test_jupyter_idle_timeout_returned --deselect=distributed/tests/test_jupyter.py::test_jupyter_server --deselect=distributed/tests/test_jupyter.py::test_shutsdown_cleanly --deselect=distributed/tests/test_profile.py::test_stack_overflow --deselect=distributed/tests/test_pubsub.py::test_client_worker --deselect=distributed/tests/test_queues.py::test_queue_in_task --deselect=distributed/tests/test_spill.py::test_spillbuffer_oserror --deselect=distributed/tests/test_steal.py::test_steal_twice --deselect=distributed/tests/test_utils_test.py::test_cluster --deselect=distributed/tests/test_variable.py::test_variable_in_task --deselect=distributed/tests/test_worker.py::test_process_executor_kills_process --deselect=distributed/tests/test_worker_memory.py::test_fail_to_pickle_execute_1 --deselect=distributed/tests/test_worker_state_machine.py::test_task_state_instance_are_garbage_collected --deselect=distributed/protocol/tests/test_protocol.py::test_deeply_nested_structures --deselect=distributed/protocol/tests/test_serialize.py::test_deeply_nested_structures --deselect=distributed/cli/tests/test_dask_scheduler.py::test_defaults --deselect=distributed/cli/tests/test_dask_scheduler.py::test_hostport --deselect=distributed/cli/tests/test_dask_spec.py::test_errors --deselect=distributed/cli/tests/test_dask_worker.py::test_scheduler_file --deselect=distributed/cli/tests/test_dask_worker.py::test_scheduler_file --deselect=distributed/cli/tests/test_dask_worker.py::test_scheduler_file --deselect=distributed/dashboard/tests/test_scheduler_bokeh.py::test_counters --deselect=distributed/dashboard/tests/test_worker_bokeh.py::test_counters --deselect=distributed/deploy/tests/test_local.py::test_adapt_then_manual --deselect=distributed/deploy/tests/test_local.py::test_async_with --deselect=distributed/deploy/tests/test_local.py::test_close_twice --deselect=distributed/deploy/tests/test_local.py::test_cluster_info_sync --deselect=distributed/deploy/tests/test_local.py::test_local_tls --deselect=distributed/deploy/tests/test_local.py::test_no_dangling_asyncio_tasks --deselect=distributed/deploy/tests/test_local.py::test_only_local_access --deselect=distributed/deploy/tests/test_local.py::test_remote_access --deselect=distributed/diagnostics/tests/test_progress_widgets.py::test_serializers --deselect=distributed/diagnostics/tests/test_scheduler_plugin.py::test_lifecycle --deselect=distributed/http/scheduler/tests/test_missing_bokeh.py::test_missing_bokeh --deselect=distributed/http/scheduler/tests/test_scheduler_http.py::test_metrics_when_prometheus_client_not_installed --deselect=distributed/protocol/tests/test_serialize.py::test_errors --deselect=distributed/tests/test_batched.py::test_BatchedSend --deselect=distributed/tests/test_batched.py::test_close_closed --deselect=distributed/tests/test_batched.py::test_close_twice --deselect=distributed/tests/test_batched.py::test_send_after_stream_start --deselect=distributed/tests/test_batched.py::test_send_before_close --deselect=distributed/tests/test_batched.py::test_send_before_start --deselect=distributed/tests/test_batched.py::test_sending_traffic_jam --deselect=distributed/tests/test_batched.py::test_serializers --deselect=distributed/tests/test_client.py::TestClientSecurityLoader::test_security_loader --deselect=distributed/tests/test_client.py::TestClientSecurityLoader::test_security_loader_ignored_if_explicit_security_provided --deselect=distributed/tests/test_client.py::TestClientSecurityLoader::test_security_loader_ignored_if_returns_none --deselect=distributed/tests/test_client.py::test_async_with --deselect=distributed/tests/test_client.py::test_client_is_quiet_cluster_close --deselect=distributed/tests/test_client.py::test_dashboard_link_cluster --deselect=distributed/tests/test_client.py::test_dashboard_link_inproc --deselect=distributed/tests/test_client.py::test_file_descriptors_dont_leak --deselect=distributed/tests/test_client.py::test_mixing_clients_different_scheduler --deselect=distributed/tests/test_client.py::test_quiet_client_close --deselect=distributed/tests/test_client.py::test_rebalance_sync --deselect=distributed/tests/test_client.py::test_repr_localcluster --deselect=distributed/tests/test_client.py::test_security_loader --deselect=distributed/tests/test_client.py::test_security_loader_ignored_if_explicit_security_provided --deselect=distributed/tests/test_client.py::test_security_loader_ignored_if_returns_none --deselect=distributed/tests/test_client.py::test_shutdown --deselect=distributed/tests/test_client.py::test_shutdown_is_quiet_with_cluster --deselect=distributed/tests/test_client.py::test_shutdown_localcluster --deselect=distributed/tests/test_client.py::test_shutdown_stops_callbacks --deselect=distributed/tests/test_client_loop.py::test_close_loop_sync_start_new_loop --deselect=distributed/tests/test_client_loop.py::test_close_loop_sync_use_running_loop --deselect=distributed/tests/test_core.py::test_close_fast_without_active_handlers --deselect=distributed/tests/test_core.py::test_close_grace_period_for_handlers --deselect=distributed/tests/test_core.py::test_close_properly --deselect=distributed/tests/test_core.py::test_compression --deselect=distributed/tests/test_core.py::test_connection_pool --deselect=distributed/tests/test_core.py::test_connection_pool_close_while_connecting --deselect=distributed/tests/test_core.py::test_connection_pool_detects_remote_close --deselect=distributed/tests/test_core.py::test_connection_pool_outside_cancellation --deselect=distributed/tests/test_core.py::test_connection_pool_remove --deselect=distributed/tests/test_core.py::test_connection_pool_respects_limit --deselect=distributed/tests/test_core.py::test_connection_pool_tls --deselect=distributed/tests/test_core.py::test_counters --deselect=distributed/tests/test_core.py::test_deserialize_error --deselect=distributed/tests/test_core.py::test_errors --deselect=distributed/tests/test_core.py::test_identity_inproc --deselect=distributed/tests/test_core.py::test_identity_tcp --deselect=distributed/tests/test_core.py::test_large_packets_inproc --deselect=distributed/tests/test_core.py::test_messages_are_ordered_bsend --deselect=distributed/tests/test_core.py::test_messages_are_ordered_raw --deselect=distributed/tests/test_core.py::test_ports --deselect=distributed/tests/test_core.py::test_rpc_default --deselect=distributed/tests/test_core.py::test_rpc_inproc --deselect=distributed/tests/test_core.py::test_rpc_message_lifetime_default --deselect=distributed/tests/test_core.py::test_rpc_message_lifetime_inproc --deselect=distributed/tests/test_core.py::test_rpc_message_lifetime_tcp --deselect=distributed/tests/test_core.py::test_rpc_serialization --deselect=distributed/tests/test_core.py::test_rpc_tcp --deselect=distributed/tests/test_core.py::test_rpc_tls --deselect=distributed/tests/test_core.py::test_rpc_with_many_connections_inproc --deselect=distributed/tests/test_core.py::test_rpc_with_many_connections_tcp --deselect=distributed/tests/test_core.py::test_send_recv_args --deselect=distributed/tests/test_core.py::test_send_recv_cancelled --deselect=distributed/tests/test_core.py::test_server --deselect=distributed/tests/test_core.py::test_server_comms_mark_active_handlers --deselect=distributed/tests/test_core.py::test_server_raises_on_blocked_handlers --deselect=distributed/tests/test_jupyter.py::test_jupyter_idle_timeout --deselect=distributed/tests/test_jupyter.py::test_jupyter_server --deselect=distributed/tests/test_locks.py::test_errors --deselect=distributed/tests/test_nanny.py::test_scheduler_file --deselect=distributed/tests/test_nanny.py::test_scheduler_file --deselect=distributed/tests/test_nanny.py::test_scheduler_file --deselect=distributed/tests/test_nanny.py::test_worker_uses_same_host_as_nanny --deselect=distributed/tests/test_preload.py::test_failure_doesnt_crash_scheduler --deselect=distributed/tests/test_preload.py::test_preload_import_time --deselect=distributed/tests/test_preload.py::test_preload_manager_sequence --deselect=distributed/tests/test_preload.py::test_worker_preload_text --deselect=distributed/tests/test_scheduler.py::test_allowed_failures_config --deselect=distributed/tests/test_scheduler.py::test_async_context_manager --deselect=distributed/tests/test_scheduler.py::test_dashboard_host --deselect=distributed/tests/test_scheduler.py::test_file_descriptors_dont_leak --deselect=distributed/tests/test_scheduler.py::test_finished --deselect=distributed/tests/test_scheduler.py::test_multiple_listeners --deselect=distributed/tests/test_scheduler.py::test_no_dangling_asyncio_tasks --deselect=distributed/tests/test_scheduler.py::test_scheduler_file --deselect=distributed/tests/test_scheduler.py::test_scheduler_file --deselect=distributed/tests/test_scheduler.py::test_scheduler_file --deselect=distributed/tests/test_security.py::test_require_encryption --deselect=distributed/tests/test_security.py::test_tls_listen_connect --deselect=distributed/tests/test_security.py::test_tls_temporary_credentials_functional --deselect=distributed/tests/test_semaphore.py::test_threadpoolworkers_pick_correct_ioloop --deselect=distributed/tests/test_tls_functional.py::test_security_dict_input_no_security --deselect=distributed/tests/test_utils_test.py::test_ensure_no_new_clients --deselect=distributed/tests/test_utils_test.py::test_freeze_batched_send --deselect=distributed/tests/test_utils_test.py::test_locked_comm_drop_in_replacement --deselect=distributed/tests/test_utils_test.py::test_locked_comm_intercept_read --deselect=distributed/tests/test_utils_test.py::test_locked_comm_intercept_write --deselect=distributed/tests/test_worker.py::test_host_uses_scheduler_protocol --deselect=distributed/tests/test_worker.py::test_plugin_exception --deselect=distributed/tests/test_worker.py::test_plugin_internal_exception --deselect=distributed/tests/test_worker.py::test_plugin_multiple_exceptions --deselect=distributed/tests/test_worker.py::test_scheduler_file --deselect=distributed/tests/test_worker.py::test_scheduler_file --deselect=distributed/tests/test_worker.py::test_scheduler_file --deselect=distributed/tests/test_worker_client.py::test_dont_override_default_get --deselect=distributed/cli/tests/test_dask_scheduler.py::test_dashboard --deselect=distributed/cli/tests/test_dask_scheduler.py::test_dashboard_allowlist --deselect=distributed/cli/tests/test_dask_scheduler.py::test_dashboard_non_standard_ports --deselect=distributed/cli/tests/test_dask_scheduler.py::test_dashboard_port_zero --deselect=distributed/cli/tests/test_dask_scheduler.py::test_defaults --deselect=distributed/cli/tests/test_dask_scheduler.py::test_hostport --deselect=distributed/cli/tests/test_dask_scheduler.py::test_multiple_protocols --deselect=distributed/cli/tests/test_dask_scheduler.py::test_multiple_workers --deselect=distributed/cli/tests/test_dask_scheduler.py::test_multiple_workers_2 --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_command --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_command_default --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_config --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_file --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_module --deselect=distributed/cli/tests/test_dask_scheduler.py::test_preload_remote_module --deselect=distributed/cli/tests/test_dask_scheduler.py::test_scheduler_port_zero --deselect=distributed/cli/tests/test_dask_scheduler.py::test_single_executable_deprecated --deselect=distributed/cli/tests/test_dask_worker.py::test_contact_listen_address --deselect=distributed/cli/tests/test_dask_worker.py::test_dashboard_non_standard_ports --deselect=distributed/cli/tests/test_dask_worker.py::test_error_during_startup --deselect=distributed/cli/tests/test_dask_worker.py::test_integer_names --deselect=distributed/cli/tests/test_dask_worker.py::test_listen_address_ipv6 --deselect=distributed/cli/tests/test_dask_worker.py::test_local_directory --deselect=distributed/cli/tests/test_dask_worker.py::test_memory_limit --deselect=distributed/cli/tests/test_dask_worker.py::test_nanny_worker_port_range --deselect=distributed/cli/tests/test_dask_worker.py::test_nanny_worker_port_range_too_many_workers_raises --deselect=distributed/cli/tests/test_dask_worker.py::test_nanny_worker_ports --deselect=distributed/cli/tests/test_dask_worker.py::test_no_nanny --deselect=distributed/cli/tests/test_dask_worker.py::test_nworkers_auto --deselect=distributed/cli/tests/test_dask_worker.py::test_nworkers_expands_name --deselect=distributed/cli/tests/test_dask_worker.py::test_nworkers_negative --deselect=distributed/cli/tests/test_dask_worker.py::test_nworkers_requires_nanny --deselect=distributed/cli/tests/test_dask_worker.py::test_preload_config --deselect=distributed/cli/tests/test_dask_worker.py::test_resources --deselect=distributed/cli/tests/test_dask_worker.py::test_respect_host_listen_address --deselect=distributed/cli/tests/test_dask_worker.py::test_scheduler_address_env --deselect=distributed/cli/tests/test_dask_worker.py::test_scheduler_file --deselect=distributed/cli/tests/test_dask_worker.py::test_set_lifetime_restart_via_env_var --deselect=distributed/cli/tests/test_dask_worker.py::test_set_lifetime_stagger_via_env_var --deselect=distributed/cli/tests/test_dask_worker.py::test_signal_handling --deselect=distributed/cli/tests/test_dask_worker.py::test_single_executable_deprecated --deselect=distributed/cli/tests/test_dask_worker.py::test_single_executable_works --deselect=distributed/cli/tests/test_dask_worker.py::test_timeout --deselect=distributed/cli/tests/test_dask_worker.py::test_worker_class --deselect=distributed/tests/test_config.py::test_logging_extended --deselect=distributed/tests/test_config.py::test_logging_file_config --deselect=distributed/tests/test_config.py::test_logging_mutual_exclusive --deselect=distributed/tests/test_config.py::test_logging_simple --deselect=distributed/tests/test_config.py::test_logging_simple_under_distributed --deselect=distributed/tests/test_queues.py::test_queue_in_task --deselect=distributed/tests/test_variable.py::test_variable_in_task +============================= test session starts ============================== +platform linux -- Python 3.13.7, pytest-8.3.5, pluggy-1.6.0 -- /usr/bin/python3.13 +cachedir: .pytest_cache +rootdir: /build/reproducible-path/dask.distributed-2024.12.1+ds/.pybuild/cpython3_3.13_distributed/build +configfile: pyproject.toml +plugins: timeout-2.3.1, rerunfailures-15.1, typeguard-4.4.4, anyio-4.8.0 +timeout: 300.0s +timeout method: signal +timeout func_only: False +collecting ... collected 1 item +run-last-failure: rerun previous 1 failure (skipped 148 files) + +distributed/tests/test_tls_functional.py::test_retire_workers PASSED [100%] + +============================= slowest 20 durations ============================= +3.97s call distributed/tests/test_tls_functional.py::test_retire_workers +0.00s setup distributed/tests/test_tls_functional.py::test_retire_workers +0.00s teardown distributed/tests/test_tls_functional.py::test_retire_workers +============================== 1 passed in 4.09s =============================== +*** END OF RUN 4: ALL TESTS RUN HAVE NOW PASSED/XFAILED *** create-stamp debian/debhelper-build-stamp dh_testroot -O--buildsystem=pybuild dh_prep -O--buildsystem=pybuild @@ -6991,8 +15415,8 @@ dh_gencontrol -O--buildsystem=pybuild dh_md5sums -O--buildsystem=pybuild dh_builddeb -O--buildsystem=pybuild -dpkg-deb: building package 'python3-distributed' in '../python3-distributed_2024.12.1+ds-1_all.deb'. dpkg-deb: building package 'python-distributed-doc' in '../python-distributed-doc_2024.12.1+ds-1_all.deb'. +dpkg-deb: building package 'python3-distributed' in '../python3-distributed_2024.12.1+ds-1_all.deb'. dpkg-genbuildinfo --build=binary -O../dask.distributed_2024.12.1+ds-1_arm64.buildinfo dpkg-genchanges --build=binary -O../dask.distributed_2024.12.1+ds-1_arm64.changes dpkg-genchanges: info: binary-only upload (no source code included) @@ -7001,12 +15425,14 @@ dpkg-buildpackage: info: binary-only upload (no source included) dpkg-genchanges: info: including full source code in upload I: copying local configuration +I: user script /srv/workspace/pbuilder/1909207/tmp/hooks/B01_cleanup starting +I: user script /srv/workspace/pbuilder/1909207/tmp/hooks/B01_cleanup finished I: unmounting dev/ptmx filesystem I: unmounting dev/pts filesystem I: unmounting dev/shm filesystem I: unmounting proc filesystem I: unmounting sys filesystem I: cleaning the build env -I: removing directory /srv/workspace/pbuilder/3739753 and its subdirectories -I: Current time: Mon Oct 5 07:29:34 -12 2026 -I: pbuilder-time-stamp: 1791228574 +I: removing directory /srv/workspace/pbuilder/1909207 and its subdirectories +I: Current time: Wed Sep 3 03:41:50 +14 2025 +I: pbuilder-time-stamp: 1756820510