Diff of the two buildlogs: -- --- b1/build.log 2025-10-27 04:04:05.309364883 +0000 +++ b2/build.log 2025-10-27 04:07:05.549604585 +0000 @@ -1,6 +1,6 @@ I: pbuilder: network access will be disabled during build -I: Current time: Sun Oct 26 16:01:10 -12 2025 -I: pbuilder-time-stamp: 1761537670 +I: Current time: Mon Nov 30 00:27:07 +14 2026 +I: pbuilder-time-stamp: 1795948027 I: Building the build Environment I: extracting base tarball [/var/cache/pbuilder/forky-reproducible-base.tgz] I: copying local configuration @@ -24,53 +24,85 @@ dpkg-source: info: applying 0001-Skip-unreliable-tests.patch I: Not using root during the build. I: Installing the build-deps -I: user script /srv/workspace/pbuilder/2445373/tmp/hooks/D02_print_environment starting +I: user script /srv/workspace/pbuilder/3049399/tmp/hooks/D01_modify_environment starting +debug: Running on codethink03-arm64. +I: Changing host+domainname to test build reproducibility +I: Adding a custom variable just for the fun of it... +I: Changing /bin/sh to bash +'/bin/sh' -> '/bin/bash' +lrwxrwxrwx 1 root root 9 Nov 29 10:27 /bin/sh -> /bin/bash +I: Setting pbuilder2's login shell to /bin/bash +I: Setting pbuilder2's GECOS to second user,second room,second work-phone,second home-phone,second other +I: user script /srv/workspace/pbuilder/3049399/tmp/hooks/D01_modify_environment finished +I: user script /srv/workspace/pbuilder/3049399/tmp/hooks/D02_print_environment starting I: set - BUILDDIR='/build/reproducible-path' - BUILDUSERGECOS='first user,first room,first work-phone,first home-phone,first other' - BUILDUSERNAME='pbuilder1' - BUILD_ARCH='arm64' - DEBIAN_FRONTEND='noninteractive' + BASH=/bin/sh + BASHOPTS=checkwinsize:cmdhist:complete_fullquote:extquote:force_fignore:globasciiranges:globskipdots:hostcomplete:interactive_comments:patsub_replacement:progcomp:promptvars:sourcepath + BASH_ALIASES=() + BASH_ARGC=() + BASH_ARGV=() + BASH_CMDS=() + BASH_LINENO=([0]="12" [1]="0") + BASH_LOADABLES_PATH=/usr/local/lib/bash:/usr/lib/bash:/opt/local/lib/bash:/usr/pkg/lib/bash:/opt/pkg/lib/bash:. + BASH_SOURCE=([0]="/tmp/hooks/D02_print_environment" [1]="/tmp/hooks/D02_print_environment") + BASH_VERSINFO=([0]="5" [1]="3" [2]="3" [3]="1" [4]="release" [5]="aarch64-unknown-linux-gnu") + BASH_VERSION='5.3.3(1)-release' + BUILDDIR=/build/reproducible-path + BUILDUSERGECOS='second user,second room,second work-phone,second home-phone,second other' + BUILDUSERNAME=pbuilder2 + BUILD_ARCH=arm64 + DEBIAN_FRONTEND=noninteractive DEB_BUILD_OPTIONS='buildinfo=+all reproducible=+all parallel=12 ' - DISTRIBUTION='forky' - HOME='/root' - HOST_ARCH='arm64' + DIRSTACK=() + DISTRIBUTION=forky + EUID=0 + FUNCNAME=([0]="Echo" [1]="main") + GROUPS=() + HOME=/root + HOSTNAME=i-capture-the-hostname + HOSTTYPE=aarch64 + HOST_ARCH=arm64 IFS=' ' - INVOCATION_ID='15a573b45c5f4506adef1d6585887d64' - LANG='C' - LANGUAGE='en_US:en' - LC_ALL='C' - MAIL='/var/mail/root' - OPTIND='1' - PATH='/usr/sbin:/usr/bin:/sbin:/bin:/usr/games' - PBCURRENTCOMMANDLINEOPERATION='build' - PBUILDER_OPERATION='build' - PBUILDER_PKGDATADIR='/usr/share/pbuilder' - PBUILDER_PKGLIBDIR='/usr/lib/pbuilder' - PBUILDER_SYSCONFDIR='/etc' - PPID='2445373' - PS1='# ' - PS2='> ' + INVOCATION_ID=ad856d4f398648c9a65667ea287bbc3a + LANG=C + LANGUAGE=nl_BE:nl + LC_ALL=C + MACHTYPE=aarch64-unknown-linux-gnu + MAIL=/var/mail/root + OPTERR=1 + OPTIND=1 + OSTYPE=linux-gnu + PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path + PBCURRENTCOMMANDLINEOPERATION=build + PBUILDER_OPERATION=build + PBUILDER_PKGDATADIR=/usr/share/pbuilder + PBUILDER_PKGLIBDIR=/usr/lib/pbuilder + PBUILDER_SYSCONFDIR=/etc + PIPESTATUS=([0]="0") + POSIXLY_CORRECT=y + PPID=3049399 PS4='+ ' - PWD='/' - SHELL='/bin/bash' - SHLVL='2' - SUDO_COMMAND='/usr/bin/timeout -k 18.1h 18h /usr/bin/ionice -c 3 /usr/bin/nice /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/r-b-build.DiI0oApi/pbuilderrc_ctl1 --distribution forky --hookdir /etc/pbuilder/first-build-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/forky-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/r-b-build.DiI0oApi/b1 --logfile b1/build.log golang-github-hashicorp-raft_1.5.0-1.dsc' - SUDO_GID='109' - SUDO_HOME='/var/lib/jenkins' - SUDO_UID='104' - SUDO_USER='jenkins' - TERM='unknown' - TZ='/usr/share/zoneinfo/Etc/GMT+12' - USER='root' - _='/usr/bin/systemd-run' - http_proxy='http://192.168.101.4:3128' + PWD=/ + SHELL=/bin/bash + SHELLOPTS=braceexpand:errexit:hashall:interactive-comments:posix + SHLVL=3 + SUDO_COMMAND='/usr/bin/timeout -k 24.1h 24h /usr/bin/ionice -c 3 /usr/bin/nice -n 11 /usr/bin/unshare --uts -- /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/r-b-build.DiI0oApi/pbuilderrc_YobA --distribution forky --hookdir /etc/pbuilder/rebuild-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/forky-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/r-b-build.DiI0oApi/b2 --logfile b2/build.log golang-github-hashicorp-raft_1.5.0-1.dsc' + SUDO_GID=109 + SUDO_HOME=/var/lib/jenkins + SUDO_UID=104 + SUDO_USER=jenkins + TERM=unknown + TZ=/usr/share/zoneinfo/Etc/GMT-14 + UID=0 + USER=root + _='I: set' + http_proxy=http://192.168.101.4:3128 I: uname -a - Linux codethink04-arm64 6.12.48+deb13-cloud-arm64 #1 SMP Debian 6.12.48-1 (2025-09-20) aarch64 GNU/Linux + Linux i-capture-the-hostname 6.12.48+deb13-cloud-arm64 #1 SMP Debian 6.12.48-1 (2025-09-20) aarch64 GNU/Linux I: ls -l /bin - lrwxrwxrwx 1 root root 7 Aug 10 12:30 /bin -> usr/bin -I: user script /srv/workspace/pbuilder/2445373/tmp/hooks/D02_print_environment finished + lrwxrwxrwx 1 root root 7 Aug 10 2025 /bin -> usr/bin +I: user script /srv/workspace/pbuilder/3049399/tmp/hooks/D02_print_environment finished -> Attempting to satisfy build-dependencies -> Creating pbuilder-satisfydepends-dummy package Package: pbuilder-satisfydepends-dummy @@ -228,7 +260,7 @@ Get: 108 http://deb.debian.org/debian forky/main arm64 golang-gopkg-check.v1-dev all 0.0+git20200902.038fdea-2 [31.2 kB] Get: 109 http://deb.debian.org/debian forky/main arm64 golang-gopkg-mgo.v2-dev all 2016.08.01-7 [316 kB] Get: 110 http://deb.debian.org/debian forky/main arm64 golang-github-hashicorp-go-msgpack-dev all 0.5.5-1 [43.3 kB] -Fetched 124 MB in 1s (208 MB/s) +Fetched 124 MB in 1s (90.6 MB/s) Preconfiguring packages ... Selecting previously unselected package sensible-utils. (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 20002 files and directories currently installed.) @@ -683,7 +715,11 @@ Building tag database... -> Finished parsing the build-deps I: Building the package -I: Running cd /build/reproducible-path/golang-github-hashicorp-raft-1.5.0/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-buildpackage -us -uc -b && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-genchanges -S > ../golang-github-hashicorp-raft_1.5.0-1_source.changes +I: user script /srv/workspace/pbuilder/3049399/tmp/hooks/A99_set_merged_usr starting +Not re-configuring usrmerge for forky +I: user script /srv/workspace/pbuilder/3049399/tmp/hooks/A99_set_merged_usr finished +hostname: Name or service not known +I: Running cd /build/reproducible-path/golang-github-hashicorp-raft-1.5.0/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-buildpackage -us -uc -b && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-genchanges -S > ../golang-github-hashicorp-raft_1.5.0-1_source.changes dpkg-buildpackage: info: source package golang-github-hashicorp-raft dpkg-buildpackage: info: source version 1.5.0-1 dpkg-buildpackage: info: source distribution unstable @@ -702,43 +738,43 @@ dh_auto_configure -O--buildsystem=golang dh_auto_build -O--buildsystem=golang cd obj-aarch64-linux-gnu && go install -trimpath -v -p 12 github.com/hashicorp/raft github.com/hashicorp/raft/bench -internal/msan +internal/coverage/rtcov internal/goos internal/asan +internal/profilerecord +internal/msan +internal/goexperiment internal/goarch -internal/coverage/rtcov internal/godebugs -internal/unsafeheader -internal/profilerecord +internal/runtime/syscall internal/byteorder -internal/goexperiment internal/cpu -internal/runtime/syscall +internal/unsafeheader sync/atomic math/bits -unicode/utf8 unicode +unicode/utf8 +container/list internal/runtime/math internal/abi -container/list -internal/chacha8rand internal/itoa crypto/internal/fips140/alias +internal/chacha8rand crypto/internal/fips140deps/byteorder +crypto/internal/fips140/subtle cmp crypto/internal/boring/sig encoding +math unicode/utf16 -crypto/internal/fips140/subtle log/internal internal/nettrace vendor/golang.org/x/crypto/cryptobyte/asn1 vendor/golang.org/x/crypto/internal/alias -math internal/bytealg -internal/runtime/atomic crypto/internal/fips140deps/cpu internal/runtime/sys +internal/runtime/atomic internal/stringslite internal/runtime/exithook internal/race @@ -754,59 +790,59 @@ slices errors sort +internal/bisect +internal/singleflight +unique +internal/testlog +runtime/cgo internal/oserror path -github.com/hashicorp/golang-lru/simplelru +io math/rand/v2 strconv -vendor/golang.org/x/net/dns/dnsmessage -io -internal/bisect -internal/singleflight -internal/testlog -unique +github.com/hashicorp/golang-lru/simplelru syscall -runtime/cgo -strings +vendor/golang.org/x/net/dns/dnsmessage +internal/godebug bytes +strings +crypto/internal/fips140deps/godebug +internal/saferio hash -internal/godebug crypto/internal/randutil -internal/saferio +net/netip reflect crypto -net/netip +math/rand hash/crc64 -vendor/golang.org/x/text/transform hash/crc32 +vendor/golang.org/x/text/transform +crypto/internal/fips140 +crypto/internal/impl github.com/hashicorp/go-immutable-radix net/http/internal/ascii -crypto/internal/impl -bufio regexp/syntax html -crypto/internal/fips140deps/godebug -math/rand -crypto/internal/fips140 -crypto/internal/fips140/sha256 -crypto/internal/fips140/sha512 +bufio crypto/internal/fips140/sha3 +crypto/internal/fips140/sha512 +crypto/internal/fips140/sha256 crypto/tls/internal/fips140tls -internal/syscall/execenv -internal/syscall/unix -time crypto/sha3 crypto/internal/fips140/hmac crypto/internal/fips140/check crypto/internal/fips140hash -regexp +time +internal/syscall/execenv +internal/syscall/unix crypto/internal/fips140/aes crypto/internal/fips140/nistec/fiat -crypto/internal/fips140/hkdf crypto/internal/fips140/bigmod -crypto/internal/fips140/tls12 +crypto/internal/fips140/hkdf crypto/internal/fips140/edwards25519/field +crypto/internal/fips140/tls12 crypto/internal/fips140/tls13 +regexp crypto/internal/fips140/edwards25519 context io/fs @@ -822,65 +858,65 @@ encoding/pem crypto/internal/sysrand fmt -os/signal +internal/sysinfo io/ioutil path/filepath -internal/sysinfo -crypto/internal/entropy +os/signal net +crypto/internal/entropy crypto/internal/fips140/drbg +crypto/internal/fips140/aes/gcm crypto/internal/fips140only crypto/internal/fips140/ecdh -crypto/internal/fips140/ecdsa crypto/internal/fips140/ed25519 -crypto/internal/fips140/mlkem -crypto/internal/fips140/aes/gcm crypto/internal/fips140/rsa +crypto/internal/fips140/ecdsa +crypto/internal/fips140/mlkem crypto/md5 crypto/rc4 crypto/cipher crypto/internal/boring crypto/des vendor/golang.org/x/crypto/chacha20 -log -net/url math/big encoding/json -encoding/hex -vendor/golang.org/x/text/unicode/norm +log compress/flate -crypto/aes -crypto/ecdh crypto/sha512 crypto/hmac +crypto/ecdh +crypto/aes crypto/sha1 crypto/sha256 -vendor/golang.org/x/crypto/chacha20poly1305 +encoding/hex +net/url +vendor/golang.org/x/text/unicode/norm vendor/golang.org/x/net/http2/hpack +vendor/golang.org/x/crypto/chacha20poly1305 mime -mime/quotedprintable vendor/golang.org/x/text/unicode/bidi +mime/quotedprintable net/http/internal encoding/gob go/token +compress/gzip text/template/parse flag -compress/gzip -vendor/golang.org/x/text/secure/bidirule runtime/debug runtime/trace -vendor/golang.org/x/net/idna +vendor/golang.org/x/text/secure/bidirule testing text/template -crypto/rand -crypto/elliptic +vendor/golang.org/x/net/idna crypto/internal/boring/bbig +crypto/elliptic encoding/asn1 crypto/dsa -github.com/mattn/go-isatty -crypto/ed25519 -crypto/rsa +crypto/rand crypto/internal/hpke +crypto/rsa +crypto/ed25519 +github.com/mattn/go-isatty github.com/mattn/go-colorable github.com/fatih/color github.com/hashicorp/go-hclog @@ -896,8 +932,8 @@ crypto/tls net/http/httptrace net/http -net/rpc github.com/armon/go-metrics +net/rpc github.com/hashicorp/go-msgpack/codec github.com/hashicorp/raft github.com/hashicorp/raft/bench @@ -946,32 +982,32 @@ === RUN TestFileSnapshotSinkImpl --- PASS: TestFileSnapshotSinkImpl (0.00s) === RUN TestFileSS_CreateSnapshotMissingParentDir -2025-10-27T04:02:05.011Z [INFO] creating new snapshot: path=/tmp/raft1499988263/raft3521652246/snapshots/3-10-1761537725011.tmp +2026-11-29T10:28:09.269Z [INFO] creating new snapshot: path=/tmp/raft3475817082/raft153657292/snapshots/3-10-1795948089269.tmp --- PASS: TestFileSS_CreateSnapshotMissingParentDir (0.00s) === RUN TestFileSS_CreateSnapshot -2025-10-27T04:02:05.011Z [INFO] creating new snapshot: path=/tmp/raft1868349294/snapshots/3-10-1761537725011.tmp -2025-10-27T04:02:05.011Z [WARN] found temporary snapshot: name=3-10-1761537725011.tmp +2026-11-29T10:28:09.270Z [INFO] creating new snapshot: path=/tmp/raft218387009/snapshots/3-10-1795948089270.tmp +2026-11-29T10:28:09.270Z [WARN] found temporary snapshot: name=3-10-1795948089270.tmp --- PASS: TestFileSS_CreateSnapshot (0.00s) === RUN TestFileSS_CancelSnapshot -2025-10-27T04:02:05.012Z [INFO] creating new snapshot: path=/tmp/raft289145920/snapshots/3-10-1761537725012.tmp +2026-11-29T10:28:09.270Z [INFO] creating new snapshot: path=/tmp/raft2416860136/snapshots/3-10-1795948089270.tmp --- PASS: TestFileSS_CancelSnapshot (0.00s) === RUN TestFileSS_Retention -2025-10-27T04:02:05.012Z [INFO] creating new snapshot: path=/tmp/raft3490230658/snapshots/3-10-1761537725012.tmp -2025-10-27T04:02:05.013Z [INFO] creating new snapshot: path=/tmp/raft3490230658/snapshots/3-11-1761537725013.tmp -2025-10-27T04:02:05.013Z [INFO] creating new snapshot: path=/tmp/raft3490230658/snapshots/3-12-1761537725013.tmp -2025-10-27T04:02:05.013Z [INFO] reaping snapshot: path=/tmp/raft3490230658/snapshots/3-10-1761537725012 -2025-10-27T04:02:05.014Z [INFO] creating new snapshot: path=/tmp/raft3490230658/snapshots/3-13-1761537725014.tmp -2025-10-27T04:02:05.014Z [INFO] reaping snapshot: path=/tmp/raft3490230658/snapshots/3-11-1761537725013 -2025-10-27T04:02:05.014Z [INFO] creating new snapshot: path=/tmp/raft3490230658/snapshots/3-14-1761537725014.tmp -2025-10-27T04:02:05.014Z [INFO] reaping snapshot: path=/tmp/raft3490230658/snapshots/3-12-1761537725013 +2026-11-29T10:28:09.270Z [INFO] creating new snapshot: path=/tmp/raft1646568099/snapshots/3-10-1795948089270.tmp +2026-11-29T10:28:09.270Z [INFO] creating new snapshot: path=/tmp/raft1646568099/snapshots/3-11-1795948089270.tmp +2026-11-29T10:28:09.271Z [INFO] creating new snapshot: path=/tmp/raft1646568099/snapshots/3-12-1795948089271.tmp +2026-11-29T10:28:09.271Z [INFO] reaping snapshot: path=/tmp/raft1646568099/snapshots/3-10-1795948089270 +2026-11-29T10:28:09.271Z [INFO] creating new snapshot: path=/tmp/raft1646568099/snapshots/3-13-1795948089271.tmp +2026-11-29T10:28:09.271Z [INFO] reaping snapshot: path=/tmp/raft1646568099/snapshots/3-11-1795948089270 +2026-11-29T10:28:09.271Z [INFO] creating new snapshot: path=/tmp/raft1646568099/snapshots/3-14-1795948089271.tmp +2026-11-29T10:28:09.271Z [INFO] reaping snapshot: path=/tmp/raft1646568099/snapshots/3-12-1795948089271 --- PASS: TestFileSS_Retention (0.00s) === RUN TestFileSS_BadPerm --- PASS: TestFileSS_BadPerm (0.00s) === RUN TestFileSS_MissingParentDir --- PASS: TestFileSS_MissingParentDir (0.00s) === RUN TestFileSS_Ordering -2025-10-27T04:02:05.015Z [INFO] creating new snapshot: path=/tmp/raft1033520234/snapshots/5-130350-1761537725015.tmp -2025-10-27T04:02:05.015Z [INFO] creating new snapshot: path=/tmp/raft1033520234/snapshots/36-204917-1761537725015.tmp +2026-11-29T10:28:09.271Z [INFO] creating new snapshot: path=/tmp/raft1068824971/snapshots/5-130350-1795948089271.tmp +2026-11-29T10:28:09.272Z [INFO] creating new snapshot: path=/tmp/raft1068824971/snapshots/36-204917-1795948089272.tmp --- PASS: TestFileSS_Ordering (0.00s) === RUN TestDeferFutureSuccess --- PASS: TestDeferFutureSuccess (0.00s) @@ -990,7 +1026,7 @@ === RUN TestInmemTransportImpl --- PASS: TestInmemTransportImpl (0.00s) === RUN TestInmemTransportWriteTimeout ---- PASS: TestInmemTransportWriteTimeout (0.01s) +--- PASS: TestInmemTransportWriteTimeout (0.03s) === RUN TestRaft_Integ --- SKIP: TestRaft_Integ (0.00s) === RUN TestRaft_RestartFollower_LongInitialHeartbeat @@ -1018,21 +1054,21 @@ === RUN TestNetworkTransport_AppendEntriesPipeline --- PASS: TestNetworkTransport_AppendEntriesPipeline (0.00s) === RUN TestNetworkTransport_AppendEntriesPipeline_CloseStreams ---- PASS: TestNetworkTransport_AppendEntriesPipeline_CloseStreams (0.01s) +--- PASS: TestNetworkTransport_AppendEntriesPipeline_CloseStreams (0.02s) === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=0 -2025-10-27T04:02:05.107Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" +2026-11-29T10:28:09.381Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=1 === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=2 +2026-11-29T10:28:09.434Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" === RUN TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=10 -2025-10-27T04:02:05.159Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" -2025-10-27T04:02:05.211Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" ---- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight (0.15s) +--- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight (0.16s) --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=0 (0.05s) --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=1 (0.00s) --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=2 (0.05s) --- PASS: TestNetworkTransport_AppendEntriesPipeline_MaxRPCsInFlight/max=10 (0.05s) === RUN TestNetworkTransport_RequestVote +2026-11-29T10:28:09.485Z [ERROR] raft-net: failed to decode incoming command: error="transport shutdown" --- PASS: TestNetworkTransport_RequestVote (0.00s) === RUN TestNetworkTransport_InstallSnapshot --- PASS: TestNetworkTransport_InstallSnapshot (0.00s) @@ -1043,27 +1079,27 @@ === RUN TestNetworkTransport_PooledConn --- PASS: TestNetworkTransport_PooledConn (0.00s) === RUN TestNetworkTransport_ListenBackoff - net_transport_test.go:839: countingWriter: 2025-10-27T04:02:05.217Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2026-11-29T10:28:09.493Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-10-27T04:02:05.222Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2026-11-29T10:28:09.499Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-10-27T04:02:05.232Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2026-11-29T10:28:09.509Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-10-27T04:02:05.253Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2026-11-29T10:28:09.530Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-10-27T04:02:05.294Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2026-11-29T10:28:09.571Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-10-27T04:02:05.375Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2026-11-29T10:28:09.651Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-10-27T04:02:05.535Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2026-11-29T10:28:09.812Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-10-27T04:02:05.856Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2026-11-29T10:28:10.132Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-10-27T04:02:06.497Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2026-11-29T10:28:10.773Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-10-27T04:02:07.499Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2026-11-29T10:28:11.774Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:839: countingWriter: 2025-10-27T04:02:08.501Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:839: countingWriter: 2026-11-29T10:28:12.776Z [ERROR] test: failed to accept connection: error="intentional error in test" --- PASS: TestNetworkTransport_ListenBackoff (4.00s) === RUN TestPeersJSON_BadConfiguration @@ -1073,2523 +1109,2516 @@ === RUN TestPeersJSON_ReadConfigJSON --- PASS: TestPeersJSON_ReadConfigJSON (0.00s) === RUN TestRaft_StartStop -2025-10-27T04:02:09.219Z [INFO] server-9b4d3f5b-73d2-71b4-2075-3548b12aabf4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9b4d3f5b-73d2-71b4-2075-3548b12aabf4 Address:9b4d3f5b-73d2-71b4-2075-3548b12aabf4}]" +2026-11-29T10:28:13.494Z [INFO] server-e85565e0-0e51-75f4-1b5a-8a5322ca5566: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e85565e0-0e51-75f4-1b5a-8a5322ca5566 Address:e85565e0-0e51-75f4-1b5a-8a5322ca5566}]" --- PASS: TestRaft_StartStop (0.00s) === RUN TestRaft_AfterShutdown -2025-10-27T04:02:09.220Z [INFO] server-c19c5b7b-41b1-d7f0-85d3-13efd361f122: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c19c5b7b-41b1-d7f0-85d3-13efd361f122 Address:c19c5b7b-41b1-d7f0-85d3-13efd361f122}]" +2026-11-29T10:28:13.494Z [INFO] server-5d3a96f2-3cd8-543e-7afb-4fb150c43e66: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5d3a96f2-3cd8-543e-7afb-4fb150c43e66 Address:5d3a96f2-3cd8-543e-7afb-4fb150c43e66}]" --- PASS: TestRaft_AfterShutdown (0.00s) === RUN TestRaft_LiveBootstrap -2025-10-27T04:02:09.220Z [INFO] server-1d33085f-923e-7bdc-310c-0c36abc0b9a2: initial configuration: index=0 servers=[] -2025-10-27T04:02:09.220Z [INFO] server-0c9e66e9-b717-2405-654d-88af12eefb32: initial configuration: index=0 servers=[] -2025-10-27T04:02:09.220Z [INFO] server-6b3fcb6c-947b-55af-2d34-b3a88dae52d9: initial configuration: index=0 servers=[] -2025-10-27T04:02:09.220Z [INFO] server-1d33085f-923e-7bdc-310c-0c36abc0b9a2: entering follower state: follower="Node at 1d33085f-923e-7bdc-310c-0c36abc0b9a2 [Follower]" leader-address= leader-id= -2025-10-27T04:02:09.220Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:02:09.220Z [INFO] server-6b3fcb6c-947b-55af-2d34-b3a88dae52d9: entering follower state: follower="Node at 6b3fcb6c-947b-55af-2d34-b3a88dae52d9 [Follower]" leader-address= leader-id= -2025-10-27T04:02:09.220Z [INFO] server-0c9e66e9-b717-2405-654d-88af12eefb32: entering follower state: follower="Node at 0c9e66e9-b717-2405-654d-88af12eefb32 [Follower]" leader-address= leader-id= -2025-10-27T04:02:09.290Z [WARN] server-6b3fcb6c-947b-55af-2d34-b3a88dae52d9: no known peers, aborting election -2025-10-27T04:02:09.309Z [WARN] server-1d33085f-923e-7bdc-310c-0c36abc0b9a2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:09.309Z [INFO] server-1d33085f-923e-7bdc-310c-0c36abc0b9a2: entering candidate state: node="Node at 1d33085f-923e-7bdc-310c-0c36abc0b9a2 [Candidate]" term=2 -2025-10-27T04:02:09.309Z [INFO] server-1d33085f-923e-7bdc-310c-0c36abc0b9a2: election won: term=2 tally=2 -2025-10-27T04:02:09.309Z [INFO] server-1d33085f-923e-7bdc-310c-0c36abc0b9a2: entering leader state: leader="Node at 1d33085f-923e-7bdc-310c-0c36abc0b9a2 [Leader]" -2025-10-27T04:02:09.309Z [INFO] server-1d33085f-923e-7bdc-310c-0c36abc0b9a2: added peer, starting replication: peer=server-0c9e66e9-b717-2405-654d-88af12eefb32 -2025-10-27T04:02:09.309Z [INFO] server-1d33085f-923e-7bdc-310c-0c36abc0b9a2: added peer, starting replication: peer=server-6b3fcb6c-947b-55af-2d34-b3a88dae52d9 -2025-10-27T04:02:09.309Z [WARN] server-6b3fcb6c-947b-55af-2d34-b3a88dae52d9: failed to get previous log: previous-index=1 last-index=0 error="log not found" -2025-10-27T04:02:09.309Z [WARN] server-1d33085f-923e-7bdc-310c-0c36abc0b9a2: appendEntries rejected, sending older logs: peer="{Voter server-6b3fcb6c-947b-55af-2d34-b3a88dae52d9 6b3fcb6c-947b-55af-2d34-b3a88dae52d9}" next=1 -2025-10-27T04:02:09.310Z [INFO] server-1d33085f-923e-7bdc-310c-0c36abc0b9a2: pipelining replication: peer="{Voter server-6b3fcb6c-947b-55af-2d34-b3a88dae52d9 6b3fcb6c-947b-55af-2d34-b3a88dae52d9}" -2025-10-27T04:02:09.309Z [WARN] server-0c9e66e9-b717-2405-654d-88af12eefb32: failed to get previous log: previous-index=1 last-index=0 error="log not found" -2025-10-27T04:02:09.310Z [WARN] server-1d33085f-923e-7bdc-310c-0c36abc0b9a2: appendEntries rejected, sending older logs: peer="{Voter server-0c9e66e9-b717-2405-654d-88af12eefb32 0c9e66e9-b717-2405-654d-88af12eefb32}" next=1 -2025-10-27T04:02:09.310Z [INFO] server-1d33085f-923e-7bdc-310c-0c36abc0b9a2: pipelining replication: peer="{Voter server-0c9e66e9-b717-2405-654d-88af12eefb32 0c9e66e9-b717-2405-654d-88af12eefb32}" -2025-10-27T04:02:09.415Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:02:09.309987428 +0000 UTC m=+4.304859906 (2 nodes), 89.117689ms from start of poll, 89.440452ms from cluster start. Timeout at 2025-10-27 04:02:09.414993668 +0000 UTC m=+4.409866146, 105.00624ms after stability -2025-10-27T04:02:09.415Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:09.521Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:09.415274067 +0000 UTC m=+4.410146545 (1 nodes), 1.16µs from start of poll, 194.727091ms from cluster start. Timeout at 2025-10-27 04:02:09.520276947 +0000 UTC m=+4.515149385, 105.00284ms after stability -2025-10-27T04:02:09.532Z [INFO] server-1d33085f-923e-7bdc-310c-0c36abc0b9a2: aborting pipeline replication: peer="{Voter server-6b3fcb6c-947b-55af-2d34-b3a88dae52d9 6b3fcb6c-947b-55af-2d34-b3a88dae52d9}" -2025-10-27T04:02:09.532Z [INFO] server-1d33085f-923e-7bdc-310c-0c36abc0b9a2: aborting pipeline replication: peer="{Voter server-0c9e66e9-b717-2405-654d-88af12eefb32 0c9e66e9-b717-2405-654d-88af12eefb32}" ---- PASS: TestRaft_LiveBootstrap (0.31s) +2026-11-29T10:28:13.495Z [INFO] server-b0e8129d-2766-fbf5-de53-1d15474424f6: initial configuration: index=0 servers=[] +2026-11-29T10:28:13.495Z [INFO] server-01d51c4f-ce0b-28cf-bc68-7fdc435f2702: initial configuration: index=0 servers=[] +2026-11-29T10:28:13.495Z [INFO] server-33410546-20e2-fd7d-175d-4a3a29b5c0ba: initial configuration: index=0 servers=[] +2026-11-29T10:28:13.495Z [INFO] server-b0e8129d-2766-fbf5-de53-1d15474424f6: entering follower state: follower="Node at b0e8129d-2766-fbf5-de53-1d15474424f6 [Follower]" leader-address= leader-id= +2026-11-29T10:28:13.495Z [INFO] server-01d51c4f-ce0b-28cf-bc68-7fdc435f2702: entering follower state: follower="Node at 01d51c4f-ce0b-28cf-bc68-7fdc435f2702 [Follower]" leader-address= leader-id= +2026-11-29T10:28:13.495Z [INFO] server-33410546-20e2-fd7d-175d-4a3a29b5c0ba: entering follower state: follower="Node at 33410546-20e2-fd7d-175d-4a3a29b5c0ba [Follower]" leader-address= leader-id= +2026-11-29T10:28:13.495Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:28:13.551Z [WARN] server-01d51c4f-ce0b-28cf-bc68-7fdc435f2702: no known peers, aborting election +2026-11-29T10:28:13.554Z [WARN] server-b0e8129d-2766-fbf5-de53-1d15474424f6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:13.554Z [INFO] server-b0e8129d-2766-fbf5-de53-1d15474424f6: entering candidate state: node="Node at b0e8129d-2766-fbf5-de53-1d15474424f6 [Candidate]" term=2 +2026-11-29T10:28:13.554Z [INFO] server-b0e8129d-2766-fbf5-de53-1d15474424f6: election won: term=2 tally=2 +2026-11-29T10:28:13.554Z [INFO] server-b0e8129d-2766-fbf5-de53-1d15474424f6: entering leader state: leader="Node at b0e8129d-2766-fbf5-de53-1d15474424f6 [Leader]" +2026-11-29T10:28:13.554Z [INFO] server-b0e8129d-2766-fbf5-de53-1d15474424f6: added peer, starting replication: peer=server-01d51c4f-ce0b-28cf-bc68-7fdc435f2702 +2026-11-29T10:28:13.554Z [INFO] server-b0e8129d-2766-fbf5-de53-1d15474424f6: added peer, starting replication: peer=server-33410546-20e2-fd7d-175d-4a3a29b5c0ba +2026-11-29T10:28:13.555Z [WARN] server-33410546-20e2-fd7d-175d-4a3a29b5c0ba: failed to get previous log: previous-index=1 last-index=0 error="log not found" +2026-11-29T10:28:13.555Z [WARN] server-01d51c4f-ce0b-28cf-bc68-7fdc435f2702: failed to get previous log: previous-index=1 last-index=0 error="log not found" +2026-11-29T10:28:13.555Z [WARN] server-b0e8129d-2766-fbf5-de53-1d15474424f6: appendEntries rejected, sending older logs: peer="{Voter server-01d51c4f-ce0b-28cf-bc68-7fdc435f2702 01d51c4f-ce0b-28cf-bc68-7fdc435f2702}" next=1 +2026-11-29T10:28:13.555Z [WARN] server-b0e8129d-2766-fbf5-de53-1d15474424f6: appendEntries rejected, sending older logs: peer="{Voter server-33410546-20e2-fd7d-175d-4a3a29b5c0ba 33410546-20e2-fd7d-175d-4a3a29b5c0ba}" next=1 +2026-11-29T10:28:13.555Z [INFO] server-b0e8129d-2766-fbf5-de53-1d15474424f6: pipelining replication: peer="{Voter server-01d51c4f-ce0b-28cf-bc68-7fdc435f2702 01d51c4f-ce0b-28cf-bc68-7fdc435f2702}" +2026-11-29T10:28:13.555Z [INFO] server-b0e8129d-2766-fbf5-de53-1d15474424f6: pipelining replication: peer="{Voter server-33410546-20e2-fd7d-175d-4a3a29b5c0ba 33410546-20e2-fd7d-175d-4a3a29b5c0ba}" +2026-11-29T10:28:13.660Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:28:13.555121419 +0000 UTC m=+4.290175481 (2 nodes), 59.81556ms from start of poll, 60.0476ms from cluster start. Timeout at 2026-11-29 10:28:13.660128779 +0000 UTC m=+4.395182921, 105.00744ms after stability +2026-11-29T10:28:13.660Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:13.766Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:13.660952899 +0000 UTC m=+4.396007041 (1 nodes), 1.88µs from start of poll, 165.87916ms from cluster start. Timeout at 2026-11-29 10:28:13.765955979 +0000 UTC m=+4.501010081, 105.00304ms after stability +2026-11-29T10:28:13.777Z [INFO] server-b0e8129d-2766-fbf5-de53-1d15474424f6: aborting pipeline replication: peer="{Voter server-33410546-20e2-fd7d-175d-4a3a29b5c0ba 33410546-20e2-fd7d-175d-4a3a29b5c0ba}" +2026-11-29T10:28:13.777Z [INFO] server-b0e8129d-2766-fbf5-de53-1d15474424f6: aborting pipeline replication: peer="{Voter server-01d51c4f-ce0b-28cf-bc68-7fdc435f2702 01d51c4f-ce0b-28cf-bc68-7fdc435f2702}" +--- PASS: TestRaft_LiveBootstrap (0.28s) === RUN TestRaft_LiveBootstrap_From_NonVoter -2025-10-27T04:02:09.533Z [INFO] server-f932cc8d-eb32-92e9-394d-1cb38c858a27: initial configuration: index=0 servers=[] -2025-10-27T04:02:09.533Z [INFO] server-4cc32aa7-5e0a-4d9a-2d25-30c8cc49de04: initial configuration: index=0 servers=[] -2025-10-27T04:02:09.533Z [INFO] server-f932cc8d-eb32-92e9-394d-1cb38c858a27: entering follower state: follower="Node at f932cc8d-eb32-92e9-394d-1cb38c858a27 [Follower]" leader-address= leader-id= -2025-10-27T04:02:09.533Z [INFO] server-4cc32aa7-5e0a-4d9a-2d25-30c8cc49de04: entering follower state: follower="Node at 4cc32aa7-5e0a-4d9a-2d25-30c8cc49de04 [Follower]" leader-address= leader-id= +2026-11-29T10:28:13.778Z [INFO] server-25d2f106-51ff-cf4a-7827-8f8c1f04e54c: initial configuration: index=0 servers=[] +2026-11-29T10:28:13.778Z [INFO] server-25d2f106-51ff-cf4a-7827-8f8c1f04e54c: entering follower state: follower="Node at 25d2f106-51ff-cf4a-7827-8f8c1f04e54c [Follower]" leader-address= leader-id= +2026-11-29T10:28:13.778Z [INFO] server-ddc86698-f1c5-75e7-1ef1-4e84aa894377: initial configuration: index=0 servers=[] --- PASS: TestRaft_LiveBootstrap_From_NonVoter (0.00s) === RUN TestRaft_RecoverCluster_NoState -2025-10-27T04:02:09.533Z [INFO] server-76334cfb-b9ef-01f7-b82c-8b0bcb9f1258: initial configuration: index=0 servers=[] -2025-10-27T04:02:09.533Z [INFO] server-76334cfb-b9ef-01f7-b82c-8b0bcb9f1258: entering follower state: follower="Node at 76334cfb-b9ef-01f7-b82c-8b0bcb9f1258 [Shutdown]" leader-address= leader-id= +2026-11-29T10:28:13.779Z [INFO] server-b2ad48ca-fe9f-ecdc-2c93-5dd9144c7f34: initial configuration: index=0 servers=[] --- PASS: TestRaft_RecoverCluster_NoState (0.00s) === RUN TestRaft_RecoverCluster === RUN TestRaft_RecoverCluster/no_snapshot,_no_trailing_logs -2025-10-27T04:02:09.534Z [INFO] server-770adc8b-eab4-25df-0c1d-d257eeac57a9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-770adc8b-eab4-25df-0c1d-d257eeac57a9 Address:770adc8b-eab4-25df-0c1d-d257eeac57a9} {Suffrage:Voter ID:server-84187b37-5eb9-42df-1206-8ceeb9a14fc4 Address:84187b37-5eb9-42df-1206-8ceeb9a14fc4} {Suffrage:Voter ID:server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6 Address:41b839c1-861d-3dc7-e4b5-4c79c9589aa6}]" -2025-10-27T04:02:09.534Z [INFO] server-84187b37-5eb9-42df-1206-8ceeb9a14fc4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-770adc8b-eab4-25df-0c1d-d257eeac57a9 Address:770adc8b-eab4-25df-0c1d-d257eeac57a9} {Suffrage:Voter ID:server-84187b37-5eb9-42df-1206-8ceeb9a14fc4 Address:84187b37-5eb9-42df-1206-8ceeb9a14fc4} {Suffrage:Voter ID:server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6 Address:41b839c1-861d-3dc7-e4b5-4c79c9589aa6}]" -2025-10-27T04:02:09.534Z [INFO] server-770adc8b-eab4-25df-0c1d-d257eeac57a9: entering follower state: follower="Node at 770adc8b-eab4-25df-0c1d-d257eeac57a9 [Follower]" leader-address= leader-id= -2025-10-27T04:02:09.534Z [INFO] server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-770adc8b-eab4-25df-0c1d-d257eeac57a9 Address:770adc8b-eab4-25df-0c1d-d257eeac57a9} {Suffrage:Voter ID:server-84187b37-5eb9-42df-1206-8ceeb9a14fc4 Address:84187b37-5eb9-42df-1206-8ceeb9a14fc4} {Suffrage:Voter ID:server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6 Address:41b839c1-861d-3dc7-e4b5-4c79c9589aa6}]" -2025-10-27T04:02:09.534Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:09.534Z [INFO] server-84187b37-5eb9-42df-1206-8ceeb9a14fc4: entering follower state: follower="Node at 84187b37-5eb9-42df-1206-8ceeb9a14fc4 [Follower]" leader-address= leader-id= -2025-10-27T04:02:09.534Z [INFO] server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6: entering follower state: follower="Node at 41b839c1-861d-3dc7-e4b5-4c79c9589aa6 [Follower]" leader-address= leader-id= -2025-10-27T04:02:09.587Z [WARN] server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:09.587Z [INFO] server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6: entering candidate state: node="Node at 41b839c1-861d-3dc7-e4b5-4c79c9589aa6 [Candidate]" term=2 -2025-10-27T04:02:09.588Z [INFO] server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6: election won: term=2 tally=2 -2025-10-27T04:02:09.588Z [INFO] server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6: entering leader state: leader="Node at 41b839c1-861d-3dc7-e4b5-4c79c9589aa6 [Leader]" -2025-10-27T04:02:09.588Z [INFO] server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6: added peer, starting replication: peer=server-770adc8b-eab4-25df-0c1d-d257eeac57a9 -2025-10-27T04:02:09.588Z [INFO] server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6: added peer, starting replication: peer=server-84187b37-5eb9-42df-1206-8ceeb9a14fc4 -2025-10-27T04:02:09.588Z [INFO] server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6: pipelining replication: peer="{Voter server-84187b37-5eb9-42df-1206-8ceeb9a14fc4 84187b37-5eb9-42df-1206-8ceeb9a14fc4}" -2025-10-27T04:02:09.588Z [INFO] server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6: pipelining replication: peer="{Voter server-770adc8b-eab4-25df-0c1d-d257eeac57a9 770adc8b-eab4-25df-0c1d-d257eeac57a9}" -2025-10-27T04:02:09.693Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:09.58830433 +0000 UTC m=+4.583176768 (1 nodes), 53.753527ms from start of poll, 53.997769ms from cluster start. Timeout at 2025-10-27 04:02:09.69330645 +0000 UTC m=+4.688178888, 105.00212ms after stability -2025-10-27T04:02:09.693Z [INFO] server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6: aborting pipeline replication: peer="{Voter server-770adc8b-eab4-25df-0c1d-d257eeac57a9 770adc8b-eab4-25df-0c1d-d257eeac57a9}" -2025-10-27T04:02:09.693Z [INFO] server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6: aborting pipeline replication: peer="{Voter server-84187b37-5eb9-42df-1206-8ceeb9a14fc4 84187b37-5eb9-42df-1206-8ceeb9a14fc4}" -2025-10-27T04:02:09.694Z [INFO] creating new snapshot: path=/tmp/raft3150493393/snapshots/2-2-1761537729694.tmp -2025-10-27T04:02:09.694Z [INFO] server-770adc8b-eab4-25df-0c1d-d257eeac57a9: starting restore from snapshot: id=2-2-1761537729694 last-index=2 last-term=2 size-in-bytes=1 -2025-10-27T04:02:09.694Z [INFO] server-770adc8b-eab4-25df-0c1d-d257eeac57a9: snapshot restore progress: id=2-2-1761537729694 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% -2025-10-27T04:02:09.694Z [INFO] server-770adc8b-eab4-25df-0c1d-d257eeac57a9: restored from snapshot: id=2-2-1761537729694 last-index=2 last-term=2 size-in-bytes=1 -2025-10-27T04:02:09.694Z [INFO] server-770adc8b-eab4-25df-0c1d-d257eeac57a9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-770adc8b-eab4-25df-0c1d-d257eeac57a9 Address:770adc8b-eab4-25df-0c1d-d257eeac57a9} {Suffrage:Voter ID:server-84187b37-5eb9-42df-1206-8ceeb9a14fc4 Address:84187b37-5eb9-42df-1206-8ceeb9a14fc4} {Suffrage:Voter ID:server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6 Address:41b839c1-861d-3dc7-e4b5-4c79c9589aa6}]" -2025-10-27T04:02:09.694Z [INFO] creating new snapshot: path=/tmp/raft2836865857/snapshots/2-2-1761537729694.tmp -2025-10-27T04:02:09.694Z [INFO] server-84187b37-5eb9-42df-1206-8ceeb9a14fc4: starting restore from snapshot: id=2-2-1761537729694 last-index=2 last-term=2 size-in-bytes=1 -2025-10-27T04:02:09.694Z [INFO] server-84187b37-5eb9-42df-1206-8ceeb9a14fc4: snapshot restore progress: id=2-2-1761537729694 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% -2025-10-27T04:02:09.694Z [INFO] server-84187b37-5eb9-42df-1206-8ceeb9a14fc4: restored from snapshot: id=2-2-1761537729694 last-index=2 last-term=2 size-in-bytes=1 -2025-10-27T04:02:09.694Z [INFO] server-84187b37-5eb9-42df-1206-8ceeb9a14fc4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-770adc8b-eab4-25df-0c1d-d257eeac57a9 Address:770adc8b-eab4-25df-0c1d-d257eeac57a9} {Suffrage:Voter ID:server-84187b37-5eb9-42df-1206-8ceeb9a14fc4 Address:84187b37-5eb9-42df-1206-8ceeb9a14fc4} {Suffrage:Voter ID:server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6 Address:41b839c1-861d-3dc7-e4b5-4c79c9589aa6}]" -2025-10-27T04:02:09.694Z [INFO] creating new snapshot: path=/tmp/raft3545354906/snapshots/2-2-1761537729694.tmp -2025-10-27T04:02:09.695Z [INFO] server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6: starting restore from snapshot: id=2-2-1761537729694 last-index=2 last-term=2 size-in-bytes=1 -2025-10-27T04:02:09.695Z [INFO] server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6: snapshot restore progress: id=2-2-1761537729694 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% -2025-10-27T04:02:09.695Z [INFO] server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6: restored from snapshot: id=2-2-1761537729694 last-index=2 last-term=2 size-in-bytes=1 -2025-10-27T04:02:09.695Z [INFO] server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-770adc8b-eab4-25df-0c1d-d257eeac57a9 Address:770adc8b-eab4-25df-0c1d-d257eeac57a9} {Suffrage:Voter ID:server-84187b37-5eb9-42df-1206-8ceeb9a14fc4 Address:84187b37-5eb9-42df-1206-8ceeb9a14fc4} {Suffrage:Voter ID:server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6 Address:41b839c1-861d-3dc7-e4b5-4c79c9589aa6}]" -2025-10-27T04:02:09.695Z [INFO] server-770adc8b-eab4-25df-0c1d-d257eeac57a9: entering follower state: follower="Node at 770adc8b-eab4-25df-0c1d-d257eeac57a9 [Follower]" leader-address= leader-id= -2025-10-27T04:02:09.695Z [INFO] server-84187b37-5eb9-42df-1206-8ceeb9a14fc4: entering follower state: follower="Node at 84187b37-5eb9-42df-1206-8ceeb9a14fc4 [Follower]" leader-address= leader-id= -2025-10-27T04:02:09.695Z [INFO] server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6: entering follower state: follower="Node at 41b839c1-861d-3dc7-e4b5-4c79c9589aa6 [Follower]" leader-address= leader-id= -2025-10-27T04:02:09.752Z [WARN] server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:09.752Z [INFO] server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6: entering candidate state: node="Node at 41b839c1-861d-3dc7-e4b5-4c79c9589aa6 [Candidate]" term=3 -2025-10-27T04:02:09.752Z [INFO] server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6: election won: term=3 tally=2 -2025-10-27T04:02:09.752Z [INFO] server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6: entering leader state: leader="Node at 41b839c1-861d-3dc7-e4b5-4c79c9589aa6 [Leader]" -2025-10-27T04:02:09.752Z [INFO] server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6: added peer, starting replication: peer=server-770adc8b-eab4-25df-0c1d-d257eeac57a9 -2025-10-27T04:02:09.752Z [INFO] server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6: added peer, starting replication: peer=server-84187b37-5eb9-42df-1206-8ceeb9a14fc4 -2025-10-27T04:02:09.752Z [INFO] server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6: pipelining replication: peer="{Voter server-770adc8b-eab4-25df-0c1d-d257eeac57a9 770adc8b-eab4-25df-0c1d-d257eeac57a9}" -2025-10-27T04:02:09.752Z [INFO] server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6: pipelining replication: peer="{Voter server-84187b37-5eb9-42df-1206-8ceeb9a14fc4 84187b37-5eb9-42df-1206-8ceeb9a14fc4}" -2025-10-27T04:02:10.010Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:10.115Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:10.010472976 +0000 UTC m=+5.005345454 (1 nodes), 34.64µs from start of poll, 476.166455ms from cluster start. Timeout at 2025-10-27 04:02:10.115479136 +0000 UTC m=+5.110351574, 105.00612ms after stability -2025-10-27T04:02:10.115Z [INFO] server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6: aborting pipeline replication: peer="{Voter server-84187b37-5eb9-42df-1206-8ceeb9a14fc4 84187b37-5eb9-42df-1206-8ceeb9a14fc4}" -2025-10-27T04:02:10.115Z [INFO] server-41b839c1-861d-3dc7-e4b5-4c79c9589aa6: aborting pipeline replication: peer="{Voter server-770adc8b-eab4-25df-0c1d-d257eeac57a9 770adc8b-eab4-25df-0c1d-d257eeac57a9}" +2026-11-29T10:28:13.779Z [INFO] server-954c2f0c-1b3e-6965-7779-652cc6fe224b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-954c2f0c-1b3e-6965-7779-652cc6fe224b Address:954c2f0c-1b3e-6965-7779-652cc6fe224b} {Suffrage:Voter ID:server-c857cd48-53fd-50e1-29d1-c720e13284de Address:c857cd48-53fd-50e1-29d1-c720e13284de} {Suffrage:Voter ID:server-ea03f2bd-13bb-1e78-5380-5b6441d513c3 Address:ea03f2bd-13bb-1e78-5380-5b6441d513c3}]" +2026-11-29T10:28:13.779Z [INFO] server-c857cd48-53fd-50e1-29d1-c720e13284de: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-954c2f0c-1b3e-6965-7779-652cc6fe224b Address:954c2f0c-1b3e-6965-7779-652cc6fe224b} {Suffrage:Voter ID:server-c857cd48-53fd-50e1-29d1-c720e13284de Address:c857cd48-53fd-50e1-29d1-c720e13284de} {Suffrage:Voter ID:server-ea03f2bd-13bb-1e78-5380-5b6441d513c3 Address:ea03f2bd-13bb-1e78-5380-5b6441d513c3}]" +2026-11-29T10:28:13.779Z [INFO] server-954c2f0c-1b3e-6965-7779-652cc6fe224b: entering follower state: follower="Node at 954c2f0c-1b3e-6965-7779-652cc6fe224b [Follower]" leader-address= leader-id= +2026-11-29T10:28:13.779Z [INFO] server-c857cd48-53fd-50e1-29d1-c720e13284de: entering follower state: follower="Node at c857cd48-53fd-50e1-29d1-c720e13284de [Follower]" leader-address= leader-id= +2026-11-29T10:28:13.779Z [INFO] server-ea03f2bd-13bb-1e78-5380-5b6441d513c3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-954c2f0c-1b3e-6965-7779-652cc6fe224b Address:954c2f0c-1b3e-6965-7779-652cc6fe224b} {Suffrage:Voter ID:server-c857cd48-53fd-50e1-29d1-c720e13284de Address:c857cd48-53fd-50e1-29d1-c720e13284de} {Suffrage:Voter ID:server-ea03f2bd-13bb-1e78-5380-5b6441d513c3 Address:ea03f2bd-13bb-1e78-5380-5b6441d513c3}]" +2026-11-29T10:28:13.779Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:13.779Z [INFO] server-ea03f2bd-13bb-1e78-5380-5b6441d513c3: entering follower state: follower="Node at ea03f2bd-13bb-1e78-5380-5b6441d513c3 [Follower]" leader-address= leader-id= +2026-11-29T10:28:13.832Z [WARN] server-ea03f2bd-13bb-1e78-5380-5b6441d513c3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:13.832Z [INFO] server-ea03f2bd-13bb-1e78-5380-5b6441d513c3: entering candidate state: node="Node at ea03f2bd-13bb-1e78-5380-5b6441d513c3 [Candidate]" term=2 +2026-11-29T10:28:13.832Z [INFO] server-ea03f2bd-13bb-1e78-5380-5b6441d513c3: election won: term=2 tally=2 +2026-11-29T10:28:13.832Z [INFO] server-ea03f2bd-13bb-1e78-5380-5b6441d513c3: entering leader state: leader="Node at ea03f2bd-13bb-1e78-5380-5b6441d513c3 [Leader]" +2026-11-29T10:28:13.832Z [INFO] server-ea03f2bd-13bb-1e78-5380-5b6441d513c3: added peer, starting replication: peer=server-954c2f0c-1b3e-6965-7779-652cc6fe224b +2026-11-29T10:28:13.832Z [INFO] server-ea03f2bd-13bb-1e78-5380-5b6441d513c3: added peer, starting replication: peer=server-c857cd48-53fd-50e1-29d1-c720e13284de +2026-11-29T10:28:13.832Z [INFO] server-ea03f2bd-13bb-1e78-5380-5b6441d513c3: pipelining replication: peer="{Voter server-954c2f0c-1b3e-6965-7779-652cc6fe224b 954c2f0c-1b3e-6965-7779-652cc6fe224b}" +2026-11-29T10:28:13.832Z [INFO] server-ea03f2bd-13bb-1e78-5380-5b6441d513c3: pipelining replication: peer="{Voter server-c857cd48-53fd-50e1-29d1-c720e13284de c857cd48-53fd-50e1-29d1-c720e13284de}" +2026-11-29T10:28:13.937Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:13.832424019 +0000 UTC m=+4.567478201 (1 nodes), 52.70836ms from start of poll, 52.9998ms from cluster start. Timeout at 2026-11-29 10:28:13.937426099 +0000 UTC m=+4.672480161, 105.00196ms after stability +2026-11-29T10:28:13.938Z [INFO] server-ea03f2bd-13bb-1e78-5380-5b6441d513c3: aborting pipeline replication: peer="{Voter server-c857cd48-53fd-50e1-29d1-c720e13284de c857cd48-53fd-50e1-29d1-c720e13284de}" +2026-11-29T10:28:13.938Z [INFO] server-ea03f2bd-13bb-1e78-5380-5b6441d513c3: aborting pipeline replication: peer="{Voter server-954c2f0c-1b3e-6965-7779-652cc6fe224b 954c2f0c-1b3e-6965-7779-652cc6fe224b}" +2026-11-29T10:28:13.938Z [INFO] creating new snapshot: path=/tmp/raft3932556438/snapshots/2-2-1795948093938.tmp +2026-11-29T10:28:13.938Z [INFO] server-954c2f0c-1b3e-6965-7779-652cc6fe224b: starting restore from snapshot: id=2-2-1795948093938 last-index=2 last-term=2 size-in-bytes=1 +2026-11-29T10:28:13.938Z [INFO] server-954c2f0c-1b3e-6965-7779-652cc6fe224b: snapshot restore progress: id=2-2-1795948093938 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% +2026-11-29T10:28:13.938Z [INFO] server-954c2f0c-1b3e-6965-7779-652cc6fe224b: restored from snapshot: id=2-2-1795948093938 last-index=2 last-term=2 size-in-bytes=1 +2026-11-29T10:28:13.938Z [INFO] server-954c2f0c-1b3e-6965-7779-652cc6fe224b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-954c2f0c-1b3e-6965-7779-652cc6fe224b Address:954c2f0c-1b3e-6965-7779-652cc6fe224b} {Suffrage:Voter ID:server-c857cd48-53fd-50e1-29d1-c720e13284de Address:c857cd48-53fd-50e1-29d1-c720e13284de} {Suffrage:Voter ID:server-ea03f2bd-13bb-1e78-5380-5b6441d513c3 Address:ea03f2bd-13bb-1e78-5380-5b6441d513c3}]" +2026-11-29T10:28:13.939Z [INFO] creating new snapshot: path=/tmp/raft3824934671/snapshots/2-2-1795948093939.tmp +2026-11-29T10:28:13.939Z [INFO] server-954c2f0c-1b3e-6965-7779-652cc6fe224b: entering follower state: follower="Node at 954c2f0c-1b3e-6965-7779-652cc6fe224b [Follower]" leader-address= leader-id= +2026-11-29T10:28:13.939Z [INFO] server-c857cd48-53fd-50e1-29d1-c720e13284de: starting restore from snapshot: id=2-2-1795948093939 last-index=2 last-term=2 size-in-bytes=1 +2026-11-29T10:28:13.939Z [INFO] server-c857cd48-53fd-50e1-29d1-c720e13284de: snapshot restore progress: id=2-2-1795948093939 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% +2026-11-29T10:28:13.939Z [INFO] server-c857cd48-53fd-50e1-29d1-c720e13284de: restored from snapshot: id=2-2-1795948093939 last-index=2 last-term=2 size-in-bytes=1 +2026-11-29T10:28:13.939Z [INFO] server-c857cd48-53fd-50e1-29d1-c720e13284de: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-954c2f0c-1b3e-6965-7779-652cc6fe224b Address:954c2f0c-1b3e-6965-7779-652cc6fe224b} {Suffrage:Voter ID:server-c857cd48-53fd-50e1-29d1-c720e13284de Address:c857cd48-53fd-50e1-29d1-c720e13284de} {Suffrage:Voter ID:server-ea03f2bd-13bb-1e78-5380-5b6441d513c3 Address:ea03f2bd-13bb-1e78-5380-5b6441d513c3}]" +2026-11-29T10:28:13.939Z [INFO] creating new snapshot: path=/tmp/raft1799701678/snapshots/2-2-1795948093939.tmp +2026-11-29T10:28:13.939Z [INFO] server-c857cd48-53fd-50e1-29d1-c720e13284de: entering follower state: follower="Node at c857cd48-53fd-50e1-29d1-c720e13284de [Follower]" leader-address= leader-id= +2026-11-29T10:28:13.939Z [INFO] server-ea03f2bd-13bb-1e78-5380-5b6441d513c3: starting restore from snapshot: id=2-2-1795948093939 last-index=2 last-term=2 size-in-bytes=1 +2026-11-29T10:28:13.940Z [INFO] server-ea03f2bd-13bb-1e78-5380-5b6441d513c3: snapshot restore progress: id=2-2-1795948093939 last-index=2 last-term=2 size-in-bytes=1 read-bytes=1 percent-complete=100.00% +2026-11-29T10:28:13.940Z [INFO] server-ea03f2bd-13bb-1e78-5380-5b6441d513c3: restored from snapshot: id=2-2-1795948093939 last-index=2 last-term=2 size-in-bytes=1 +2026-11-29T10:28:13.940Z [INFO] server-ea03f2bd-13bb-1e78-5380-5b6441d513c3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-954c2f0c-1b3e-6965-7779-652cc6fe224b Address:954c2f0c-1b3e-6965-7779-652cc6fe224b} {Suffrage:Voter ID:server-c857cd48-53fd-50e1-29d1-c720e13284de Address:c857cd48-53fd-50e1-29d1-c720e13284de} {Suffrage:Voter ID:server-ea03f2bd-13bb-1e78-5380-5b6441d513c3 Address:ea03f2bd-13bb-1e78-5380-5b6441d513c3}]" +2026-11-29T10:28:13.940Z [INFO] server-ea03f2bd-13bb-1e78-5380-5b6441d513c3: entering follower state: follower="Node at ea03f2bd-13bb-1e78-5380-5b6441d513c3 [Follower]" leader-address= leader-id= +2026-11-29T10:28:14.011Z [WARN] server-ea03f2bd-13bb-1e78-5380-5b6441d513c3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:14.011Z [INFO] server-ea03f2bd-13bb-1e78-5380-5b6441d513c3: entering candidate state: node="Node at ea03f2bd-13bb-1e78-5380-5b6441d513c3 [Candidate]" term=3 +2026-11-29T10:28:14.011Z [INFO] server-ea03f2bd-13bb-1e78-5380-5b6441d513c3: election won: term=3 tally=2 +2026-11-29T10:28:14.011Z [INFO] server-ea03f2bd-13bb-1e78-5380-5b6441d513c3: entering leader state: leader="Node at ea03f2bd-13bb-1e78-5380-5b6441d513c3 [Leader]" +2026-11-29T10:28:14.011Z [INFO] server-ea03f2bd-13bb-1e78-5380-5b6441d513c3: added peer, starting replication: peer=server-954c2f0c-1b3e-6965-7779-652cc6fe224b +2026-11-29T10:28:14.011Z [INFO] server-ea03f2bd-13bb-1e78-5380-5b6441d513c3: added peer, starting replication: peer=server-c857cd48-53fd-50e1-29d1-c720e13284de +2026-11-29T10:28:14.011Z [INFO] server-ea03f2bd-13bb-1e78-5380-5b6441d513c3: pipelining replication: peer="{Voter server-c857cd48-53fd-50e1-29d1-c720e13284de c857cd48-53fd-50e1-29d1-c720e13284de}" +2026-11-29T10:28:14.011Z [INFO] server-ea03f2bd-13bb-1e78-5380-5b6441d513c3: pipelining replication: peer="{Voter server-954c2f0c-1b3e-6965-7779-652cc6fe224b 954c2f0c-1b3e-6965-7779-652cc6fe224b}" +2026-11-29T10:28:14.255Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:14.360Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:14.255239939 +0000 UTC m=+4.990294041 (1 nodes), 23.24µs from start of poll, 475.81564ms from cluster start. Timeout at 2026-11-29 10:28:14.360247699 +0000 UTC m=+5.095301841, 105.0078ms after stability +2026-11-29T10:28:14.360Z [INFO] server-ea03f2bd-13bb-1e78-5380-5b6441d513c3: aborting pipeline replication: peer="{Voter server-c857cd48-53fd-50e1-29d1-c720e13284de c857cd48-53fd-50e1-29d1-c720e13284de}" +2026-11-29T10:28:14.360Z [INFO] server-ea03f2bd-13bb-1e78-5380-5b6441d513c3: aborting pipeline replication: peer="{Voter server-954c2f0c-1b3e-6965-7779-652cc6fe224b 954c2f0c-1b3e-6965-7779-652cc6fe224b}" === RUN TestRaft_RecoverCluster/no_snapshot,_some_trailing_logs -2025-10-27T04:02:10.116Z [INFO] server-c9b111a1-852c-fc12-5a64-ef95c3a82a39: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c9b111a1-852c-fc12-5a64-ef95c3a82a39 Address:c9b111a1-852c-fc12-5a64-ef95c3a82a39} {Suffrage:Voter ID:server-86ae9085-65c2-80aa-8585-6562f06236ae Address:86ae9085-65c2-80aa-8585-6562f06236ae} {Suffrage:Voter ID:server-cf7fb860-b643-286d-d56d-627298cce1ab Address:cf7fb860-b643-286d-d56d-627298cce1ab}]" -2025-10-27T04:02:10.116Z [INFO] server-86ae9085-65c2-80aa-8585-6562f06236ae: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c9b111a1-852c-fc12-5a64-ef95c3a82a39 Address:c9b111a1-852c-fc12-5a64-ef95c3a82a39} {Suffrage:Voter ID:server-86ae9085-65c2-80aa-8585-6562f06236ae Address:86ae9085-65c2-80aa-8585-6562f06236ae} {Suffrage:Voter ID:server-cf7fb860-b643-286d-d56d-627298cce1ab Address:cf7fb860-b643-286d-d56d-627298cce1ab}]" -2025-10-27T04:02:10.116Z [INFO] server-cf7fb860-b643-286d-d56d-627298cce1ab: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c9b111a1-852c-fc12-5a64-ef95c3a82a39 Address:c9b111a1-852c-fc12-5a64-ef95c3a82a39} {Suffrage:Voter ID:server-86ae9085-65c2-80aa-8585-6562f06236ae Address:86ae9085-65c2-80aa-8585-6562f06236ae} {Suffrage:Voter ID:server-cf7fb860-b643-286d-d56d-627298cce1ab Address:cf7fb860-b643-286d-d56d-627298cce1ab}]" -2025-10-27T04:02:10.116Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:10.116Z [INFO] server-c9b111a1-852c-fc12-5a64-ef95c3a82a39: entering follower state: follower="Node at c9b111a1-852c-fc12-5a64-ef95c3a82a39 [Follower]" leader-address= leader-id= -2025-10-27T04:02:10.116Z [INFO] server-86ae9085-65c2-80aa-8585-6562f06236ae: entering follower state: follower="Node at 86ae9085-65c2-80aa-8585-6562f06236ae [Follower]" leader-address= leader-id= -2025-10-27T04:02:10.116Z [INFO] server-cf7fb860-b643-286d-d56d-627298cce1ab: entering follower state: follower="Node at cf7fb860-b643-286d-d56d-627298cce1ab [Follower]" leader-address= leader-id= -2025-10-27T04:02:10.182Z [WARN] server-cf7fb860-b643-286d-d56d-627298cce1ab: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:10.182Z [INFO] server-cf7fb860-b643-286d-d56d-627298cce1ab: entering candidate state: node="Node at cf7fb860-b643-286d-d56d-627298cce1ab [Candidate]" term=2 -2025-10-27T04:02:10.182Z [INFO] server-cf7fb860-b643-286d-d56d-627298cce1ab: election won: term=2 tally=2 -2025-10-27T04:02:10.182Z [INFO] server-cf7fb860-b643-286d-d56d-627298cce1ab: entering leader state: leader="Node at cf7fb860-b643-286d-d56d-627298cce1ab [Leader]" -2025-10-27T04:02:10.182Z [INFO] server-cf7fb860-b643-286d-d56d-627298cce1ab: added peer, starting replication: peer=server-c9b111a1-852c-fc12-5a64-ef95c3a82a39 -2025-10-27T04:02:10.182Z [INFO] server-cf7fb860-b643-286d-d56d-627298cce1ab: added peer, starting replication: peer=server-86ae9085-65c2-80aa-8585-6562f06236ae -2025-10-27T04:02:10.182Z [INFO] server-cf7fb860-b643-286d-d56d-627298cce1ab: pipelining replication: peer="{Voter server-86ae9085-65c2-80aa-8585-6562f06236ae 86ae9085-65c2-80aa-8585-6562f06236ae}" -2025-10-27T04:02:10.182Z [INFO] server-cf7fb860-b643-286d-d56d-627298cce1ab: pipelining replication: peer="{Voter server-c9b111a1-852c-fc12-5a64-ef95c3a82a39 c9b111a1-852c-fc12-5a64-ef95c3a82a39}" -2025-10-27T04:02:10.288Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:10.182810514 +0000 UTC m=+5.177682952 (1 nodes), 66.372814ms from start of poll, 66.552455ms from cluster start. Timeout at 2025-10-27 04:02:10.287814674 +0000 UTC m=+5.282687112, 105.00416ms after stability -2025-10-27T04:02:10.288Z [INFO] server-cf7fb860-b643-286d-d56d-627298cce1ab: aborting pipeline replication: peer="{Voter server-86ae9085-65c2-80aa-8585-6562f06236ae 86ae9085-65c2-80aa-8585-6562f06236ae}" -2025-10-27T04:02:10.288Z [INFO] server-cf7fb860-b643-286d-d56d-627298cce1ab: aborting pipeline replication: peer="{Voter server-c9b111a1-852c-fc12-5a64-ef95c3a82a39 c9b111a1-852c-fc12-5a64-ef95c3a82a39}" -2025-10-27T04:02:10.288Z [INFO] creating new snapshot: path=/tmp/raft662961119/snapshots/2-6-1761537730288.tmp -2025-10-27T04:02:10.288Z [INFO] server-c9b111a1-852c-fc12-5a64-ef95c3a82a39: starting restore from snapshot: id=2-6-1761537730288 last-index=6 last-term=2 size-in-bytes=25 -2025-10-27T04:02:10.288Z [INFO] server-c9b111a1-852c-fc12-5a64-ef95c3a82a39: snapshot restore progress: id=2-6-1761537730288 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% -2025-10-27T04:02:10.289Z [INFO] server-c9b111a1-852c-fc12-5a64-ef95c3a82a39: restored from snapshot: id=2-6-1761537730288 last-index=6 last-term=2 size-in-bytes=25 -2025-10-27T04:02:10.289Z [INFO] server-c9b111a1-852c-fc12-5a64-ef95c3a82a39: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c9b111a1-852c-fc12-5a64-ef95c3a82a39 Address:c9b111a1-852c-fc12-5a64-ef95c3a82a39} {Suffrage:Voter ID:server-86ae9085-65c2-80aa-8585-6562f06236ae Address:86ae9085-65c2-80aa-8585-6562f06236ae} {Suffrage:Voter ID:server-cf7fb860-b643-286d-d56d-627298cce1ab Address:cf7fb860-b643-286d-d56d-627298cce1ab}]" -2025-10-27T04:02:10.289Z [INFO] creating new snapshot: path=/tmp/raft4042769892/snapshots/2-6-1761537730289.tmp -2025-10-27T04:02:10.289Z [INFO] server-c9b111a1-852c-fc12-5a64-ef95c3a82a39: entering follower state: follower="Node at c9b111a1-852c-fc12-5a64-ef95c3a82a39 [Follower]" leader-address= leader-id= -2025-10-27T04:02:10.289Z [INFO] server-86ae9085-65c2-80aa-8585-6562f06236ae: starting restore from snapshot: id=2-6-1761537730289 last-index=6 last-term=2 size-in-bytes=25 -2025-10-27T04:02:10.289Z [INFO] server-86ae9085-65c2-80aa-8585-6562f06236ae: snapshot restore progress: id=2-6-1761537730289 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% -2025-10-27T04:02:10.289Z [INFO] server-86ae9085-65c2-80aa-8585-6562f06236ae: restored from snapshot: id=2-6-1761537730289 last-index=6 last-term=2 size-in-bytes=25 -2025-10-27T04:02:10.289Z [INFO] server-86ae9085-65c2-80aa-8585-6562f06236ae: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c9b111a1-852c-fc12-5a64-ef95c3a82a39 Address:c9b111a1-852c-fc12-5a64-ef95c3a82a39} {Suffrage:Voter ID:server-86ae9085-65c2-80aa-8585-6562f06236ae Address:86ae9085-65c2-80aa-8585-6562f06236ae} {Suffrage:Voter ID:server-cf7fb860-b643-286d-d56d-627298cce1ab Address:cf7fb860-b643-286d-d56d-627298cce1ab}]" -2025-10-27T04:02:10.289Z [INFO] creating new snapshot: path=/tmp/raft1161440124/snapshots/2-6-1761537730289.tmp -2025-10-27T04:02:10.289Z [INFO] server-86ae9085-65c2-80aa-8585-6562f06236ae: entering follower state: follower="Node at 86ae9085-65c2-80aa-8585-6562f06236ae [Follower]" leader-address= leader-id= -2025-10-27T04:02:10.289Z [INFO] server-cf7fb860-b643-286d-d56d-627298cce1ab: starting restore from snapshot: id=2-6-1761537730289 last-index=6 last-term=2 size-in-bytes=25 -2025-10-27T04:02:10.289Z [INFO] server-cf7fb860-b643-286d-d56d-627298cce1ab: snapshot restore progress: id=2-6-1761537730289 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% -2025-10-27T04:02:10.289Z [INFO] server-cf7fb860-b643-286d-d56d-627298cce1ab: restored from snapshot: id=2-6-1761537730289 last-index=6 last-term=2 size-in-bytes=25 -2025-10-27T04:02:10.289Z [INFO] server-cf7fb860-b643-286d-d56d-627298cce1ab: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c9b111a1-852c-fc12-5a64-ef95c3a82a39 Address:c9b111a1-852c-fc12-5a64-ef95c3a82a39} {Suffrage:Voter ID:server-86ae9085-65c2-80aa-8585-6562f06236ae Address:86ae9085-65c2-80aa-8585-6562f06236ae} {Suffrage:Voter ID:server-cf7fb860-b643-286d-d56d-627298cce1ab Address:cf7fb860-b643-286d-d56d-627298cce1ab}]" -2025-10-27T04:02:10.289Z [INFO] server-cf7fb860-b643-286d-d56d-627298cce1ab: entering follower state: follower="Node at cf7fb860-b643-286d-d56d-627298cce1ab [Follower]" leader-address= leader-id= -2025-10-27T04:02:10.341Z [WARN] server-86ae9085-65c2-80aa-8585-6562f06236ae: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:10.341Z [INFO] server-86ae9085-65c2-80aa-8585-6562f06236ae: entering candidate state: node="Node at 86ae9085-65c2-80aa-8585-6562f06236ae [Candidate]" term=3 -2025-10-27T04:02:10.341Z [INFO] server-86ae9085-65c2-80aa-8585-6562f06236ae: election won: term=3 tally=2 -2025-10-27T04:02:10.341Z [INFO] server-86ae9085-65c2-80aa-8585-6562f06236ae: entering leader state: leader="Node at 86ae9085-65c2-80aa-8585-6562f06236ae [Leader]" -2025-10-27T04:02:10.342Z [INFO] server-86ae9085-65c2-80aa-8585-6562f06236ae: added peer, starting replication: peer=server-c9b111a1-852c-fc12-5a64-ef95c3a82a39 -2025-10-27T04:02:10.342Z [INFO] server-86ae9085-65c2-80aa-8585-6562f06236ae: added peer, starting replication: peer=server-cf7fb860-b643-286d-d56d-627298cce1ab -2025-10-27T04:02:10.342Z [INFO] server-86ae9085-65c2-80aa-8585-6562f06236ae: pipelining replication: peer="{Voter server-cf7fb860-b643-286d-d56d-627298cce1ab cf7fb860-b643-286d-d56d-627298cce1ab}" -2025-10-27T04:02:10.342Z [INFO] server-86ae9085-65c2-80aa-8585-6562f06236ae: pipelining replication: peer="{Voter server-c9b111a1-852c-fc12-5a64-ef95c3a82a39 c9b111a1-852c-fc12-5a64-ef95c3a82a39}" -2025-10-27T04:02:10.605Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:10.710Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:10.605390002 +0000 UTC m=+5.600262480 (1 nodes), 45.84µs from start of poll, 489.131983ms from cluster start. Timeout at 2025-10-27 04:02:10.710394642 +0000 UTC m=+5.705267120, 105.00464ms after stability -2025-10-27T04:02:10.710Z [INFO] server-86ae9085-65c2-80aa-8585-6562f06236ae: aborting pipeline replication: peer="{Voter server-cf7fb860-b643-286d-d56d-627298cce1ab cf7fb860-b643-286d-d56d-627298cce1ab}" -2025-10-27T04:02:10.711Z [INFO] server-86ae9085-65c2-80aa-8585-6562f06236ae: aborting pipeline replication: peer="{Voter server-c9b111a1-852c-fc12-5a64-ef95c3a82a39 c9b111a1-852c-fc12-5a64-ef95c3a82a39}" +2026-11-29T10:28:14.361Z [INFO] server-cb5fb2a8-a5af-122f-2b65-c9319119b09c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cb5fb2a8-a5af-122f-2b65-c9319119b09c Address:cb5fb2a8-a5af-122f-2b65-c9319119b09c} {Suffrage:Voter ID:server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc Address:6b1acc43-ddf2-07ef-f3d0-be1175faddbc} {Suffrage:Voter ID:server-427b29e1-b0bd-5fb1-559d-47094326947f Address:427b29e1-b0bd-5fb1-559d-47094326947f}]" +2026-11-29T10:28:14.362Z [INFO] server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cb5fb2a8-a5af-122f-2b65-c9319119b09c Address:cb5fb2a8-a5af-122f-2b65-c9319119b09c} {Suffrage:Voter ID:server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc Address:6b1acc43-ddf2-07ef-f3d0-be1175faddbc} {Suffrage:Voter ID:server-427b29e1-b0bd-5fb1-559d-47094326947f Address:427b29e1-b0bd-5fb1-559d-47094326947f}]" +2026-11-29T10:28:14.362Z [INFO] server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc: entering follower state: follower="Node at 6b1acc43-ddf2-07ef-f3d0-be1175faddbc [Follower]" leader-address= leader-id= +2026-11-29T10:28:14.362Z [INFO] server-cb5fb2a8-a5af-122f-2b65-c9319119b09c: entering follower state: follower="Node at cb5fb2a8-a5af-122f-2b65-c9319119b09c [Follower]" leader-address= leader-id= +2026-11-29T10:28:14.362Z [INFO] server-427b29e1-b0bd-5fb1-559d-47094326947f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cb5fb2a8-a5af-122f-2b65-c9319119b09c Address:cb5fb2a8-a5af-122f-2b65-c9319119b09c} {Suffrage:Voter ID:server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc Address:6b1acc43-ddf2-07ef-f3d0-be1175faddbc} {Suffrage:Voter ID:server-427b29e1-b0bd-5fb1-559d-47094326947f Address:427b29e1-b0bd-5fb1-559d-47094326947f}]" +2026-11-29T10:28:14.362Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:14.362Z [INFO] server-427b29e1-b0bd-5fb1-559d-47094326947f: entering follower state: follower="Node at 427b29e1-b0bd-5fb1-559d-47094326947f [Follower]" leader-address= leader-id= +2026-11-29T10:28:14.421Z [WARN] server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:14.421Z [INFO] server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc: entering candidate state: node="Node at 6b1acc43-ddf2-07ef-f3d0-be1175faddbc [Candidate]" term=2 +2026-11-29T10:28:14.421Z [INFO] server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc: election won: term=2 tally=2 +2026-11-29T10:28:14.421Z [INFO] server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc: entering leader state: leader="Node at 6b1acc43-ddf2-07ef-f3d0-be1175faddbc [Leader]" +2026-11-29T10:28:14.421Z [INFO] server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc: added peer, starting replication: peer=server-cb5fb2a8-a5af-122f-2b65-c9319119b09c +2026-11-29T10:28:14.421Z [INFO] server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc: added peer, starting replication: peer=server-427b29e1-b0bd-5fb1-559d-47094326947f +2026-11-29T10:28:14.421Z [INFO] server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc: pipelining replication: peer="{Voter server-427b29e1-b0bd-5fb1-559d-47094326947f 427b29e1-b0bd-5fb1-559d-47094326947f}" +2026-11-29T10:28:14.421Z [INFO] server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc: pipelining replication: peer="{Voter server-cb5fb2a8-a5af-122f-2b65-c9319119b09c cb5fb2a8-a5af-122f-2b65-c9319119b09c}" +2026-11-29T10:28:14.526Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:14.421340219 +0000 UTC m=+5.156394401 (1 nodes), 58.9896ms from start of poll, 59.5054ms from cluster start. Timeout at 2026-11-29 10:28:14.526345059 +0000 UTC m=+5.261399161, 105.00476ms after stability +2026-11-29T10:28:14.527Z [INFO] server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc: aborting pipeline replication: peer="{Voter server-cb5fb2a8-a5af-122f-2b65-c9319119b09c cb5fb2a8-a5af-122f-2b65-c9319119b09c}" +2026-11-29T10:28:14.527Z [INFO] server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc: aborting pipeline replication: peer="{Voter server-427b29e1-b0bd-5fb1-559d-47094326947f 427b29e1-b0bd-5fb1-559d-47094326947f}" +2026-11-29T10:28:14.527Z [INFO] creating new snapshot: path=/tmp/raft1504024623/snapshots/2-6-1795948094527.tmp +2026-11-29T10:28:14.528Z [INFO] server-cb5fb2a8-a5af-122f-2b65-c9319119b09c: starting restore from snapshot: id=2-6-1795948094527 last-index=6 last-term=2 size-in-bytes=25 +2026-11-29T10:28:14.528Z [INFO] server-cb5fb2a8-a5af-122f-2b65-c9319119b09c: snapshot restore progress: id=2-6-1795948094527 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% +2026-11-29T10:28:14.528Z [INFO] server-cb5fb2a8-a5af-122f-2b65-c9319119b09c: restored from snapshot: id=2-6-1795948094527 last-index=6 last-term=2 size-in-bytes=25 +2026-11-29T10:28:14.528Z [INFO] server-cb5fb2a8-a5af-122f-2b65-c9319119b09c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cb5fb2a8-a5af-122f-2b65-c9319119b09c Address:cb5fb2a8-a5af-122f-2b65-c9319119b09c} {Suffrage:Voter ID:server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc Address:6b1acc43-ddf2-07ef-f3d0-be1175faddbc} {Suffrage:Voter ID:server-427b29e1-b0bd-5fb1-559d-47094326947f Address:427b29e1-b0bd-5fb1-559d-47094326947f}]" +2026-11-29T10:28:14.528Z [INFO] creating new snapshot: path=/tmp/raft4026454772/snapshots/2-6-1795948094528.tmp +2026-11-29T10:28:14.528Z [INFO] server-cb5fb2a8-a5af-122f-2b65-c9319119b09c: entering follower state: follower="Node at cb5fb2a8-a5af-122f-2b65-c9319119b09c [Follower]" leader-address= leader-id= +2026-11-29T10:28:14.528Z [INFO] server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc: starting restore from snapshot: id=2-6-1795948094528 last-index=6 last-term=2 size-in-bytes=25 +2026-11-29T10:28:14.529Z [INFO] server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc: snapshot restore progress: id=2-6-1795948094528 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% +2026-11-29T10:28:14.529Z [INFO] server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc: restored from snapshot: id=2-6-1795948094528 last-index=6 last-term=2 size-in-bytes=25 +2026-11-29T10:28:14.529Z [INFO] server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cb5fb2a8-a5af-122f-2b65-c9319119b09c Address:cb5fb2a8-a5af-122f-2b65-c9319119b09c} {Suffrage:Voter ID:server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc Address:6b1acc43-ddf2-07ef-f3d0-be1175faddbc} {Suffrage:Voter ID:server-427b29e1-b0bd-5fb1-559d-47094326947f Address:427b29e1-b0bd-5fb1-559d-47094326947f}]" +2026-11-29T10:28:14.529Z [INFO] server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc: entering follower state: follower="Node at 6b1acc43-ddf2-07ef-f3d0-be1175faddbc [Follower]" leader-address= leader-id= +2026-11-29T10:28:14.529Z [INFO] creating new snapshot: path=/tmp/raft1275814079/snapshots/2-6-1795948094529.tmp +2026-11-29T10:28:14.529Z [INFO] server-427b29e1-b0bd-5fb1-559d-47094326947f: starting restore from snapshot: id=2-6-1795948094529 last-index=6 last-term=2 size-in-bytes=25 +2026-11-29T10:28:14.529Z [INFO] server-427b29e1-b0bd-5fb1-559d-47094326947f: snapshot restore progress: id=2-6-1795948094529 last-index=6 last-term=2 size-in-bytes=25 read-bytes=25 percent-complete=100.00% +2026-11-29T10:28:14.529Z [INFO] server-427b29e1-b0bd-5fb1-559d-47094326947f: restored from snapshot: id=2-6-1795948094529 last-index=6 last-term=2 size-in-bytes=25 +2026-11-29T10:28:14.529Z [INFO] server-427b29e1-b0bd-5fb1-559d-47094326947f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cb5fb2a8-a5af-122f-2b65-c9319119b09c Address:cb5fb2a8-a5af-122f-2b65-c9319119b09c} {Suffrage:Voter ID:server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc Address:6b1acc43-ddf2-07ef-f3d0-be1175faddbc} {Suffrage:Voter ID:server-427b29e1-b0bd-5fb1-559d-47094326947f Address:427b29e1-b0bd-5fb1-559d-47094326947f}]" +2026-11-29T10:28:14.529Z [INFO] server-427b29e1-b0bd-5fb1-559d-47094326947f: entering follower state: follower="Node at 427b29e1-b0bd-5fb1-559d-47094326947f [Follower]" leader-address= leader-id= +2026-11-29T10:28:14.596Z [WARN] server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:14.596Z [INFO] server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc: entering candidate state: node="Node at 6b1acc43-ddf2-07ef-f3d0-be1175faddbc [Candidate]" term=3 +2026-11-29T10:28:14.596Z [INFO] server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc: election won: term=3 tally=2 +2026-11-29T10:28:14.596Z [INFO] server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc: entering leader state: leader="Node at 6b1acc43-ddf2-07ef-f3d0-be1175faddbc [Leader]" +2026-11-29T10:28:14.596Z [INFO] server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc: added peer, starting replication: peer=server-cb5fb2a8-a5af-122f-2b65-c9319119b09c +2026-11-29T10:28:14.596Z [INFO] server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc: added peer, starting replication: peer=server-427b29e1-b0bd-5fb1-559d-47094326947f +2026-11-29T10:28:14.597Z [INFO] server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc: pipelining replication: peer="{Voter server-427b29e1-b0bd-5fb1-559d-47094326947f 427b29e1-b0bd-5fb1-559d-47094326947f}" +2026-11-29T10:28:14.597Z [INFO] server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc: pipelining replication: peer="{Voter server-cb5fb2a8-a5af-122f-2b65-c9319119b09c cb5fb2a8-a5af-122f-2b65-c9319119b09c}" +2026-11-29T10:28:14.845Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:14.951Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:14.845474819 +0000 UTC m=+5.580529001 (1 nodes), 43.32µs from start of poll, 483.64ms from cluster start. Timeout at 2026-11-29 10:28:14.950481099 +0000 UTC m=+5.685535241, 105.00624ms after stability +2026-11-29T10:28:14.951Z [INFO] server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc: aborting pipeline replication: peer="{Voter server-cb5fb2a8-a5af-122f-2b65-c9319119b09c cb5fb2a8-a5af-122f-2b65-c9319119b09c}" +2026-11-29T10:28:14.951Z [INFO] server-6b1acc43-ddf2-07ef-f3d0-be1175faddbc: aborting pipeline replication: peer="{Voter server-427b29e1-b0bd-5fb1-559d-47094326947f 427b29e1-b0bd-5fb1-559d-47094326947f}" === RUN TestRaft_RecoverCluster/snapshot,_with_trailing_logs -2025-10-27T04:02:10.711Z [INFO] server-fde19d20-a2d5-9ea6-fe4c-0ff719f8b9b3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fde19d20-a2d5-9ea6-fe4c-0ff719f8b9b3 Address:fde19d20-a2d5-9ea6-fe4c-0ff719f8b9b3} {Suffrage:Voter ID:server-b051131a-951b-0c1a-1b29-f403ae234d25 Address:b051131a-951b-0c1a-1b29-f403ae234d25} {Suffrage:Voter ID:server-9097e2f0-4979-8f92-7e95-73901f9bd43b Address:9097e2f0-4979-8f92-7e95-73901f9bd43b}]" -2025-10-27T04:02:10.711Z [INFO] server-b051131a-951b-0c1a-1b29-f403ae234d25: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fde19d20-a2d5-9ea6-fe4c-0ff719f8b9b3 Address:fde19d20-a2d5-9ea6-fe4c-0ff719f8b9b3} {Suffrage:Voter ID:server-b051131a-951b-0c1a-1b29-f403ae234d25 Address:b051131a-951b-0c1a-1b29-f403ae234d25} {Suffrage:Voter ID:server-9097e2f0-4979-8f92-7e95-73901f9bd43b Address:9097e2f0-4979-8f92-7e95-73901f9bd43b}]" -2025-10-27T04:02:10.711Z [INFO] server-9097e2f0-4979-8f92-7e95-73901f9bd43b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fde19d20-a2d5-9ea6-fe4c-0ff719f8b9b3 Address:fde19d20-a2d5-9ea6-fe4c-0ff719f8b9b3} {Suffrage:Voter ID:server-b051131a-951b-0c1a-1b29-f403ae234d25 Address:b051131a-951b-0c1a-1b29-f403ae234d25} {Suffrage:Voter ID:server-9097e2f0-4979-8f92-7e95-73901f9bd43b Address:9097e2f0-4979-8f92-7e95-73901f9bd43b}]" -2025-10-27T04:02:10.711Z [INFO] server-fde19d20-a2d5-9ea6-fe4c-0ff719f8b9b3: entering follower state: follower="Node at fde19d20-a2d5-9ea6-fe4c-0ff719f8b9b3 [Follower]" leader-address= leader-id= -2025-10-27T04:02:10.711Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:10.711Z [INFO] server-b051131a-951b-0c1a-1b29-f403ae234d25: entering follower state: follower="Node at b051131a-951b-0c1a-1b29-f403ae234d25 [Follower]" leader-address= leader-id= -2025-10-27T04:02:10.711Z [INFO] server-9097e2f0-4979-8f92-7e95-73901f9bd43b: entering follower state: follower="Node at 9097e2f0-4979-8f92-7e95-73901f9bd43b [Follower]" leader-address= leader-id= -2025-10-27T04:02:10.790Z [WARN] server-9097e2f0-4979-8f92-7e95-73901f9bd43b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:10.790Z [INFO] server-9097e2f0-4979-8f92-7e95-73901f9bd43b: entering candidate state: node="Node at 9097e2f0-4979-8f92-7e95-73901f9bd43b [Candidate]" term=2 -2025-10-27T04:02:10.790Z [INFO] server-9097e2f0-4979-8f92-7e95-73901f9bd43b: election won: term=2 tally=2 -2025-10-27T04:02:10.790Z [INFO] server-9097e2f0-4979-8f92-7e95-73901f9bd43b: entering leader state: leader="Node at 9097e2f0-4979-8f92-7e95-73901f9bd43b [Leader]" -2025-10-27T04:02:10.790Z [INFO] server-9097e2f0-4979-8f92-7e95-73901f9bd43b: added peer, starting replication: peer=server-fde19d20-a2d5-9ea6-fe4c-0ff719f8b9b3 -2025-10-27T04:02:10.790Z [INFO] server-9097e2f0-4979-8f92-7e95-73901f9bd43b: added peer, starting replication: peer=server-b051131a-951b-0c1a-1b29-f403ae234d25 -2025-10-27T04:02:10.790Z [INFO] server-9097e2f0-4979-8f92-7e95-73901f9bd43b: pipelining replication: peer="{Voter server-b051131a-951b-0c1a-1b29-f403ae234d25 b051131a-951b-0c1a-1b29-f403ae234d25}" -2025-10-27T04:02:10.790Z [INFO] server-9097e2f0-4979-8f92-7e95-73901f9bd43b: pipelining replication: peer="{Voter server-fde19d20-a2d5-9ea6-fe4c-0ff719f8b9b3 fde19d20-a2d5-9ea6-fe4c-0ff719f8b9b3}" -2025-10-27T04:02:10.895Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:10.790366626 +0000 UTC m=+5.785239144 (1 nodes), 78.619697ms from start of poll, 78.858899ms from cluster start. Timeout at 2025-10-27 04:02:10.895369306 +0000 UTC m=+5.890241784, 105.00264ms after stability -2025-10-27T04:02:10.898Z [INFO] server-9097e2f0-4979-8f92-7e95-73901f9bd43b: aborting pipeline replication: peer="{Voter server-fde19d20-a2d5-9ea6-fe4c-0ff719f8b9b3 fde19d20-a2d5-9ea6-fe4c-0ff719f8b9b3}" -2025-10-27T04:02:10.898Z [INFO] server-9097e2f0-4979-8f92-7e95-73901f9bd43b: aborting pipeline replication: peer="{Voter server-b051131a-951b-0c1a-1b29-f403ae234d25 b051131a-951b-0c1a-1b29-f403ae234d25}" -2025-10-27T04:02:10.898Z [INFO] creating new snapshot: path=/tmp/raft4036136737/snapshots/2-27-1761537730898.tmp -2025-10-27T04:02:10.899Z [INFO] server-fde19d20-a2d5-9ea6-fe4c-0ff719f8b9b3: starting restore from snapshot: id=2-27-1761537730898 last-index=27 last-term=2 size-in-bytes=168 -2025-10-27T04:02:10.899Z [INFO] server-fde19d20-a2d5-9ea6-fe4c-0ff719f8b9b3: snapshot restore progress: id=2-27-1761537730898 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% -2025-10-27T04:02:10.899Z [INFO] server-fde19d20-a2d5-9ea6-fe4c-0ff719f8b9b3: restored from snapshot: id=2-27-1761537730898 last-index=27 last-term=2 size-in-bytes=168 -2025-10-27T04:02:10.899Z [INFO] server-fde19d20-a2d5-9ea6-fe4c-0ff719f8b9b3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fde19d20-a2d5-9ea6-fe4c-0ff719f8b9b3 Address:fde19d20-a2d5-9ea6-fe4c-0ff719f8b9b3} {Suffrage:Voter ID:server-b051131a-951b-0c1a-1b29-f403ae234d25 Address:b051131a-951b-0c1a-1b29-f403ae234d25} {Suffrage:Voter ID:server-9097e2f0-4979-8f92-7e95-73901f9bd43b Address:9097e2f0-4979-8f92-7e95-73901f9bd43b}]" -2025-10-27T04:02:10.899Z [INFO] creating new snapshot: path=/tmp/raft1813480034/snapshots/2-27-1761537730899.tmp -2025-10-27T04:02:10.899Z [INFO] server-fde19d20-a2d5-9ea6-fe4c-0ff719f8b9b3: entering follower state: follower="Node at fde19d20-a2d5-9ea6-fe4c-0ff719f8b9b3 [Follower]" leader-address= leader-id= -2025-10-27T04:02:10.900Z [INFO] server-b051131a-951b-0c1a-1b29-f403ae234d25: starting restore from snapshot: id=2-27-1761537730899 last-index=27 last-term=2 size-in-bytes=168 -2025-10-27T04:02:10.900Z [INFO] server-b051131a-951b-0c1a-1b29-f403ae234d25: snapshot restore progress: id=2-27-1761537730899 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% -2025-10-27T04:02:10.900Z [INFO] server-b051131a-951b-0c1a-1b29-f403ae234d25: restored from snapshot: id=2-27-1761537730899 last-index=27 last-term=2 size-in-bytes=168 -2025-10-27T04:02:10.900Z [INFO] server-b051131a-951b-0c1a-1b29-f403ae234d25: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fde19d20-a2d5-9ea6-fe4c-0ff719f8b9b3 Address:fde19d20-a2d5-9ea6-fe4c-0ff719f8b9b3} {Suffrage:Voter ID:server-b051131a-951b-0c1a-1b29-f403ae234d25 Address:b051131a-951b-0c1a-1b29-f403ae234d25} {Suffrage:Voter ID:server-9097e2f0-4979-8f92-7e95-73901f9bd43b Address:9097e2f0-4979-8f92-7e95-73901f9bd43b}]" -2025-10-27T04:02:10.900Z [INFO] creating new snapshot: path=/tmp/raft3446456533/snapshots/2-27-1761537730900.tmp -2025-10-27T04:02:10.900Z [INFO] server-b051131a-951b-0c1a-1b29-f403ae234d25: entering follower state: follower="Node at b051131a-951b-0c1a-1b29-f403ae234d25 [Follower]" leader-address= leader-id= -2025-10-27T04:02:10.901Z [INFO] server-9097e2f0-4979-8f92-7e95-73901f9bd43b: starting restore from snapshot: id=2-27-1761537730900 last-index=27 last-term=2 size-in-bytes=168 -2025-10-27T04:02:10.901Z [INFO] server-9097e2f0-4979-8f92-7e95-73901f9bd43b: snapshot restore progress: id=2-27-1761537730900 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% -2025-10-27T04:02:10.901Z [INFO] server-9097e2f0-4979-8f92-7e95-73901f9bd43b: restored from snapshot: id=2-27-1761537730900 last-index=27 last-term=2 size-in-bytes=168 -2025-10-27T04:02:10.901Z [INFO] server-9097e2f0-4979-8f92-7e95-73901f9bd43b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fde19d20-a2d5-9ea6-fe4c-0ff719f8b9b3 Address:fde19d20-a2d5-9ea6-fe4c-0ff719f8b9b3} {Suffrage:Voter ID:server-b051131a-951b-0c1a-1b29-f403ae234d25 Address:b051131a-951b-0c1a-1b29-f403ae234d25} {Suffrage:Voter ID:server-9097e2f0-4979-8f92-7e95-73901f9bd43b Address:9097e2f0-4979-8f92-7e95-73901f9bd43b}]" -2025-10-27T04:02:10.901Z [INFO] server-9097e2f0-4979-8f92-7e95-73901f9bd43b: entering follower state: follower="Node at 9097e2f0-4979-8f92-7e95-73901f9bd43b [Follower]" leader-address= leader-id= -2025-10-27T04:02:10.962Z [WARN] server-b051131a-951b-0c1a-1b29-f403ae234d25: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:10.962Z [INFO] server-b051131a-951b-0c1a-1b29-f403ae234d25: entering candidate state: node="Node at b051131a-951b-0c1a-1b29-f403ae234d25 [Candidate]" term=3 -2025-10-27T04:02:10.962Z [INFO] server-b051131a-951b-0c1a-1b29-f403ae234d25: election won: term=3 tally=2 -2025-10-27T04:02:10.962Z [INFO] server-b051131a-951b-0c1a-1b29-f403ae234d25: entering leader state: leader="Node at b051131a-951b-0c1a-1b29-f403ae234d25 [Leader]" -2025-10-27T04:02:10.962Z [INFO] server-b051131a-951b-0c1a-1b29-f403ae234d25: added peer, starting replication: peer=server-fde19d20-a2d5-9ea6-fe4c-0ff719f8b9b3 -2025-10-27T04:02:10.962Z [INFO] server-b051131a-951b-0c1a-1b29-f403ae234d25: added peer, starting replication: peer=server-9097e2f0-4979-8f92-7e95-73901f9bd43b -2025-10-27T04:02:10.962Z [INFO] server-b051131a-951b-0c1a-1b29-f403ae234d25: pipelining replication: peer="{Voter server-9097e2f0-4979-8f92-7e95-73901f9bd43b 9097e2f0-4979-8f92-7e95-73901f9bd43b}" -2025-10-27T04:02:10.962Z [INFO] server-b051131a-951b-0c1a-1b29-f403ae234d25: pipelining replication: peer="{Voter server-fde19d20-a2d5-9ea6-fe4c-0ff719f8b9b3 fde19d20-a2d5-9ea6-fe4c-0ff719f8b9b3}" -2025-10-27T04:02:11.216Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:11.322Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:11.216758501 +0000 UTC m=+6.211631019 (1 nodes), 63.841µs from start of poll, 505.250774ms from cluster start. Timeout at 2025-10-27 04:02:11.321766021 +0000 UTC m=+6.316638499, 105.00748ms after stability -2025-10-27T04:02:11.322Z [INFO] server-b051131a-951b-0c1a-1b29-f403ae234d25: aborting pipeline replication: peer="{Voter server-9097e2f0-4979-8f92-7e95-73901f9bd43b 9097e2f0-4979-8f92-7e95-73901f9bd43b}" -2025-10-27T04:02:11.322Z [INFO] server-b051131a-951b-0c1a-1b29-f403ae234d25: aborting pipeline replication: peer="{Voter server-fde19d20-a2d5-9ea6-fe4c-0ff719f8b9b3 fde19d20-a2d5-9ea6-fe4c-0ff719f8b9b3}" +2026-11-29T10:28:14.952Z [INFO] server-ec6cfc21-77af-035c-836e-d2654e5cb85c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ec6cfc21-77af-035c-836e-d2654e5cb85c Address:ec6cfc21-77af-035c-836e-d2654e5cb85c} {Suffrage:Voter ID:server-58ada528-690c-0c06-436c-849b52868f70 Address:58ada528-690c-0c06-436c-849b52868f70} {Suffrage:Voter ID:server-b1d62b7b-bb22-d74f-ef0c-70f85b5f265b Address:b1d62b7b-bb22-d74f-ef0c-70f85b5f265b}]" +2026-11-29T10:28:14.952Z [INFO] server-58ada528-690c-0c06-436c-849b52868f70: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ec6cfc21-77af-035c-836e-d2654e5cb85c Address:ec6cfc21-77af-035c-836e-d2654e5cb85c} {Suffrage:Voter ID:server-58ada528-690c-0c06-436c-849b52868f70 Address:58ada528-690c-0c06-436c-849b52868f70} {Suffrage:Voter ID:server-b1d62b7b-bb22-d74f-ef0c-70f85b5f265b Address:b1d62b7b-bb22-d74f-ef0c-70f85b5f265b}]" +2026-11-29T10:28:14.952Z [INFO] server-ec6cfc21-77af-035c-836e-d2654e5cb85c: entering follower state: follower="Node at ec6cfc21-77af-035c-836e-d2654e5cb85c [Follower]" leader-address= leader-id= +2026-11-29T10:28:14.952Z [INFO] server-58ada528-690c-0c06-436c-849b52868f70: entering follower state: follower="Node at 58ada528-690c-0c06-436c-849b52868f70 [Follower]" leader-address= leader-id= +2026-11-29T10:28:14.952Z [INFO] server-b1d62b7b-bb22-d74f-ef0c-70f85b5f265b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ec6cfc21-77af-035c-836e-d2654e5cb85c Address:ec6cfc21-77af-035c-836e-d2654e5cb85c} {Suffrage:Voter ID:server-58ada528-690c-0c06-436c-849b52868f70 Address:58ada528-690c-0c06-436c-849b52868f70} {Suffrage:Voter ID:server-b1d62b7b-bb22-d74f-ef0c-70f85b5f265b Address:b1d62b7b-bb22-d74f-ef0c-70f85b5f265b}]" +2026-11-29T10:28:14.952Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:14.952Z [INFO] server-b1d62b7b-bb22-d74f-ef0c-70f85b5f265b: entering follower state: follower="Node at b1d62b7b-bb22-d74f-ef0c-70f85b5f265b [Follower]" leader-address= leader-id= +2026-11-29T10:28:15.033Z [WARN] server-b1d62b7b-bb22-d74f-ef0c-70f85b5f265b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:15.033Z [INFO] server-b1d62b7b-bb22-d74f-ef0c-70f85b5f265b: entering candidate state: node="Node at b1d62b7b-bb22-d74f-ef0c-70f85b5f265b [Candidate]" term=2 +2026-11-29T10:28:15.033Z [INFO] server-b1d62b7b-bb22-d74f-ef0c-70f85b5f265b: election won: term=2 tally=2 +2026-11-29T10:28:15.033Z [INFO] server-b1d62b7b-bb22-d74f-ef0c-70f85b5f265b: entering leader state: leader="Node at b1d62b7b-bb22-d74f-ef0c-70f85b5f265b [Leader]" +2026-11-29T10:28:15.034Z [INFO] server-b1d62b7b-bb22-d74f-ef0c-70f85b5f265b: added peer, starting replication: peer=server-ec6cfc21-77af-035c-836e-d2654e5cb85c +2026-11-29T10:28:15.034Z [INFO] server-b1d62b7b-bb22-d74f-ef0c-70f85b5f265b: added peer, starting replication: peer=server-58ada528-690c-0c06-436c-849b52868f70 +2026-11-29T10:28:15.034Z [INFO] server-b1d62b7b-bb22-d74f-ef0c-70f85b5f265b: pipelining replication: peer="{Voter server-ec6cfc21-77af-035c-836e-d2654e5cb85c ec6cfc21-77af-035c-836e-d2654e5cb85c}" +2026-11-29T10:28:15.034Z [INFO] server-b1d62b7b-bb22-d74f-ef0c-70f85b5f265b: pipelining replication: peer="{Voter server-58ada528-690c-0c06-436c-849b52868f70 58ada528-690c-0c06-436c-849b52868f70}" +2026-11-29T10:28:15.139Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:15.034002539 +0000 UTC m=+5.769056681 (1 nodes), 81.26896ms from start of poll, 81.66884ms from cluster start. Timeout at 2026-11-29 10:28:15.139004659 +0000 UTC m=+5.874058761, 105.00208ms after stability +2026-11-29T10:28:15.142Z [INFO] server-b1d62b7b-bb22-d74f-ef0c-70f85b5f265b: aborting pipeline replication: peer="{Voter server-58ada528-690c-0c06-436c-849b52868f70 58ada528-690c-0c06-436c-849b52868f70}" +2026-11-29T10:28:15.142Z [INFO] server-b1d62b7b-bb22-d74f-ef0c-70f85b5f265b: aborting pipeline replication: peer="{Voter server-ec6cfc21-77af-035c-836e-d2654e5cb85c ec6cfc21-77af-035c-836e-d2654e5cb85c}" +2026-11-29T10:28:15.142Z [INFO] creating new snapshot: path=/tmp/raft1400097339/snapshots/2-27-1795948095142.tmp +2026-11-29T10:28:15.143Z [INFO] server-ec6cfc21-77af-035c-836e-d2654e5cb85c: starting restore from snapshot: id=2-27-1795948095142 last-index=27 last-term=2 size-in-bytes=168 +2026-11-29T10:28:15.144Z [INFO] server-ec6cfc21-77af-035c-836e-d2654e5cb85c: snapshot restore progress: id=2-27-1795948095142 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% +2026-11-29T10:28:15.144Z [INFO] server-ec6cfc21-77af-035c-836e-d2654e5cb85c: restored from snapshot: id=2-27-1795948095142 last-index=27 last-term=2 size-in-bytes=168 +2026-11-29T10:28:15.144Z [INFO] server-ec6cfc21-77af-035c-836e-d2654e5cb85c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ec6cfc21-77af-035c-836e-d2654e5cb85c Address:ec6cfc21-77af-035c-836e-d2654e5cb85c} {Suffrage:Voter ID:server-58ada528-690c-0c06-436c-849b52868f70 Address:58ada528-690c-0c06-436c-849b52868f70} {Suffrage:Voter ID:server-b1d62b7b-bb22-d74f-ef0c-70f85b5f265b Address:b1d62b7b-bb22-d74f-ef0c-70f85b5f265b}]" +2026-11-29T10:28:15.144Z [INFO] creating new snapshot: path=/tmp/raft582200568/snapshots/2-27-1795948095144.tmp +2026-11-29T10:28:15.144Z [INFO] server-ec6cfc21-77af-035c-836e-d2654e5cb85c: entering follower state: follower="Node at ec6cfc21-77af-035c-836e-d2654e5cb85c [Follower]" leader-address= leader-id= +2026-11-29T10:28:15.144Z [INFO] server-58ada528-690c-0c06-436c-849b52868f70: starting restore from snapshot: id=2-27-1795948095144 last-index=27 last-term=2 size-in-bytes=168 +2026-11-29T10:28:15.145Z [INFO] server-58ada528-690c-0c06-436c-849b52868f70: snapshot restore progress: id=2-27-1795948095144 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% +2026-11-29T10:28:15.145Z [INFO] server-58ada528-690c-0c06-436c-849b52868f70: restored from snapshot: id=2-27-1795948095144 last-index=27 last-term=2 size-in-bytes=168 +2026-11-29T10:28:15.145Z [INFO] server-58ada528-690c-0c06-436c-849b52868f70: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ec6cfc21-77af-035c-836e-d2654e5cb85c Address:ec6cfc21-77af-035c-836e-d2654e5cb85c} {Suffrage:Voter ID:server-58ada528-690c-0c06-436c-849b52868f70 Address:58ada528-690c-0c06-436c-849b52868f70} {Suffrage:Voter ID:server-b1d62b7b-bb22-d74f-ef0c-70f85b5f265b Address:b1d62b7b-bb22-d74f-ef0c-70f85b5f265b}]" +2026-11-29T10:28:15.145Z [INFO] server-58ada528-690c-0c06-436c-849b52868f70: entering follower state: follower="Node at 58ada528-690c-0c06-436c-849b52868f70 [Follower]" leader-address= leader-id= +2026-11-29T10:28:15.145Z [INFO] creating new snapshot: path=/tmp/raft2621123967/snapshots/2-27-1795948095145.tmp +2026-11-29T10:28:15.145Z [INFO] server-b1d62b7b-bb22-d74f-ef0c-70f85b5f265b: starting restore from snapshot: id=2-27-1795948095145 last-index=27 last-term=2 size-in-bytes=168 +2026-11-29T10:28:15.145Z [INFO] server-b1d62b7b-bb22-d74f-ef0c-70f85b5f265b: snapshot restore progress: id=2-27-1795948095145 last-index=27 last-term=2 size-in-bytes=168 read-bytes=168 percent-complete=100.00% +2026-11-29T10:28:15.145Z [INFO] server-b1d62b7b-bb22-d74f-ef0c-70f85b5f265b: restored from snapshot: id=2-27-1795948095145 last-index=27 last-term=2 size-in-bytes=168 +2026-11-29T10:28:15.145Z [INFO] server-b1d62b7b-bb22-d74f-ef0c-70f85b5f265b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ec6cfc21-77af-035c-836e-d2654e5cb85c Address:ec6cfc21-77af-035c-836e-d2654e5cb85c} {Suffrage:Voter ID:server-58ada528-690c-0c06-436c-849b52868f70 Address:58ada528-690c-0c06-436c-849b52868f70} {Suffrage:Voter ID:server-b1d62b7b-bb22-d74f-ef0c-70f85b5f265b Address:b1d62b7b-bb22-d74f-ef0c-70f85b5f265b}]" +2026-11-29T10:28:15.145Z [INFO] server-b1d62b7b-bb22-d74f-ef0c-70f85b5f265b: entering follower state: follower="Node at b1d62b7b-bb22-d74f-ef0c-70f85b5f265b [Follower]" leader-address= leader-id= +2026-11-29T10:28:15.197Z [WARN] server-ec6cfc21-77af-035c-836e-d2654e5cb85c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:15.197Z [INFO] server-ec6cfc21-77af-035c-836e-d2654e5cb85c: entering candidate state: node="Node at ec6cfc21-77af-035c-836e-d2654e5cb85c [Candidate]" term=3 +2026-11-29T10:28:15.197Z [INFO] server-ec6cfc21-77af-035c-836e-d2654e5cb85c: election won: term=3 tally=2 +2026-11-29T10:28:15.197Z [INFO] server-ec6cfc21-77af-035c-836e-d2654e5cb85c: entering leader state: leader="Node at ec6cfc21-77af-035c-836e-d2654e5cb85c [Leader]" +2026-11-29T10:28:15.197Z [INFO] server-ec6cfc21-77af-035c-836e-d2654e5cb85c: added peer, starting replication: peer=server-58ada528-690c-0c06-436c-849b52868f70 +2026-11-29T10:28:15.197Z [INFO] server-ec6cfc21-77af-035c-836e-d2654e5cb85c: added peer, starting replication: peer=server-b1d62b7b-bb22-d74f-ef0c-70f85b5f265b +2026-11-29T10:28:15.197Z [INFO] server-ec6cfc21-77af-035c-836e-d2654e5cb85c: pipelining replication: peer="{Voter server-b1d62b7b-bb22-d74f-ef0c-70f85b5f265b b1d62b7b-bb22-d74f-ef0c-70f85b5f265b}" +2026-11-29T10:28:15.197Z [INFO] server-ec6cfc21-77af-035c-836e-d2654e5cb85c: pipelining replication: peer="{Voter server-58ada528-690c-0c06-436c-849b52868f70 58ada528-690c-0c06-436c-849b52868f70}" +2026-11-29T10:28:15.461Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:15.567Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:15.461606979 +0000 UTC m=+6.196661121 (1 nodes), 44.56µs from start of poll, 509.27328ms from cluster start. Timeout at 2026-11-29 10:28:15.566616339 +0000 UTC m=+6.301670481, 105.00936ms after stability +2026-11-29T10:28:15.567Z [INFO] server-ec6cfc21-77af-035c-836e-d2654e5cb85c: aborting pipeline replication: peer="{Voter server-b1d62b7b-bb22-d74f-ef0c-70f85b5f265b b1d62b7b-bb22-d74f-ef0c-70f85b5f265b}" +2026-11-29T10:28:15.567Z [INFO] server-ec6cfc21-77af-035c-836e-d2654e5cb85c: aborting pipeline replication: peer="{Voter server-58ada528-690c-0c06-436c-849b52868f70 58ada528-690c-0c06-436c-849b52868f70}" --- PASS: TestRaft_RecoverCluster (1.79s) --- PASS: TestRaft_RecoverCluster/no_snapshot,_no_trailing_logs (0.58s) - --- PASS: TestRaft_RecoverCluster/no_snapshot,_some_trailing_logs (0.60s) - --- PASS: TestRaft_RecoverCluster/snapshot,_with_trailing_logs (0.61s) + --- PASS: TestRaft_RecoverCluster/no_snapshot,_some_trailing_logs (0.59s) + --- PASS: TestRaft_RecoverCluster/snapshot,_with_trailing_logs (0.62s) === RUN TestRaft_HasExistingState raft_test.go:258: Skipping unreliable test --- SKIP: TestRaft_HasExistingState (0.00s) === RUN TestRaft_SingleNode -2025-10-27T04:02:11.323Z [INFO] server-d64463ce-b841-eef9-eec8-cdf94fd7872f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d64463ce-b841-eef9-eec8-cdf94fd7872f Address:d64463ce-b841-eef9-eec8-cdf94fd7872f}]" -2025-10-27T04:02:11.323Z [INFO] server-d64463ce-b841-eef9-eec8-cdf94fd7872f: entering follower state: follower="Node at d64463ce-b841-eef9-eec8-cdf94fd7872f [Follower]" leader-address= leader-id= -2025-10-27T04:02:11.419Z [WARN] server-d64463ce-b841-eef9-eec8-cdf94fd7872f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:11.419Z [INFO] server-d64463ce-b841-eef9-eec8-cdf94fd7872f: entering candidate state: node="Node at d64463ce-b841-eef9-eec8-cdf94fd7872f [Candidate]" term=2 -2025-10-27T04:02:11.419Z [INFO] server-d64463ce-b841-eef9-eec8-cdf94fd7872f: election won: term=2 tally=1 -2025-10-27T04:02:11.419Z [INFO] server-d64463ce-b841-eef9-eec8-cdf94fd7872f: entering leader state: leader="Node at d64463ce-b841-eef9-eec8-cdf94fd7872f [Leader]" ---- PASS: TestRaft_SingleNode (0.10s) +2026-11-29T10:28:15.568Z [INFO] server-879a0c35-ad5f-f884-b3dd-2f02638481b0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-879a0c35-ad5f-f884-b3dd-2f02638481b0 Address:879a0c35-ad5f-f884-b3dd-2f02638481b0}]" +2026-11-29T10:28:15.568Z [INFO] server-879a0c35-ad5f-f884-b3dd-2f02638481b0: entering follower state: follower="Node at 879a0c35-ad5f-f884-b3dd-2f02638481b0 [Follower]" leader-address= leader-id= +2026-11-29T10:28:15.620Z [WARN] server-879a0c35-ad5f-f884-b3dd-2f02638481b0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:15.620Z [INFO] server-879a0c35-ad5f-f884-b3dd-2f02638481b0: entering candidate state: node="Node at 879a0c35-ad5f-f884-b3dd-2f02638481b0 [Candidate]" term=2 +2026-11-29T10:28:15.620Z [INFO] server-879a0c35-ad5f-f884-b3dd-2f02638481b0: election won: term=2 tally=1 +2026-11-29T10:28:15.620Z [INFO] server-879a0c35-ad5f-f884-b3dd-2f02638481b0: entering leader state: leader="Node at 879a0c35-ad5f-f884-b3dd-2f02638481b0 [Leader]" +--- PASS: TestRaft_SingleNode (0.05s) === RUN TestRaft_TripleNode -2025-10-27T04:02:11.420Z [INFO] server-a177a662-a235-715e-193f-6ae8a1cf206e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a177a662-a235-715e-193f-6ae8a1cf206e Address:a177a662-a235-715e-193f-6ae8a1cf206e} {Suffrage:Voter ID:server-f17b52bf-c018-dc54-860b-2f477374ac1d Address:f17b52bf-c018-dc54-860b-2f477374ac1d} {Suffrage:Voter ID:server-583b636c-46ca-1812-3f98-b47d8d09b716 Address:583b636c-46ca-1812-3f98-b47d8d09b716}]" -2025-10-27T04:02:11.420Z [INFO] server-a177a662-a235-715e-193f-6ae8a1cf206e: entering follower state: follower="Node at a177a662-a235-715e-193f-6ae8a1cf206e [Follower]" leader-address= leader-id= -2025-10-27T04:02:11.420Z [INFO] server-f17b52bf-c018-dc54-860b-2f477374ac1d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a177a662-a235-715e-193f-6ae8a1cf206e Address:a177a662-a235-715e-193f-6ae8a1cf206e} {Suffrage:Voter ID:server-f17b52bf-c018-dc54-860b-2f477374ac1d Address:f17b52bf-c018-dc54-860b-2f477374ac1d} {Suffrage:Voter ID:server-583b636c-46ca-1812-3f98-b47d8d09b716 Address:583b636c-46ca-1812-3f98-b47d8d09b716}]" -2025-10-27T04:02:11.420Z [INFO] server-f17b52bf-c018-dc54-860b-2f477374ac1d: entering follower state: follower="Node at f17b52bf-c018-dc54-860b-2f477374ac1d [Follower]" leader-address= leader-id= -2025-10-27T04:02:11.420Z [INFO] server-583b636c-46ca-1812-3f98-b47d8d09b716: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a177a662-a235-715e-193f-6ae8a1cf206e Address:a177a662-a235-715e-193f-6ae8a1cf206e} {Suffrage:Voter ID:server-f17b52bf-c018-dc54-860b-2f477374ac1d Address:f17b52bf-c018-dc54-860b-2f477374ac1d} {Suffrage:Voter ID:server-583b636c-46ca-1812-3f98-b47d8d09b716 Address:583b636c-46ca-1812-3f98-b47d8d09b716}]" -2025-10-27T04:02:11.420Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:02:11.420Z [INFO] server-583b636c-46ca-1812-3f98-b47d8d09b716: entering follower state: follower="Node at 583b636c-46ca-1812-3f98-b47d8d09b716 [Follower]" leader-address= leader-id= -2025-10-27T04:02:11.472Z [WARN] server-a177a662-a235-715e-193f-6ae8a1cf206e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:11.472Z [INFO] server-a177a662-a235-715e-193f-6ae8a1cf206e: entering candidate state: node="Node at a177a662-a235-715e-193f-6ae8a1cf206e [Candidate]" term=2 -2025-10-27T04:02:11.472Z [INFO] server-a177a662-a235-715e-193f-6ae8a1cf206e: election won: term=2 tally=2 -2025-10-27T04:02:11.472Z [INFO] server-a177a662-a235-715e-193f-6ae8a1cf206e: entering leader state: leader="Node at a177a662-a235-715e-193f-6ae8a1cf206e [Leader]" -2025-10-27T04:02:11.472Z [INFO] server-a177a662-a235-715e-193f-6ae8a1cf206e: added peer, starting replication: peer=server-f17b52bf-c018-dc54-860b-2f477374ac1d -2025-10-27T04:02:11.472Z [INFO] server-a177a662-a235-715e-193f-6ae8a1cf206e: added peer, starting replication: peer=server-583b636c-46ca-1812-3f98-b47d8d09b716 -2025-10-27T04:02:11.472Z [INFO] server-a177a662-a235-715e-193f-6ae8a1cf206e: pipelining replication: peer="{Voter server-583b636c-46ca-1812-3f98-b47d8d09b716 583b636c-46ca-1812-3f98-b47d8d09b716}" -2025-10-27T04:02:11.472Z [INFO] server-a177a662-a235-715e-193f-6ae8a1cf206e: pipelining replication: peer="{Voter server-f17b52bf-c018-dc54-860b-2f477374ac1d f17b52bf-c018-dc54-860b-2f477374ac1d}" -2025-10-27T04:02:11.577Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:02:11.472286527 +0000 UTC m=+6.467159045 (2 nodes), 51.711593ms from start of poll, 52.213196ms from cluster start. Timeout at 2025-10-27 04:02:11.577290127 +0000 UTC m=+6.572162645, 105.0036ms after stability -2025-10-27T04:02:11.578Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:11.683Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:11.57802785 +0000 UTC m=+6.572900328 (1 nodes), 840ns from start of poll, 157.954479ms from cluster start. Timeout at 2025-10-27 04:02:11.68303797 +0000 UTC m=+6.677910448, 105.01012ms after stability -2025-10-27T04:02:11.694Z [INFO] server-a177a662-a235-715e-193f-6ae8a1cf206e: aborting pipeline replication: peer="{Voter server-f17b52bf-c018-dc54-860b-2f477374ac1d f17b52bf-c018-dc54-860b-2f477374ac1d}" -2025-10-27T04:02:11.694Z [INFO] server-a177a662-a235-715e-193f-6ae8a1cf206e: aborting pipeline replication: peer="{Voter server-583b636c-46ca-1812-3f98-b47d8d09b716 583b636c-46ca-1812-3f98-b47d8d09b716}" -2025-10-27T04:02:12.195Z [ERROR] server-a177a662-a235-715e-193f-6ae8a1cf206e: failed to heartbeat to: peer=583b636c-46ca-1812-3f98-b47d8d09b716 backoff time=10ms error="command timed out" ---- PASS: TestRaft_TripleNode (0.78s) +2026-11-29T10:28:15.621Z [INFO] server-a3a7c343-f703-d9d9-049d-5fab749198da: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a3a7c343-f703-d9d9-049d-5fab749198da Address:a3a7c343-f703-d9d9-049d-5fab749198da} {Suffrage:Voter ID:server-af6917ce-509a-6899-ba7b-1bc9c747ab4e Address:af6917ce-509a-6899-ba7b-1bc9c747ab4e} {Suffrage:Voter ID:server-6618a971-ec10-61f1-7712-e5af04c44587 Address:6618a971-ec10-61f1-7712-e5af04c44587}]" +2026-11-29T10:28:15.621Z [INFO] server-af6917ce-509a-6899-ba7b-1bc9c747ab4e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a3a7c343-f703-d9d9-049d-5fab749198da Address:a3a7c343-f703-d9d9-049d-5fab749198da} {Suffrage:Voter ID:server-af6917ce-509a-6899-ba7b-1bc9c747ab4e Address:af6917ce-509a-6899-ba7b-1bc9c747ab4e} {Suffrage:Voter ID:server-6618a971-ec10-61f1-7712-e5af04c44587 Address:6618a971-ec10-61f1-7712-e5af04c44587}]" +2026-11-29T10:28:15.621Z [INFO] server-a3a7c343-f703-d9d9-049d-5fab749198da: entering follower state: follower="Node at a3a7c343-f703-d9d9-049d-5fab749198da [Follower]" leader-address= leader-id= +2026-11-29T10:28:15.622Z [INFO] server-af6917ce-509a-6899-ba7b-1bc9c747ab4e: entering follower state: follower="Node at af6917ce-509a-6899-ba7b-1bc9c747ab4e [Follower]" leader-address= leader-id= +2026-11-29T10:28:15.622Z [INFO] server-6618a971-ec10-61f1-7712-e5af04c44587: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a3a7c343-f703-d9d9-049d-5fab749198da Address:a3a7c343-f703-d9d9-049d-5fab749198da} {Suffrage:Voter ID:server-af6917ce-509a-6899-ba7b-1bc9c747ab4e Address:af6917ce-509a-6899-ba7b-1bc9c747ab4e} {Suffrage:Voter ID:server-6618a971-ec10-61f1-7712-e5af04c44587 Address:6618a971-ec10-61f1-7712-e5af04c44587}]" +2026-11-29T10:28:15.622Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:28:15.622Z [INFO] server-6618a971-ec10-61f1-7712-e5af04c44587: entering follower state: follower="Node at 6618a971-ec10-61f1-7712-e5af04c44587 [Follower]" leader-address= leader-id= +2026-11-29T10:28:15.696Z [WARN] server-af6917ce-509a-6899-ba7b-1bc9c747ab4e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:15.696Z [INFO] server-af6917ce-509a-6899-ba7b-1bc9c747ab4e: entering candidate state: node="Node at af6917ce-509a-6899-ba7b-1bc9c747ab4e [Candidate]" term=2 +2026-11-29T10:28:15.696Z [INFO] server-af6917ce-509a-6899-ba7b-1bc9c747ab4e: election won: term=2 tally=2 +2026-11-29T10:28:15.696Z [INFO] server-af6917ce-509a-6899-ba7b-1bc9c747ab4e: entering leader state: leader="Node at af6917ce-509a-6899-ba7b-1bc9c747ab4e [Leader]" +2026-11-29T10:28:15.696Z [INFO] server-af6917ce-509a-6899-ba7b-1bc9c747ab4e: added peer, starting replication: peer=server-a3a7c343-f703-d9d9-049d-5fab749198da +2026-11-29T10:28:15.696Z [INFO] server-af6917ce-509a-6899-ba7b-1bc9c747ab4e: added peer, starting replication: peer=server-6618a971-ec10-61f1-7712-e5af04c44587 +2026-11-29T10:28:15.696Z [INFO] server-af6917ce-509a-6899-ba7b-1bc9c747ab4e: pipelining replication: peer="{Voter server-a3a7c343-f703-d9d9-049d-5fab749198da a3a7c343-f703-d9d9-049d-5fab749198da}" +2026-11-29T10:28:15.696Z [INFO] server-af6917ce-509a-6899-ba7b-1bc9c747ab4e: pipelining replication: peer="{Voter server-6618a971-ec10-61f1-7712-e5af04c44587 6618a971-ec10-61f1-7712-e5af04c44587}" +2026-11-29T10:28:15.802Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:28:15.696395059 +0000 UTC m=+6.431449241 (2 nodes), 74.2126ms from start of poll, 74.6746ms from cluster start. Timeout at 2026-11-29 10:28:15.801397179 +0000 UTC m=+6.536451321, 105.00208ms after stability +2026-11-29T10:28:15.802Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:15.907Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:15.802399779 +0000 UTC m=+6.537453921 (1 nodes), 840ns from start of poll, 180.67928ms from cluster start. Timeout at 2026-11-29 10:28:15.907406339 +0000 UTC m=+6.642460481, 105.00656ms after stability +2026-11-29T10:28:15.919Z [INFO] server-af6917ce-509a-6899-ba7b-1bc9c747ab4e: aborting pipeline replication: peer="{Voter server-6618a971-ec10-61f1-7712-e5af04c44587 6618a971-ec10-61f1-7712-e5af04c44587}" +2026-11-29T10:28:15.919Z [INFO] server-af6917ce-509a-6899-ba7b-1bc9c747ab4e: aborting pipeline replication: peer="{Voter server-a3a7c343-f703-d9d9-049d-5fab749198da a3a7c343-f703-d9d9-049d-5fab749198da}" +--- PASS: TestRaft_TripleNode (0.30s) === RUN TestRaft_LeaderFail -2025-10-27T04:02:12.196Z [INFO] server-8567da01-0865-aaa3-c97e-64e31d903537: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8567da01-0865-aaa3-c97e-64e31d903537 Address:8567da01-0865-aaa3-c97e-64e31d903537} {Suffrage:Voter ID:server-f54fa321-622e-6891-79e1-1aba09017168 Address:f54fa321-622e-6891-79e1-1aba09017168} {Suffrage:Voter ID:server-9f056d43-573a-c0c0-49a6-cd435009bb20 Address:9f056d43-573a-c0c0-49a6-cd435009bb20}]" -2025-10-27T04:02:12.196Z [INFO] server-f54fa321-622e-6891-79e1-1aba09017168: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8567da01-0865-aaa3-c97e-64e31d903537 Address:8567da01-0865-aaa3-c97e-64e31d903537} {Suffrage:Voter ID:server-f54fa321-622e-6891-79e1-1aba09017168 Address:f54fa321-622e-6891-79e1-1aba09017168} {Suffrage:Voter ID:server-9f056d43-573a-c0c0-49a6-cd435009bb20 Address:9f056d43-573a-c0c0-49a6-cd435009bb20}]" -2025-10-27T04:02:12.196Z [INFO] server-9f056d43-573a-c0c0-49a6-cd435009bb20: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8567da01-0865-aaa3-c97e-64e31d903537 Address:8567da01-0865-aaa3-c97e-64e31d903537} {Suffrage:Voter ID:server-f54fa321-622e-6891-79e1-1aba09017168 Address:f54fa321-622e-6891-79e1-1aba09017168} {Suffrage:Voter ID:server-9f056d43-573a-c0c0-49a6-cd435009bb20 Address:9f056d43-573a-c0c0-49a6-cd435009bb20}]" -2025-10-27T04:02:12.196Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:02:12.196Z [INFO] server-8567da01-0865-aaa3-c97e-64e31d903537: entering follower state: follower="Node at 8567da01-0865-aaa3-c97e-64e31d903537 [Follower]" leader-address= leader-id= -2025-10-27T04:02:12.196Z [INFO] server-f54fa321-622e-6891-79e1-1aba09017168: entering follower state: follower="Node at f54fa321-622e-6891-79e1-1aba09017168 [Follower]" leader-address= leader-id= -2025-10-27T04:02:12.196Z [INFO] server-9f056d43-573a-c0c0-49a6-cd435009bb20: entering follower state: follower="Node at 9f056d43-573a-c0c0-49a6-cd435009bb20 [Follower]" leader-address= leader-id= -2025-10-27T04:02:12.268Z [WARN] server-8567da01-0865-aaa3-c97e-64e31d903537: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:12.268Z [INFO] server-8567da01-0865-aaa3-c97e-64e31d903537: entering candidate state: node="Node at 8567da01-0865-aaa3-c97e-64e31d903537 [Candidate]" term=2 -2025-10-27T04:02:12.268Z [INFO] server-8567da01-0865-aaa3-c97e-64e31d903537: election won: term=2 tally=2 -2025-10-27T04:02:12.268Z [INFO] server-8567da01-0865-aaa3-c97e-64e31d903537: entering leader state: leader="Node at 8567da01-0865-aaa3-c97e-64e31d903537 [Leader]" -2025-10-27T04:02:12.268Z [INFO] server-8567da01-0865-aaa3-c97e-64e31d903537: added peer, starting replication: peer=server-f54fa321-622e-6891-79e1-1aba09017168 -2025-10-27T04:02:12.268Z [INFO] server-8567da01-0865-aaa3-c97e-64e31d903537: added peer, starting replication: peer=server-9f056d43-573a-c0c0-49a6-cd435009bb20 -2025-10-27T04:02:12.268Z [WARN] server-f54fa321-622e-6891-79e1-1aba09017168: rejecting vote request since our last term is greater: candidate=8567da01-0865-aaa3-c97e-64e31d903537 last-term=2 last-candidate-term=1 -2025-10-27T04:02:12.268Z [INFO] server-8567da01-0865-aaa3-c97e-64e31d903537: pipelining replication: peer="{Voter server-9f056d43-573a-c0c0-49a6-cd435009bb20 9f056d43-573a-c0c0-49a6-cd435009bb20}" -2025-10-27T04:02:12.268Z [INFO] server-8567da01-0865-aaa3-c97e-64e31d903537: pipelining replication: peer="{Voter server-f54fa321-622e-6891-79e1-1aba09017168 f54fa321-622e-6891-79e1-1aba09017168}" -2025-10-27T04:02:12.373Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:02:12.268671451 +0000 UTC m=+7.263543889 (2 nodes), 72.096533ms from start of poll, 72.374855ms from cluster start. Timeout at 2025-10-27 04:02:12.373674811 +0000 UTC m=+7.368547249, 105.00336ms after stability -2025-10-27T04:02:12.373Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:12.479Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:12.37393861 +0000 UTC m=+7.368811008 (1 nodes), 1.48µs from start of poll, 177.641974ms from cluster start. Timeout at 2025-10-27 04:02:12.47894369 +0000 UTC m=+7.473816128, 105.00512ms after stability - raft_test.go:378: [INFO] Disconnecting Node at 8567da01-0865-aaa3-c97e-64e31d903537 [Leader] -2025-10-27T04:02:12.496Z [ERROR] server-8567da01-0865-aaa3-c97e-64e31d903537: failed to pipeline appendEntries: peer="{Voter server-f54fa321-622e-6891-79e1-1aba09017168 f54fa321-622e-6891-79e1-1aba09017168}" error="append pipeline closed" -2025-10-27T04:02:12.496Z [INFO] server-8567da01-0865-aaa3-c97e-64e31d903537: aborting pipeline replication: peer="{Voter server-f54fa321-622e-6891-79e1-1aba09017168 f54fa321-622e-6891-79e1-1aba09017168}" -2025-10-27T04:02:12.497Z [ERROR] server-8567da01-0865-aaa3-c97e-64e31d903537: failed to pipeline appendEntries: peer="{Voter server-9f056d43-573a-c0c0-49a6-cd435009bb20 9f056d43-573a-c0c0-49a6-cd435009bb20}" error="append pipeline closed" -2025-10-27T04:02:12.497Z [INFO] server-8567da01-0865-aaa3-c97e-64e31d903537: aborting pipeline replication: peer="{Voter server-9f056d43-573a-c0c0-49a6-cd435009bb20 9f056d43-573a-c0c0-49a6-cd435009bb20}" -2025-10-27T04:02:12.498Z [ERROR] server-8567da01-0865-aaa3-c97e-64e31d903537: failed to heartbeat to: peer=9f056d43-573a-c0c0-49a6-cd435009bb20 backoff time=10ms error="failed to connect to peer: 9f056d43-573a-c0c0-49a6-cd435009bb20" -2025-10-27T04:02:12.498Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:12.502Z [ERROR] server-8567da01-0865-aaa3-c97e-64e31d903537: failed to heartbeat to: peer=f54fa321-622e-6891-79e1-1aba09017168 backoff time=10ms error="failed to connect to peer: f54fa321-622e-6891-79e1-1aba09017168" -2025-10-27T04:02:12.503Z [ERROR] server-8567da01-0865-aaa3-c97e-64e31d903537: failed to appendEntries to: peer="{Voter server-9f056d43-573a-c0c0-49a6-cd435009bb20 9f056d43-573a-c0c0-49a6-cd435009bb20}" error="failed to connect to peer: 9f056d43-573a-c0c0-49a6-cd435009bb20" -2025-10-27T04:02:12.503Z [ERROR] server-8567da01-0865-aaa3-c97e-64e31d903537: failed to appendEntries to: peer="{Voter server-f54fa321-622e-6891-79e1-1aba09017168 f54fa321-622e-6891-79e1-1aba09017168}" error="failed to connect to peer: f54fa321-622e-6891-79e1-1aba09017168" -2025-10-27T04:02:12.519Z [ERROR] server-8567da01-0865-aaa3-c97e-64e31d903537: failed to heartbeat to: peer=f54fa321-622e-6891-79e1-1aba09017168 backoff time=10ms error="failed to connect to peer: f54fa321-622e-6891-79e1-1aba09017168" -2025-10-27T04:02:12.519Z [ERROR] server-8567da01-0865-aaa3-c97e-64e31d903537: failed to heartbeat to: peer=9f056d43-573a-c0c0-49a6-cd435009bb20 backoff time=10ms error="failed to connect to peer: 9f056d43-573a-c0c0-49a6-cd435009bb20" -2025-10-27T04:02:12.519Z [ERROR] server-8567da01-0865-aaa3-c97e-64e31d903537: failed to appendEntries to: peer="{Voter server-f54fa321-622e-6891-79e1-1aba09017168 f54fa321-622e-6891-79e1-1aba09017168}" error="failed to connect to peer: f54fa321-622e-6891-79e1-1aba09017168" -2025-10-27T04:02:12.524Z [ERROR] server-8567da01-0865-aaa3-c97e-64e31d903537: failed to appendEntries to: peer="{Voter server-9f056d43-573a-c0c0-49a6-cd435009bb20 9f056d43-573a-c0c0-49a6-cd435009bb20}" error="failed to connect to peer: 9f056d43-573a-c0c0-49a6-cd435009bb20" -2025-10-27T04:02:12.539Z [ERROR] server-8567da01-0865-aaa3-c97e-64e31d903537: failed to heartbeat to: peer=f54fa321-622e-6891-79e1-1aba09017168 backoff time=10ms error="failed to connect to peer: f54fa321-622e-6891-79e1-1aba09017168" -2025-10-27T04:02:12.539Z [ERROR] server-8567da01-0865-aaa3-c97e-64e31d903537: failed to heartbeat to: peer=9f056d43-573a-c0c0-49a6-cd435009bb20 backoff time=10ms error="failed to connect to peer: 9f056d43-573a-c0c0-49a6-cd435009bb20" -2025-10-27T04:02:12.539Z [WARN] server-8567da01-0865-aaa3-c97e-64e31d903537: failed to contact: server-id=server-9f056d43-573a-c0c0-49a6-cd435009bb20 time=50.362224ms -2025-10-27T04:02:12.540Z [ERROR] server-8567da01-0865-aaa3-c97e-64e31d903537: failed to appendEntries to: peer="{Voter server-f54fa321-622e-6891-79e1-1aba09017168 f54fa321-622e-6891-79e1-1aba09017168}" error="failed to connect to peer: f54fa321-622e-6891-79e1-1aba09017168" -2025-10-27T04:02:12.543Z [ERROR] server-8567da01-0865-aaa3-c97e-64e31d903537: failed to appendEntries to: peer="{Voter server-9f056d43-573a-c0c0-49a6-cd435009bb20 9f056d43-573a-c0c0-49a6-cd435009bb20}" error="failed to connect to peer: 9f056d43-573a-c0c0-49a6-cd435009bb20" -2025-10-27T04:02:12.547Z [WARN] server-9f056d43-573a-c0c0-49a6-cd435009bb20: heartbeat timeout reached, starting election: last-leader-addr=8567da01-0865-aaa3-c97e-64e31d903537 last-leader-id=server-8567da01-0865-aaa3-c97e-64e31d903537 -2025-10-27T04:02:12.547Z [INFO] server-9f056d43-573a-c0c0-49a6-cd435009bb20: entering candidate state: node="Node at 9f056d43-573a-c0c0-49a6-cd435009bb20 [Candidate]" term=3 -2025-10-27T04:02:12.547Z [WARN] server-f54fa321-622e-6891-79e1-1aba09017168: rejecting vote request since we have a leader: from=9f056d43-573a-c0c0-49a6-cd435009bb20 leader=8567da01-0865-aaa3-c97e-64e31d903537 leader-id=server-8567da01-0865-aaa3-c97e-64e31d903537 -2025-10-27T04:02:12.547Z [ERROR] server-9f056d43-573a-c0c0-49a6-cd435009bb20: failed to make requestVote RPC: target="{Voter server-8567da01-0865-aaa3-c97e-64e31d903537 8567da01-0865-aaa3-c97e-64e31d903537}" error="failed to connect to peer: 8567da01-0865-aaa3-c97e-64e31d903537" term=3 -2025-10-27T04:02:12.550Z [WARN] server-8567da01-0865-aaa3-c97e-64e31d903537: failed to contact: server-id=server-f54fa321-622e-6891-79e1-1aba09017168 time=54.860775ms -2025-10-27T04:02:12.550Z [WARN] server-8567da01-0865-aaa3-c97e-64e31d903537: failed to contact: server-id=server-9f056d43-573a-c0c0-49a6-cd435009bb20 time=61.190258ms -2025-10-27T04:02:12.550Z [WARN] server-8567da01-0865-aaa3-c97e-64e31d903537: failed to contact quorum of nodes, stepping down -2025-10-27T04:02:12.550Z [INFO] server-8567da01-0865-aaa3-c97e-64e31d903537: entering follower state: follower="Node at 8567da01-0865-aaa3-c97e-64e31d903537 [Follower]" leader-address= leader-id= -2025-10-27T04:02:12.558Z [ERROR] server-8567da01-0865-aaa3-c97e-64e31d903537: failed to heartbeat to: peer=9f056d43-573a-c0c0-49a6-cd435009bb20 backoff time=20ms error="failed to connect to peer: 9f056d43-573a-c0c0-49a6-cd435009bb20" -2025-10-27T04:02:12.559Z [ERROR] server-8567da01-0865-aaa3-c97e-64e31d903537: failed to heartbeat to: peer=f54fa321-622e-6891-79e1-1aba09017168 backoff time=20ms error="failed to connect to peer: f54fa321-622e-6891-79e1-1aba09017168" -2025-10-27T04:02:12.564Z [WARN] server-f54fa321-622e-6891-79e1-1aba09017168: heartbeat timeout reached, starting election: last-leader-addr=8567da01-0865-aaa3-c97e-64e31d903537 last-leader-id=server-8567da01-0865-aaa3-c97e-64e31d903537 -2025-10-27T04:02:12.564Z [INFO] server-f54fa321-622e-6891-79e1-1aba09017168: entering candidate state: node="Node at f54fa321-622e-6891-79e1-1aba09017168 [Candidate]" term=3 -2025-10-27T04:02:12.564Z [INFO] server-9f056d43-573a-c0c0-49a6-cd435009bb20: duplicate requestVote for same term: term=3 -2025-10-27T04:02:12.564Z [ERROR] server-f54fa321-622e-6891-79e1-1aba09017168: failed to make requestVote RPC: target="{Voter server-8567da01-0865-aaa3-c97e-64e31d903537 8567da01-0865-aaa3-c97e-64e31d903537}" error="failed to connect to peer: 8567da01-0865-aaa3-c97e-64e31d903537" term=3 -2025-10-27T04:02:12.569Z [ERROR] server-8567da01-0865-aaa3-c97e-64e31d903537: failed to appendEntries to: peer="{Voter server-f54fa321-622e-6891-79e1-1aba09017168 f54fa321-622e-6891-79e1-1aba09017168}" error="failed to connect to peer: f54fa321-622e-6891-79e1-1aba09017168" -2025-10-27T04:02:12.570Z [ERROR] server-8567da01-0865-aaa3-c97e-64e31d903537: failed to appendEntries to: peer="{Voter server-9f056d43-573a-c0c0-49a6-cd435009bb20 9f056d43-573a-c0c0-49a6-cd435009bb20}" error="failed to connect to peer: 9f056d43-573a-c0c0-49a6-cd435009bb20" -2025-10-27T04:02:12.619Z [WARN] server-f54fa321-622e-6891-79e1-1aba09017168: Election timeout reached, restarting election -2025-10-27T04:02:12.619Z [INFO] server-f54fa321-622e-6891-79e1-1aba09017168: entering candidate state: node="Node at f54fa321-622e-6891-79e1-1aba09017168 [Candidate]" term=4 -2025-10-27T04:02:12.619Z [INFO] server-9f056d43-573a-c0c0-49a6-cd435009bb20: entering follower state: follower="Node at 9f056d43-573a-c0c0-49a6-cd435009bb20 [Follower]" leader-address= leader-id= -2025-10-27T04:02:12.619Z [INFO] server-f54fa321-622e-6891-79e1-1aba09017168: election won: term=4 tally=2 -2025-10-27T04:02:12.619Z [INFO] server-f54fa321-622e-6891-79e1-1aba09017168: entering leader state: leader="Node at f54fa321-622e-6891-79e1-1aba09017168 [Leader]" -2025-10-27T04:02:12.619Z [INFO] server-f54fa321-622e-6891-79e1-1aba09017168: added peer, starting replication: peer=server-8567da01-0865-aaa3-c97e-64e31d903537 -2025-10-27T04:02:12.619Z [INFO] server-f54fa321-622e-6891-79e1-1aba09017168: added peer, starting replication: peer=server-9f056d43-573a-c0c0-49a6-cd435009bb20 -2025-10-27T04:02:12.619Z [ERROR] server-f54fa321-622e-6891-79e1-1aba09017168: failed to make requestVote RPC: target="{Voter server-8567da01-0865-aaa3-c97e-64e31d903537 8567da01-0865-aaa3-c97e-64e31d903537}" error="failed to connect to peer: 8567da01-0865-aaa3-c97e-64e31d903537" term=4 -2025-10-27T04:02:12.619Z [ERROR] server-f54fa321-622e-6891-79e1-1aba09017168: failed to appendEntries to: peer="{Voter server-8567da01-0865-aaa3-c97e-64e31d903537 8567da01-0865-aaa3-c97e-64e31d903537}" error="failed to connect to peer: 8567da01-0865-aaa3-c97e-64e31d903537" -2025-10-27T04:02:12.619Z [INFO] server-f54fa321-622e-6891-79e1-1aba09017168: pipelining replication: peer="{Voter server-9f056d43-573a-c0c0-49a6-cd435009bb20 9f056d43-573a-c0c0-49a6-cd435009bb20}" -2025-10-27T04:02:12.620Z [WARN] server-8567da01-0865-aaa3-c97e-64e31d903537: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:12.620Z [INFO] server-8567da01-0865-aaa3-c97e-64e31d903537: entering candidate state: node="Node at 8567da01-0865-aaa3-c97e-64e31d903537 [Candidate]" term=3 -2025-10-27T04:02:12.620Z [ERROR] server-8567da01-0865-aaa3-c97e-64e31d903537: failed to make requestVote RPC: target="{Voter server-9f056d43-573a-c0c0-49a6-cd435009bb20 9f056d43-573a-c0c0-49a6-cd435009bb20}" error="failed to connect to peer: 9f056d43-573a-c0c0-49a6-cd435009bb20" term=3 -2025-10-27T04:02:12.620Z [ERROR] server-8567da01-0865-aaa3-c97e-64e31d903537: failed to make requestVote RPC: target="{Voter server-f54fa321-622e-6891-79e1-1aba09017168 f54fa321-622e-6891-79e1-1aba09017168}" error="failed to connect to peer: f54fa321-622e-6891-79e1-1aba09017168" term=3 -2025-10-27T04:02:12.628Z [ERROR] server-f54fa321-622e-6891-79e1-1aba09017168: failed to heartbeat to: peer=8567da01-0865-aaa3-c97e-64e31d903537 backoff time=10ms error="failed to connect to peer: 8567da01-0865-aaa3-c97e-64e31d903537" -2025-10-27T04:02:12.637Z [ERROR] server-f54fa321-622e-6891-79e1-1aba09017168: failed to appendEntries to: peer="{Voter server-8567da01-0865-aaa3-c97e-64e31d903537 8567da01-0865-aaa3-c97e-64e31d903537}" error="failed to connect to peer: 8567da01-0865-aaa3-c97e-64e31d903537" -2025-10-27T04:02:12.647Z [ERROR] server-f54fa321-622e-6891-79e1-1aba09017168: failed to heartbeat to: peer=8567da01-0865-aaa3-c97e-64e31d903537 backoff time=10ms error="failed to connect to peer: 8567da01-0865-aaa3-c97e-64e31d903537" -2025-10-27T04:02:12.655Z [ERROR] server-f54fa321-622e-6891-79e1-1aba09017168: failed to appendEntries to: peer="{Voter server-8567da01-0865-aaa3-c97e-64e31d903537 8567da01-0865-aaa3-c97e-64e31d903537}" error="failed to connect to peer: 8567da01-0865-aaa3-c97e-64e31d903537" -2025-10-27T04:02:12.665Z [ERROR] server-f54fa321-622e-6891-79e1-1aba09017168: failed to heartbeat to: peer=8567da01-0865-aaa3-c97e-64e31d903537 backoff time=10ms error="failed to connect to peer: 8567da01-0865-aaa3-c97e-64e31d903537" -2025-10-27T04:02:12.670Z [WARN] server-f54fa321-622e-6891-79e1-1aba09017168: failed to contact: server-id=server-8567da01-0865-aaa3-c97e-64e31d903537 time=50.342544ms -2025-10-27T04:02:12.681Z [ERROR] server-f54fa321-622e-6891-79e1-1aba09017168: failed to heartbeat to: peer=8567da01-0865-aaa3-c97e-64e31d903537 backoff time=20ms error="failed to connect to peer: 8567da01-0865-aaa3-c97e-64e31d903537" -2025-10-27T04:02:12.685Z [ERROR] server-f54fa321-622e-6891-79e1-1aba09017168: failed to appendEntries to: peer="{Voter server-8567da01-0865-aaa3-c97e-64e31d903537 8567da01-0865-aaa3-c97e-64e31d903537}" error="failed to connect to peer: 8567da01-0865-aaa3-c97e-64e31d903537" -2025-10-27T04:02:12.705Z [WARN] server-8567da01-0865-aaa3-c97e-64e31d903537: Election timeout reached, restarting election -2025-10-27T04:02:12.705Z [INFO] server-8567da01-0865-aaa3-c97e-64e31d903537: entering candidate state: node="Node at 8567da01-0865-aaa3-c97e-64e31d903537 [Candidate]" term=4 -2025-10-27T04:02:12.705Z [ERROR] server-8567da01-0865-aaa3-c97e-64e31d903537: failed to make requestVote RPC: target="{Voter server-9f056d43-573a-c0c0-49a6-cd435009bb20 9f056d43-573a-c0c0-49a6-cd435009bb20}" error="failed to connect to peer: 9f056d43-573a-c0c0-49a6-cd435009bb20" term=4 -2025-10-27T04:02:12.705Z [ERROR] server-8567da01-0865-aaa3-c97e-64e31d903537: failed to make requestVote RPC: target="{Voter server-f54fa321-622e-6891-79e1-1aba09017168 f54fa321-622e-6891-79e1-1aba09017168}" error="failed to connect to peer: f54fa321-622e-6891-79e1-1aba09017168" term=4 -2025-10-27T04:02:12.711Z [ERROR] server-f54fa321-622e-6891-79e1-1aba09017168: failed to heartbeat to: peer=8567da01-0865-aaa3-c97e-64e31d903537 backoff time=25ms error="failed to connect to peer: 8567da01-0865-aaa3-c97e-64e31d903537" -2025-10-27T04:02:12.721Z [WARN] server-f54fa321-622e-6891-79e1-1aba09017168: failed to contact: server-id=server-8567da01-0865-aaa3-c97e-64e31d903537 time=101.160652ms -2025-10-27T04:02:12.726Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:12.620852458 +0000 UTC m=+7.615724856 (1 nodes), 122.345996ms from start of poll, 424.555822ms from cluster start. Timeout at 2025-10-27 04:02:12.725853978 +0000 UTC m=+7.720726416, 105.00156ms after stability - raft_test.go:413: [INFO] Reconnecting Node at 8567da01-0865-aaa3-c97e-64e31d903537 [Candidate] -2025-10-27T04:02:12.733Z [INFO] server-8567da01-0865-aaa3-c97e-64e31d903537: entering follower state: follower="Node at 8567da01-0865-aaa3-c97e-64e31d903537 [Follower]" leader-address=f54fa321-622e-6891-79e1-1aba09017168 leader-id=server-f54fa321-622e-6891-79e1-1aba09017168 -2025-10-27T04:02:12.734Z [INFO] server-f54fa321-622e-6891-79e1-1aba09017168: pipelining replication: peer="{Voter server-8567da01-0865-aaa3-c97e-64e31d903537 8567da01-0865-aaa3-c97e-64e31d903537}" -2025-10-27T04:02:12.739Z [INFO] server-f54fa321-622e-6891-79e1-1aba09017168: aborting pipeline replication: peer="{Voter server-9f056d43-573a-c0c0-49a6-cd435009bb20 9f056d43-573a-c0c0-49a6-cd435009bb20}" -2025-10-27T04:02:12.739Z [INFO] server-f54fa321-622e-6891-79e1-1aba09017168: aborting pipeline replication: peer="{Voter server-8567da01-0865-aaa3-c97e-64e31d903537 8567da01-0865-aaa3-c97e-64e31d903537}" +2026-11-29T10:28:15.920Z [INFO] server-9fec4e0b-da28-b197-8975-df29c2af1bea: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9fec4e0b-da28-b197-8975-df29c2af1bea Address:9fec4e0b-da28-b197-8975-df29c2af1bea} {Suffrage:Voter ID:server-1e7d22c6-0109-5302-e057-132632171018 Address:1e7d22c6-0109-5302-e057-132632171018} {Suffrage:Voter ID:server-0d56fb65-0575-9050-5780-b8ac900c0633 Address:0d56fb65-0575-9050-5780-b8ac900c0633}]" +2026-11-29T10:28:15.920Z [INFO] server-1e7d22c6-0109-5302-e057-132632171018: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9fec4e0b-da28-b197-8975-df29c2af1bea Address:9fec4e0b-da28-b197-8975-df29c2af1bea} {Suffrage:Voter ID:server-1e7d22c6-0109-5302-e057-132632171018 Address:1e7d22c6-0109-5302-e057-132632171018} {Suffrage:Voter ID:server-0d56fb65-0575-9050-5780-b8ac900c0633 Address:0d56fb65-0575-9050-5780-b8ac900c0633}]" +2026-11-29T10:28:15.920Z [INFO] server-9fec4e0b-da28-b197-8975-df29c2af1bea: entering follower state: follower="Node at 9fec4e0b-da28-b197-8975-df29c2af1bea [Follower]" leader-address= leader-id= +2026-11-29T10:28:15.920Z [INFO] server-0d56fb65-0575-9050-5780-b8ac900c0633: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9fec4e0b-da28-b197-8975-df29c2af1bea Address:9fec4e0b-da28-b197-8975-df29c2af1bea} {Suffrage:Voter ID:server-1e7d22c6-0109-5302-e057-132632171018 Address:1e7d22c6-0109-5302-e057-132632171018} {Suffrage:Voter ID:server-0d56fb65-0575-9050-5780-b8ac900c0633 Address:0d56fb65-0575-9050-5780-b8ac900c0633}]" +2026-11-29T10:28:15.920Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:28:15.920Z [INFO] server-1e7d22c6-0109-5302-e057-132632171018: entering follower state: follower="Node at 1e7d22c6-0109-5302-e057-132632171018 [Follower]" leader-address= leader-id= +2026-11-29T10:28:15.920Z [INFO] server-0d56fb65-0575-9050-5780-b8ac900c0633: entering follower state: follower="Node at 0d56fb65-0575-9050-5780-b8ac900c0633 [Follower]" leader-address= leader-id= +2026-11-29T10:28:15.983Z [WARN] server-0d56fb65-0575-9050-5780-b8ac900c0633: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:15.983Z [INFO] server-0d56fb65-0575-9050-5780-b8ac900c0633: entering candidate state: node="Node at 0d56fb65-0575-9050-5780-b8ac900c0633 [Candidate]" term=2 +2026-11-29T10:28:15.983Z [INFO] server-0d56fb65-0575-9050-5780-b8ac900c0633: election won: term=2 tally=2 +2026-11-29T10:28:15.983Z [INFO] server-0d56fb65-0575-9050-5780-b8ac900c0633: entering leader state: leader="Node at 0d56fb65-0575-9050-5780-b8ac900c0633 [Leader]" +2026-11-29T10:28:15.984Z [INFO] server-0d56fb65-0575-9050-5780-b8ac900c0633: added peer, starting replication: peer=server-9fec4e0b-da28-b197-8975-df29c2af1bea +2026-11-29T10:28:15.984Z [INFO] server-0d56fb65-0575-9050-5780-b8ac900c0633: added peer, starting replication: peer=server-1e7d22c6-0109-5302-e057-132632171018 +2026-11-29T10:28:15.984Z [INFO] server-0d56fb65-0575-9050-5780-b8ac900c0633: pipelining replication: peer="{Voter server-9fec4e0b-da28-b197-8975-df29c2af1bea 9fec4e0b-da28-b197-8975-df29c2af1bea}" +2026-11-29T10:28:15.984Z [INFO] server-0d56fb65-0575-9050-5780-b8ac900c0633: pipelining replication: peer="{Voter server-1e7d22c6-0109-5302-e057-132632171018 1e7d22c6-0109-5302-e057-132632171018}" +2026-11-29T10:28:16.089Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:28:15.984063659 +0000 UTC m=+6.719117801 (2 nodes), 63.44528ms from start of poll, 63.8946ms from cluster start. Timeout at 2026-11-29 10:28:16.089065779 +0000 UTC m=+6.824119921, 105.00212ms after stability +2026-11-29T10:28:16.089Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:16.195Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:16.089588179 +0000 UTC m=+6.824642321 (1 nodes), 880ns from start of poll, 169.41912ms from cluster start. Timeout at 2026-11-29 10:28:16.194592059 +0000 UTC m=+6.929646161, 105.00384ms after stability + raft_test.go:378: [INFO] Disconnecting Node at 0d56fb65-0575-9050-5780-b8ac900c0633 [Leader] +2026-11-29T10:28:16.209Z [ERROR] server-0d56fb65-0575-9050-5780-b8ac900c0633: failed to pipeline appendEntries: peer="{Voter server-9fec4e0b-da28-b197-8975-df29c2af1bea 9fec4e0b-da28-b197-8975-df29c2af1bea}" error="append pipeline closed" +2026-11-29T10:28:16.209Z [INFO] server-0d56fb65-0575-9050-5780-b8ac900c0633: aborting pipeline replication: peer="{Voter server-9fec4e0b-da28-b197-8975-df29c2af1bea 9fec4e0b-da28-b197-8975-df29c2af1bea}" +2026-11-29T10:28:16.211Z [ERROR] server-0d56fb65-0575-9050-5780-b8ac900c0633: failed to heartbeat to: peer=1e7d22c6-0109-5302-e057-132632171018 backoff time=10ms error="failed to connect to peer: 1e7d22c6-0109-5302-e057-132632171018" +2026-11-29T10:28:16.211Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:16.212Z [ERROR] server-0d56fb65-0575-9050-5780-b8ac900c0633: failed to pipeline appendEntries: peer="{Voter server-1e7d22c6-0109-5302-e057-132632171018 1e7d22c6-0109-5302-e057-132632171018}" error="append pipeline closed" +2026-11-29T10:28:16.212Z [INFO] server-0d56fb65-0575-9050-5780-b8ac900c0633: aborting pipeline replication: peer="{Voter server-1e7d22c6-0109-5302-e057-132632171018 1e7d22c6-0109-5302-e057-132632171018}" +2026-11-29T10:28:16.213Z [ERROR] server-0d56fb65-0575-9050-5780-b8ac900c0633: failed to heartbeat to: peer=9fec4e0b-da28-b197-8975-df29c2af1bea backoff time=10ms error="failed to connect to peer: 9fec4e0b-da28-b197-8975-df29c2af1bea" +2026-11-29T10:28:16.216Z [ERROR] server-0d56fb65-0575-9050-5780-b8ac900c0633: failed to appendEntries to: peer="{Voter server-9fec4e0b-da28-b197-8975-df29c2af1bea 9fec4e0b-da28-b197-8975-df29c2af1bea}" error="failed to connect to peer: 9fec4e0b-da28-b197-8975-df29c2af1bea" +2026-11-29T10:28:16.220Z [ERROR] server-0d56fb65-0575-9050-5780-b8ac900c0633: failed to appendEntries to: peer="{Voter server-1e7d22c6-0109-5302-e057-132632171018 1e7d22c6-0109-5302-e057-132632171018}" error="failed to connect to peer: 1e7d22c6-0109-5302-e057-132632171018" +2026-11-29T10:28:16.231Z [ERROR] server-0d56fb65-0575-9050-5780-b8ac900c0633: failed to heartbeat to: peer=9fec4e0b-da28-b197-8975-df29c2af1bea backoff time=10ms error="failed to connect to peer: 9fec4e0b-da28-b197-8975-df29c2af1bea" +2026-11-29T10:28:16.232Z [ERROR] server-0d56fb65-0575-9050-5780-b8ac900c0633: failed to heartbeat to: peer=1e7d22c6-0109-5302-e057-132632171018 backoff time=10ms error="failed to connect to peer: 1e7d22c6-0109-5302-e057-132632171018" +2026-11-29T10:28:16.232Z [ERROR] server-0d56fb65-0575-9050-5780-b8ac900c0633: failed to appendEntries to: peer="{Voter server-9fec4e0b-da28-b197-8975-df29c2af1bea 9fec4e0b-da28-b197-8975-df29c2af1bea}" error="failed to connect to peer: 9fec4e0b-da28-b197-8975-df29c2af1bea" +2026-11-29T10:28:16.238Z [ERROR] server-0d56fb65-0575-9050-5780-b8ac900c0633: failed to appendEntries to: peer="{Voter server-1e7d22c6-0109-5302-e057-132632171018 1e7d22c6-0109-5302-e057-132632171018}" error="failed to connect to peer: 1e7d22c6-0109-5302-e057-132632171018" +2026-11-29T10:28:16.248Z [ERROR] server-0d56fb65-0575-9050-5780-b8ac900c0633: failed to heartbeat to: peer=9fec4e0b-da28-b197-8975-df29c2af1bea backoff time=10ms error="failed to connect to peer: 9fec4e0b-da28-b197-8975-df29c2af1bea" +2026-11-29T10:28:16.250Z [ERROR] server-0d56fb65-0575-9050-5780-b8ac900c0633: failed to appendEntries to: peer="{Voter server-9fec4e0b-da28-b197-8975-df29c2af1bea 9fec4e0b-da28-b197-8975-df29c2af1bea}" error="failed to connect to peer: 9fec4e0b-da28-b197-8975-df29c2af1bea" +2026-11-29T10:28:16.252Z [ERROR] server-0d56fb65-0575-9050-5780-b8ac900c0633: failed to heartbeat to: peer=1e7d22c6-0109-5302-e057-132632171018 backoff time=10ms error="failed to connect to peer: 1e7d22c6-0109-5302-e057-132632171018" +2026-11-29T10:28:16.252Z [WARN] server-0d56fb65-0575-9050-5780-b8ac900c0633: failed to contact: server-id=server-1e7d22c6-0109-5302-e057-132632171018 time=50.68756ms +2026-11-29T10:28:16.259Z [ERROR] server-0d56fb65-0575-9050-5780-b8ac900c0633: failed to appendEntries to: peer="{Voter server-1e7d22c6-0109-5302-e057-132632171018 1e7d22c6-0109-5302-e057-132632171018}" error="failed to connect to peer: 1e7d22c6-0109-5302-e057-132632171018" +2026-11-29T10:28:16.263Z [WARN] server-0d56fb65-0575-9050-5780-b8ac900c0633: failed to contact: server-id=server-9fec4e0b-da28-b197-8975-df29c2af1bea time=60.55912ms +2026-11-29T10:28:16.263Z [WARN] server-0d56fb65-0575-9050-5780-b8ac900c0633: failed to contact: server-id=server-1e7d22c6-0109-5302-e057-132632171018 time=61.70824ms +2026-11-29T10:28:16.264Z [WARN] server-0d56fb65-0575-9050-5780-b8ac900c0633: failed to contact quorum of nodes, stepping down +2026-11-29T10:28:16.264Z [INFO] server-0d56fb65-0575-9050-5780-b8ac900c0633: entering follower state: follower="Node at 0d56fb65-0575-9050-5780-b8ac900c0633 [Follower]" leader-address= leader-id= +2026-11-29T10:28:16.265Z [ERROR] server-0d56fb65-0575-9050-5780-b8ac900c0633: failed to heartbeat to: peer=9fec4e0b-da28-b197-8975-df29c2af1bea backoff time=20ms error="failed to connect to peer: 9fec4e0b-da28-b197-8975-df29c2af1bea" +2026-11-29T10:28:16.273Z [WARN] server-9fec4e0b-da28-b197-8975-df29c2af1bea: heartbeat timeout reached, starting election: last-leader-addr=0d56fb65-0575-9050-5780-b8ac900c0633 last-leader-id=server-0d56fb65-0575-9050-5780-b8ac900c0633 +2026-11-29T10:28:16.273Z [INFO] server-9fec4e0b-da28-b197-8975-df29c2af1bea: entering candidate state: node="Node at 9fec4e0b-da28-b197-8975-df29c2af1bea [Candidate]" term=3 +2026-11-29T10:28:16.273Z [WARN] server-1e7d22c6-0109-5302-e057-132632171018: rejecting vote request since we have a leader: from=9fec4e0b-da28-b197-8975-df29c2af1bea leader=0d56fb65-0575-9050-5780-b8ac900c0633 leader-id=server-0d56fb65-0575-9050-5780-b8ac900c0633 +2026-11-29T10:28:16.273Z [ERROR] server-9fec4e0b-da28-b197-8975-df29c2af1bea: failed to make requestVote RPC: target="{Voter server-0d56fb65-0575-9050-5780-b8ac900c0633 0d56fb65-0575-9050-5780-b8ac900c0633}" error="failed to connect to peer: 0d56fb65-0575-9050-5780-b8ac900c0633" term=3 +2026-11-29T10:28:16.281Z [ERROR] server-0d56fb65-0575-9050-5780-b8ac900c0633: failed to appendEntries to: peer="{Voter server-9fec4e0b-da28-b197-8975-df29c2af1bea 9fec4e0b-da28-b197-8975-df29c2af1bea}" error="failed to connect to peer: 9fec4e0b-da28-b197-8975-df29c2af1bea" +2026-11-29T10:28:16.301Z [WARN] server-1e7d22c6-0109-5302-e057-132632171018: heartbeat timeout reached, starting election: last-leader-addr=0d56fb65-0575-9050-5780-b8ac900c0633 last-leader-id=server-0d56fb65-0575-9050-5780-b8ac900c0633 +2026-11-29T10:28:16.301Z [INFO] server-1e7d22c6-0109-5302-e057-132632171018: entering candidate state: node="Node at 1e7d22c6-0109-5302-e057-132632171018 [Candidate]" term=3 +2026-11-29T10:28:16.301Z [ERROR] server-1e7d22c6-0109-5302-e057-132632171018: failed to make requestVote RPC: target="{Voter server-0d56fb65-0575-9050-5780-b8ac900c0633 0d56fb65-0575-9050-5780-b8ac900c0633}" error="failed to connect to peer: 0d56fb65-0575-9050-5780-b8ac900c0633" term=3 +2026-11-29T10:28:16.301Z [INFO] server-9fec4e0b-da28-b197-8975-df29c2af1bea: duplicate requestVote for same term: term=3 +2026-11-29T10:28:16.329Z [WARN] server-0d56fb65-0575-9050-5780-b8ac900c0633: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:16.329Z [INFO] server-0d56fb65-0575-9050-5780-b8ac900c0633: entering candidate state: node="Node at 0d56fb65-0575-9050-5780-b8ac900c0633 [Candidate]" term=3 +2026-11-29T10:28:16.329Z [ERROR] server-0d56fb65-0575-9050-5780-b8ac900c0633: failed to make requestVote RPC: target="{Voter server-1e7d22c6-0109-5302-e057-132632171018 1e7d22c6-0109-5302-e057-132632171018}" error="failed to connect to peer: 1e7d22c6-0109-5302-e057-132632171018" term=3 +2026-11-29T10:28:16.330Z [ERROR] server-0d56fb65-0575-9050-5780-b8ac900c0633: failed to make requestVote RPC: target="{Voter server-9fec4e0b-da28-b197-8975-df29c2af1bea 9fec4e0b-da28-b197-8975-df29c2af1bea}" error="failed to connect to peer: 9fec4e0b-da28-b197-8975-df29c2af1bea" term=3 +2026-11-29T10:28:16.333Z [WARN] server-9fec4e0b-da28-b197-8975-df29c2af1bea: Election timeout reached, restarting election +2026-11-29T10:28:16.333Z [INFO] server-9fec4e0b-da28-b197-8975-df29c2af1bea: entering candidate state: node="Node at 9fec4e0b-da28-b197-8975-df29c2af1bea [Candidate]" term=4 +2026-11-29T10:28:16.333Z [ERROR] server-9fec4e0b-da28-b197-8975-df29c2af1bea: failed to make requestVote RPC: target="{Voter server-0d56fb65-0575-9050-5780-b8ac900c0633 0d56fb65-0575-9050-5780-b8ac900c0633}" error="failed to connect to peer: 0d56fb65-0575-9050-5780-b8ac900c0633" term=4 +2026-11-29T10:28:16.333Z [INFO] server-1e7d22c6-0109-5302-e057-132632171018: entering follower state: follower="Node at 1e7d22c6-0109-5302-e057-132632171018 [Follower]" leader-address= leader-id= +2026-11-29T10:28:16.333Z [INFO] server-9fec4e0b-da28-b197-8975-df29c2af1bea: election won: term=4 tally=2 +2026-11-29T10:28:16.333Z [INFO] server-9fec4e0b-da28-b197-8975-df29c2af1bea: entering leader state: leader="Node at 9fec4e0b-da28-b197-8975-df29c2af1bea [Leader]" +2026-11-29T10:28:16.333Z [INFO] server-9fec4e0b-da28-b197-8975-df29c2af1bea: added peer, starting replication: peer=server-1e7d22c6-0109-5302-e057-132632171018 +2026-11-29T10:28:16.333Z [INFO] server-9fec4e0b-da28-b197-8975-df29c2af1bea: added peer, starting replication: peer=server-0d56fb65-0575-9050-5780-b8ac900c0633 +2026-11-29T10:28:16.333Z [ERROR] server-9fec4e0b-da28-b197-8975-df29c2af1bea: failed to appendEntries to: peer="{Voter server-0d56fb65-0575-9050-5780-b8ac900c0633 0d56fb65-0575-9050-5780-b8ac900c0633}" error="failed to connect to peer: 0d56fb65-0575-9050-5780-b8ac900c0633" +2026-11-29T10:28:16.333Z [INFO] server-9fec4e0b-da28-b197-8975-df29c2af1bea: pipelining replication: peer="{Voter server-1e7d22c6-0109-5302-e057-132632171018 1e7d22c6-0109-5302-e057-132632171018}" +2026-11-29T10:28:16.342Z [ERROR] server-9fec4e0b-da28-b197-8975-df29c2af1bea: failed to heartbeat to: peer=0d56fb65-0575-9050-5780-b8ac900c0633 backoff time=10ms error="failed to connect to peer: 0d56fb65-0575-9050-5780-b8ac900c0633" +2026-11-29T10:28:16.353Z [ERROR] server-9fec4e0b-da28-b197-8975-df29c2af1bea: failed to appendEntries to: peer="{Voter server-0d56fb65-0575-9050-5780-b8ac900c0633 0d56fb65-0575-9050-5780-b8ac900c0633}" error="failed to connect to peer: 0d56fb65-0575-9050-5780-b8ac900c0633" +2026-11-29T10:28:16.360Z [ERROR] server-9fec4e0b-da28-b197-8975-df29c2af1bea: failed to heartbeat to: peer=0d56fb65-0575-9050-5780-b8ac900c0633 backoff time=10ms error="failed to connect to peer: 0d56fb65-0575-9050-5780-b8ac900c0633" +2026-11-29T10:28:16.374Z [ERROR] server-9fec4e0b-da28-b197-8975-df29c2af1bea: failed to appendEntries to: peer="{Voter server-0d56fb65-0575-9050-5780-b8ac900c0633 0d56fb65-0575-9050-5780-b8ac900c0633}" error="failed to connect to peer: 0d56fb65-0575-9050-5780-b8ac900c0633" +2026-11-29T10:28:16.377Z [ERROR] server-9fec4e0b-da28-b197-8975-df29c2af1bea: failed to heartbeat to: peer=0d56fb65-0575-9050-5780-b8ac900c0633 backoff time=10ms error="failed to connect to peer: 0d56fb65-0575-9050-5780-b8ac900c0633" +2026-11-29T10:28:16.384Z [WARN] server-9fec4e0b-da28-b197-8975-df29c2af1bea: failed to contact: server-id=server-0d56fb65-0575-9050-5780-b8ac900c0633 time=51.07172ms +2026-11-29T10:28:16.395Z [ERROR] server-9fec4e0b-da28-b197-8975-df29c2af1bea: failed to heartbeat to: peer=0d56fb65-0575-9050-5780-b8ac900c0633 backoff time=20ms error="failed to connect to peer: 0d56fb65-0575-9050-5780-b8ac900c0633" +2026-11-29T10:28:16.402Z [ERROR] server-9fec4e0b-da28-b197-8975-df29c2af1bea: failed to appendEntries to: peer="{Voter server-0d56fb65-0575-9050-5780-b8ac900c0633 0d56fb65-0575-9050-5780-b8ac900c0633}" error="failed to connect to peer: 0d56fb65-0575-9050-5780-b8ac900c0633" +2026-11-29T10:28:16.426Z [ERROR] server-9fec4e0b-da28-b197-8975-df29c2af1bea: failed to heartbeat to: peer=0d56fb65-0575-9050-5780-b8ac900c0633 backoff time=25ms error="failed to connect to peer: 0d56fb65-0575-9050-5780-b8ac900c0633" +2026-11-29T10:28:16.430Z [WARN] server-9fec4e0b-da28-b197-8975-df29c2af1bea: failed to contact: server-id=server-0d56fb65-0575-9050-5780-b8ac900c0633 time=97.27028ms +2026-11-29T10:28:16.430Z [WARN] server-0d56fb65-0575-9050-5780-b8ac900c0633: Election timeout reached, restarting election +2026-11-29T10:28:16.430Z [INFO] server-0d56fb65-0575-9050-5780-b8ac900c0633: entering candidate state: node="Node at 0d56fb65-0575-9050-5780-b8ac900c0633 [Candidate]" term=4 +2026-11-29T10:28:16.430Z [ERROR] server-0d56fb65-0575-9050-5780-b8ac900c0633: failed to make requestVote RPC: target="{Voter server-1e7d22c6-0109-5302-e057-132632171018 1e7d22c6-0109-5302-e057-132632171018}" error="failed to connect to peer: 1e7d22c6-0109-5302-e057-132632171018" term=4 +2026-11-29T10:28:16.430Z [ERROR] server-0d56fb65-0575-9050-5780-b8ac900c0633: failed to make requestVote RPC: target="{Voter server-9fec4e0b-da28-b197-8975-df29c2af1bea 9fec4e0b-da28-b197-8975-df29c2af1bea}" error="failed to connect to peer: 9fec4e0b-da28-b197-8975-df29c2af1bea" term=4 +2026-11-29T10:28:16.439Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:16.333445419 +0000 UTC m=+7.068499601 (1 nodes), 122.28948ms from start of poll, 413.2764ms from cluster start. Timeout at 2026-11-29 10:28:16.438449219 +0000 UTC m=+7.173503361, 105.00376ms after stability + raft_test.go:413: [INFO] Reconnecting Node at 0d56fb65-0575-9050-5780-b8ac900c0633 [Candidate] +2026-11-29T10:28:16.451Z [INFO] server-0d56fb65-0575-9050-5780-b8ac900c0633: entering follower state: follower="Node at 0d56fb65-0575-9050-5780-b8ac900c0633 [Follower]" leader-address=9fec4e0b-da28-b197-8975-df29c2af1bea leader-id=server-9fec4e0b-da28-b197-8975-df29c2af1bea +2026-11-29T10:28:16.451Z [INFO] server-9fec4e0b-da28-b197-8975-df29c2af1bea: pipelining replication: peer="{Voter server-0d56fb65-0575-9050-5780-b8ac900c0633 0d56fb65-0575-9050-5780-b8ac900c0633}" +2026-11-29T10:28:16.456Z [INFO] server-9fec4e0b-da28-b197-8975-df29c2af1bea: aborting pipeline replication: peer="{Voter server-1e7d22c6-0109-5302-e057-132632171018 1e7d22c6-0109-5302-e057-132632171018}" +2026-11-29T10:28:16.456Z [INFO] server-9fec4e0b-da28-b197-8975-df29c2af1bea: aborting pipeline replication: peer="{Voter server-0d56fb65-0575-9050-5780-b8ac900c0633 0d56fb65-0575-9050-5780-b8ac900c0633}" --- PASS: TestRaft_LeaderFail (0.54s) === RUN TestRaft_BehindFollower -2025-10-27T04:02:12.739Z [INFO] server-99337ba7-ef42-5bae-54d6-8eec098f1b0a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-99337ba7-ef42-5bae-54d6-8eec098f1b0a Address:99337ba7-ef42-5bae-54d6-8eec098f1b0a} {Suffrage:Voter ID:server-836b1778-ff10-5f1d-91ed-aa93b77c6406 Address:836b1778-ff10-5f1d-91ed-aa93b77c6406} {Suffrage:Voter ID:server-dae5e950-54cc-680c-3082-0e49e261d69e Address:dae5e950-54cc-680c-3082-0e49e261d69e}]" -2025-10-27T04:02:12.739Z [INFO] server-836b1778-ff10-5f1d-91ed-aa93b77c6406: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-99337ba7-ef42-5bae-54d6-8eec098f1b0a Address:99337ba7-ef42-5bae-54d6-8eec098f1b0a} {Suffrage:Voter ID:server-836b1778-ff10-5f1d-91ed-aa93b77c6406 Address:836b1778-ff10-5f1d-91ed-aa93b77c6406} {Suffrage:Voter ID:server-dae5e950-54cc-680c-3082-0e49e261d69e Address:dae5e950-54cc-680c-3082-0e49e261d69e}]" -2025-10-27T04:02:12.739Z [INFO] server-99337ba7-ef42-5bae-54d6-8eec098f1b0a: entering follower state: follower="Node at 99337ba7-ef42-5bae-54d6-8eec098f1b0a [Follower]" leader-address= leader-id= -2025-10-27T04:02:12.739Z [INFO] server-dae5e950-54cc-680c-3082-0e49e261d69e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-99337ba7-ef42-5bae-54d6-8eec098f1b0a Address:99337ba7-ef42-5bae-54d6-8eec098f1b0a} {Suffrage:Voter ID:server-836b1778-ff10-5f1d-91ed-aa93b77c6406 Address:836b1778-ff10-5f1d-91ed-aa93b77c6406} {Suffrage:Voter ID:server-dae5e950-54cc-680c-3082-0e49e261d69e Address:dae5e950-54cc-680c-3082-0e49e261d69e}]" -2025-10-27T04:02:12.739Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:12.739Z [INFO] server-dae5e950-54cc-680c-3082-0e49e261d69e: entering follower state: follower="Node at dae5e950-54cc-680c-3082-0e49e261d69e [Follower]" leader-address= leader-id= -2025-10-27T04:02:12.739Z [INFO] server-836b1778-ff10-5f1d-91ed-aa93b77c6406: entering follower state: follower="Node at 836b1778-ff10-5f1d-91ed-aa93b77c6406 [Follower]" leader-address= leader-id= -2025-10-27T04:02:12.809Z [WARN] server-99337ba7-ef42-5bae-54d6-8eec098f1b0a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:12.809Z [INFO] server-99337ba7-ef42-5bae-54d6-8eec098f1b0a: entering candidate state: node="Node at 99337ba7-ef42-5bae-54d6-8eec098f1b0a [Candidate]" term=2 -2025-10-27T04:02:12.809Z [INFO] server-99337ba7-ef42-5bae-54d6-8eec098f1b0a: election won: term=2 tally=2 -2025-10-27T04:02:12.809Z [INFO] server-99337ba7-ef42-5bae-54d6-8eec098f1b0a: entering leader state: leader="Node at 99337ba7-ef42-5bae-54d6-8eec098f1b0a [Leader]" -2025-10-27T04:02:12.809Z [INFO] server-99337ba7-ef42-5bae-54d6-8eec098f1b0a: added peer, starting replication: peer=server-836b1778-ff10-5f1d-91ed-aa93b77c6406 -2025-10-27T04:02:12.809Z [INFO] server-99337ba7-ef42-5bae-54d6-8eec098f1b0a: added peer, starting replication: peer=server-dae5e950-54cc-680c-3082-0e49e261d69e -2025-10-27T04:02:12.809Z [INFO] server-99337ba7-ef42-5bae-54d6-8eec098f1b0a: pipelining replication: peer="{Voter server-dae5e950-54cc-680c-3082-0e49e261d69e dae5e950-54cc-680c-3082-0e49e261d69e}" -2025-10-27T04:02:12.809Z [INFO] server-99337ba7-ef42-5bae-54d6-8eec098f1b0a: pipelining replication: peer="{Voter server-836b1778-ff10-5f1d-91ed-aa93b77c6406 836b1778-ff10-5f1d-91ed-aa93b77c6406}" -2025-10-27T04:02:12.915Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:12.809289466 +0000 UTC m=+7.804161904 (1 nodes), 69.333354ms from start of poll, 69.529955ms from cluster start. Timeout at 2025-10-27 04:02:12.914292546 +0000 UTC m=+7.909165064, 105.00316ms after stability -2025-10-27T04:02:12.915Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:02:13.021Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:02:12.91523883 +0000 UTC m=+7.910111308 (2 nodes), 1.48µs from start of poll, 175.479359ms from cluster start. Timeout at 2025-10-27 04:02:13.02024267 +0000 UTC m=+8.015115108, 105.0038ms after stability -2025-10-27T04:02:13.021Z [ERROR] server-99337ba7-ef42-5bae-54d6-8eec098f1b0a: failed to pipeline appendEntries: peer="{Voter server-836b1778-ff10-5f1d-91ed-aa93b77c6406 836b1778-ff10-5f1d-91ed-aa93b77c6406}" error="append pipeline closed" -2025-10-27T04:02:13.021Z [INFO] server-99337ba7-ef42-5bae-54d6-8eec098f1b0a: aborting pipeline replication: peer="{Voter server-836b1778-ff10-5f1d-91ed-aa93b77c6406 836b1778-ff10-5f1d-91ed-aa93b77c6406}" -2025-10-27T04:02:13.021Z [ERROR] server-99337ba7-ef42-5bae-54d6-8eec098f1b0a: failed to appendEntries to: peer="{Voter server-836b1778-ff10-5f1d-91ed-aa93b77c6406 836b1778-ff10-5f1d-91ed-aa93b77c6406}" error="failed to connect to peer: 836b1778-ff10-5f1d-91ed-aa93b77c6406" -2025-10-27T04:02:13.023Z [ERROR] server-99337ba7-ef42-5bae-54d6-8eec098f1b0a: failed to heartbeat to: peer=836b1778-ff10-5f1d-91ed-aa93b77c6406 backoff time=10ms error="failed to connect to peer: 836b1778-ff10-5f1d-91ed-aa93b77c6406" +2026-11-29T10:28:16.457Z [INFO] server-704b4ba5-2f19-ecd8-15b0-add4fac8748c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-704b4ba5-2f19-ecd8-15b0-add4fac8748c Address:704b4ba5-2f19-ecd8-15b0-add4fac8748c} {Suffrage:Voter ID:server-5aa09eb9-c32e-b3ce-3a10-97b4510aa2f7 Address:5aa09eb9-c32e-b3ce-3a10-97b4510aa2f7} {Suffrage:Voter ID:server-7ff319d9-3cb7-234a-1d4d-042c770948d6 Address:7ff319d9-3cb7-234a-1d4d-042c770948d6}]" +2026-11-29T10:28:16.457Z [INFO] server-704b4ba5-2f19-ecd8-15b0-add4fac8748c: entering follower state: follower="Node at 704b4ba5-2f19-ecd8-15b0-add4fac8748c [Follower]" leader-address= leader-id= +2026-11-29T10:28:16.457Z [INFO] server-5aa09eb9-c32e-b3ce-3a10-97b4510aa2f7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-704b4ba5-2f19-ecd8-15b0-add4fac8748c Address:704b4ba5-2f19-ecd8-15b0-add4fac8748c} {Suffrage:Voter ID:server-5aa09eb9-c32e-b3ce-3a10-97b4510aa2f7 Address:5aa09eb9-c32e-b3ce-3a10-97b4510aa2f7} {Suffrage:Voter ID:server-7ff319d9-3cb7-234a-1d4d-042c770948d6 Address:7ff319d9-3cb7-234a-1d4d-042c770948d6}]" +2026-11-29T10:28:16.457Z [INFO] server-5aa09eb9-c32e-b3ce-3a10-97b4510aa2f7: entering follower state: follower="Node at 5aa09eb9-c32e-b3ce-3a10-97b4510aa2f7 [Follower]" leader-address= leader-id= +2026-11-29T10:28:16.458Z [INFO] server-7ff319d9-3cb7-234a-1d4d-042c770948d6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-704b4ba5-2f19-ecd8-15b0-add4fac8748c Address:704b4ba5-2f19-ecd8-15b0-add4fac8748c} {Suffrage:Voter ID:server-5aa09eb9-c32e-b3ce-3a10-97b4510aa2f7 Address:5aa09eb9-c32e-b3ce-3a10-97b4510aa2f7} {Suffrage:Voter ID:server-7ff319d9-3cb7-234a-1d4d-042c770948d6 Address:7ff319d9-3cb7-234a-1d4d-042c770948d6}]" +2026-11-29T10:28:16.458Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:16.458Z [INFO] server-7ff319d9-3cb7-234a-1d4d-042c770948d6: entering follower state: follower="Node at 7ff319d9-3cb7-234a-1d4d-042c770948d6 [Follower]" leader-address= leader-id= +2026-11-29T10:28:16.519Z [WARN] server-5aa09eb9-c32e-b3ce-3a10-97b4510aa2f7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:16.519Z [INFO] server-5aa09eb9-c32e-b3ce-3a10-97b4510aa2f7: entering candidate state: node="Node at 5aa09eb9-c32e-b3ce-3a10-97b4510aa2f7 [Candidate]" term=2 +2026-11-29T10:28:16.519Z [INFO] server-5aa09eb9-c32e-b3ce-3a10-97b4510aa2f7: election won: term=2 tally=2 +2026-11-29T10:28:16.519Z [INFO] server-5aa09eb9-c32e-b3ce-3a10-97b4510aa2f7: entering leader state: leader="Node at 5aa09eb9-c32e-b3ce-3a10-97b4510aa2f7 [Leader]" +2026-11-29T10:28:16.519Z [INFO] server-5aa09eb9-c32e-b3ce-3a10-97b4510aa2f7: added peer, starting replication: peer=server-704b4ba5-2f19-ecd8-15b0-add4fac8748c +2026-11-29T10:28:16.519Z [INFO] server-5aa09eb9-c32e-b3ce-3a10-97b4510aa2f7: added peer, starting replication: peer=server-7ff319d9-3cb7-234a-1d4d-042c770948d6 +2026-11-29T10:28:16.520Z [INFO] server-5aa09eb9-c32e-b3ce-3a10-97b4510aa2f7: pipelining replication: peer="{Voter server-704b4ba5-2f19-ecd8-15b0-add4fac8748c 704b4ba5-2f19-ecd8-15b0-add4fac8748c}" +2026-11-29T10:28:16.520Z [INFO] server-5aa09eb9-c32e-b3ce-3a10-97b4510aa2f7: pipelining replication: peer="{Voter server-7ff319d9-3cb7-234a-1d4d-042c770948d6 7ff319d9-3cb7-234a-1d4d-042c770948d6}" +2026-11-29T10:28:16.625Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:16.519909139 +0000 UTC m=+7.254963281 (1 nodes), 61.77088ms from start of poll, 62.2912ms from cluster start. Timeout at 2026-11-29 10:28:16.624912459 +0000 UTC m=+7.359966601, 105.00332ms after stability +2026-11-29T10:28:16.625Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:28:16.730Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:28:16.625138939 +0000 UTC m=+7.360193081 (2 nodes), 1.36µs from start of poll, 167.521ms from cluster start. Timeout at 2026-11-29 10:28:16.730147739 +0000 UTC m=+7.465201921, 105.00884ms after stability +2026-11-29T10:28:16.730Z [ERROR] server-5aa09eb9-c32e-b3ce-3a10-97b4510aa2f7: failed to pipeline appendEntries: peer="{Voter server-704b4ba5-2f19-ecd8-15b0-add4fac8748c 704b4ba5-2f19-ecd8-15b0-add4fac8748c}" error="append pipeline closed" +2026-11-29T10:28:16.730Z [INFO] server-5aa09eb9-c32e-b3ce-3a10-97b4510aa2f7: aborting pipeline replication: peer="{Voter server-704b4ba5-2f19-ecd8-15b0-add4fac8748c 704b4ba5-2f19-ecd8-15b0-add4fac8748c}" +2026-11-29T10:28:16.730Z [ERROR] server-5aa09eb9-c32e-b3ce-3a10-97b4510aa2f7: failed to appendEntries to: peer="{Voter server-704b4ba5-2f19-ecd8-15b0-add4fac8748c 704b4ba5-2f19-ecd8-15b0-add4fac8748c}" error="failed to connect to peer: 704b4ba5-2f19-ecd8-15b0-add4fac8748c" +2026-11-29T10:28:16.732Z [ERROR] server-5aa09eb9-c32e-b3ce-3a10-97b4510aa2f7: failed to heartbeat to: peer=704b4ba5-2f19-ecd8-15b0-add4fac8748c backoff time=10ms error="failed to connect to peer: 704b4ba5-2f19-ecd8-15b0-add4fac8748c" raft_test.go:459: [INFO] Finished apply without behind follower -2025-10-27T04:02:13.031Z [INFO] server-99337ba7-ef42-5bae-54d6-8eec098f1b0a: pipelining replication: peer="{Voter server-836b1778-ff10-5f1d-91ed-aa93b77c6406 836b1778-ff10-5f1d-91ed-aa93b77c6406}" -2025-10-27T04:02:13.041Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:13.147Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:13.041833335 +0000 UTC m=+8.036705773 (1 nodes), 1.2µs from start of poll, 302.073824ms from cluster start. Timeout at 2025-10-27 04:02:13.146841215 +0000 UTC m=+8.141713693, 105.00792ms after stability -2025-10-27T04:02:13.147Z [INFO] server-99337ba7-ef42-5bae-54d6-8eec098f1b0a: aborting pipeline replication: peer="{Voter server-dae5e950-54cc-680c-3082-0e49e261d69e dae5e950-54cc-680c-3082-0e49e261d69e}" -2025-10-27T04:02:13.147Z [INFO] server-99337ba7-ef42-5bae-54d6-8eec098f1b0a: aborting pipeline replication: peer="{Voter server-836b1778-ff10-5f1d-91ed-aa93b77c6406 836b1778-ff10-5f1d-91ed-aa93b77c6406}" ---- PASS: TestRaft_BehindFollower (0.41s) +2026-11-29T10:28:16.741Z [INFO] server-5aa09eb9-c32e-b3ce-3a10-97b4510aa2f7: pipelining replication: peer="{Voter server-704b4ba5-2f19-ecd8-15b0-add4fac8748c 704b4ba5-2f19-ecd8-15b0-add4fac8748c}" +2026-11-29T10:28:16.750Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:16.856Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:16.750854819 +0000 UTC m=+7.485908961 (1 nodes), 1.28µs from start of poll, 293.23688ms from cluster start. Timeout at 2026-11-29 10:28:16.855860139 +0000 UTC m=+7.590914281, 105.00532ms after stability +2026-11-29T10:28:16.856Z [INFO] server-5aa09eb9-c32e-b3ce-3a10-97b4510aa2f7: aborting pipeline replication: peer="{Voter server-7ff319d9-3cb7-234a-1d4d-042c770948d6 7ff319d9-3cb7-234a-1d4d-042c770948d6}" +2026-11-29T10:28:16.856Z [INFO] server-5aa09eb9-c32e-b3ce-3a10-97b4510aa2f7: aborting pipeline replication: peer="{Voter server-704b4ba5-2f19-ecd8-15b0-add4fac8748c 704b4ba5-2f19-ecd8-15b0-add4fac8748c}" +--- PASS: TestRaft_BehindFollower (0.40s) === RUN TestRaft_ApplyNonLeader -2025-10-27T04:02:13.148Z [INFO] server-f02c5dcd-3747-62ab-1eb6-2d781491f0a9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f02c5dcd-3747-62ab-1eb6-2d781491f0a9 Address:f02c5dcd-3747-62ab-1eb6-2d781491f0a9} {Suffrage:Voter ID:server-05e372db-37fb-5fbf-147f-f02a656533aa Address:05e372db-37fb-5fbf-147f-f02a656533aa} {Suffrage:Voter ID:server-7e2926f4-6b7d-897a-3138-0ede53320888 Address:7e2926f4-6b7d-897a-3138-0ede53320888}]" -2025-10-27T04:02:13.148Z [INFO] server-05e372db-37fb-5fbf-147f-f02a656533aa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f02c5dcd-3747-62ab-1eb6-2d781491f0a9 Address:f02c5dcd-3747-62ab-1eb6-2d781491f0a9} {Suffrage:Voter ID:server-05e372db-37fb-5fbf-147f-f02a656533aa Address:05e372db-37fb-5fbf-147f-f02a656533aa} {Suffrage:Voter ID:server-7e2926f4-6b7d-897a-3138-0ede53320888 Address:7e2926f4-6b7d-897a-3138-0ede53320888}]" -2025-10-27T04:02:13.148Z [INFO] server-7e2926f4-6b7d-897a-3138-0ede53320888: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f02c5dcd-3747-62ab-1eb6-2d781491f0a9 Address:f02c5dcd-3747-62ab-1eb6-2d781491f0a9} {Suffrage:Voter ID:server-05e372db-37fb-5fbf-147f-f02a656533aa Address:05e372db-37fb-5fbf-147f-f02a656533aa} {Suffrage:Voter ID:server-7e2926f4-6b7d-897a-3138-0ede53320888 Address:7e2926f4-6b7d-897a-3138-0ede53320888}]" -2025-10-27T04:02:13.148Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:13.149Z [INFO] server-f02c5dcd-3747-62ab-1eb6-2d781491f0a9: entering follower state: follower="Node at f02c5dcd-3747-62ab-1eb6-2d781491f0a9 [Follower]" leader-address= leader-id= -2025-10-27T04:02:13.149Z [INFO] server-05e372db-37fb-5fbf-147f-f02a656533aa: entering follower state: follower="Node at 05e372db-37fb-5fbf-147f-f02a656533aa [Follower]" leader-address= leader-id= -2025-10-27T04:02:13.149Z [INFO] server-7e2926f4-6b7d-897a-3138-0ede53320888: entering follower state: follower="Node at 7e2926f4-6b7d-897a-3138-0ede53320888 [Follower]" leader-address= leader-id= -2025-10-27T04:02:13.209Z [WARN] server-7e2926f4-6b7d-897a-3138-0ede53320888: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:13.209Z [INFO] server-7e2926f4-6b7d-897a-3138-0ede53320888: entering candidate state: node="Node at 7e2926f4-6b7d-897a-3138-0ede53320888 [Candidate]" term=2 -2025-10-27T04:02:13.209Z [INFO] server-7e2926f4-6b7d-897a-3138-0ede53320888: election won: term=2 tally=2 -2025-10-27T04:02:13.209Z [INFO] server-7e2926f4-6b7d-897a-3138-0ede53320888: entering leader state: leader="Node at 7e2926f4-6b7d-897a-3138-0ede53320888 [Leader]" -2025-10-27T04:02:13.209Z [INFO] server-7e2926f4-6b7d-897a-3138-0ede53320888: added peer, starting replication: peer=server-f02c5dcd-3747-62ab-1eb6-2d781491f0a9 -2025-10-27T04:02:13.209Z [INFO] server-7e2926f4-6b7d-897a-3138-0ede53320888: added peer, starting replication: peer=server-05e372db-37fb-5fbf-147f-f02a656533aa -2025-10-27T04:02:13.209Z [INFO] server-7e2926f4-6b7d-897a-3138-0ede53320888: pipelining replication: peer="{Voter server-05e372db-37fb-5fbf-147f-f02a656533aa 05e372db-37fb-5fbf-147f-f02a656533aa}" -2025-10-27T04:02:13.209Z [INFO] server-7e2926f4-6b7d-897a-3138-0ede53320888: pipelining replication: peer="{Voter server-f02c5dcd-3747-62ab-1eb6-2d781491f0a9 f02c5dcd-3747-62ab-1eb6-2d781491f0a9}" -2025-10-27T04:02:13.315Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:13.209720803 +0000 UTC m=+8.204593201 (1 nodes), 60.707775ms from start of poll, 61.111018ms from cluster start. Timeout at 2025-10-27 04:02:13.314723203 +0000 UTC m=+8.309595641, 105.00244ms after stability -2025-10-27T04:02:13.315Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:02:13.421Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:02:13.315300204 +0000 UTC m=+8.310172642 (2 nodes), 920ns from start of poll, 166.690459ms from cluster start. Timeout at 2025-10-27 04:02:13.420308524 +0000 UTC m=+8.415181002, 105.00836ms after stability -2025-10-27T04:02:13.421Z [INFO] server-7e2926f4-6b7d-897a-3138-0ede53320888: aborting pipeline replication: peer="{Voter server-05e372db-37fb-5fbf-147f-f02a656533aa 05e372db-37fb-5fbf-147f-f02a656533aa}" -2025-10-27T04:02:13.421Z [INFO] server-7e2926f4-6b7d-897a-3138-0ede53320888: aborting pipeline replication: peer="{Voter server-f02c5dcd-3747-62ab-1eb6-2d781491f0a9 f02c5dcd-3747-62ab-1eb6-2d781491f0a9}" +2026-11-29T10:28:16.857Z [INFO] server-85ba3159-d1c1-709e-7c05-26b87430a51e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-85ba3159-d1c1-709e-7c05-26b87430a51e Address:85ba3159-d1c1-709e-7c05-26b87430a51e} {Suffrage:Voter ID:server-6b62a8e3-f973-e4b5-2b2d-b81127c48e73 Address:6b62a8e3-f973-e4b5-2b2d-b81127c48e73} {Suffrage:Voter ID:server-5129c6ab-abc9-5b0c-aa3c-426a8f32ba45 Address:5129c6ab-abc9-5b0c-aa3c-426a8f32ba45}]" +2026-11-29T10:28:16.857Z [INFO] server-6b62a8e3-f973-e4b5-2b2d-b81127c48e73: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-85ba3159-d1c1-709e-7c05-26b87430a51e Address:85ba3159-d1c1-709e-7c05-26b87430a51e} {Suffrage:Voter ID:server-6b62a8e3-f973-e4b5-2b2d-b81127c48e73 Address:6b62a8e3-f973-e4b5-2b2d-b81127c48e73} {Suffrage:Voter ID:server-5129c6ab-abc9-5b0c-aa3c-426a8f32ba45 Address:5129c6ab-abc9-5b0c-aa3c-426a8f32ba45}]" +2026-11-29T10:28:16.857Z [INFO] server-85ba3159-d1c1-709e-7c05-26b87430a51e: entering follower state: follower="Node at 85ba3159-d1c1-709e-7c05-26b87430a51e [Follower]" leader-address= leader-id= +2026-11-29T10:28:16.857Z [INFO] server-6b62a8e3-f973-e4b5-2b2d-b81127c48e73: entering follower state: follower="Node at 6b62a8e3-f973-e4b5-2b2d-b81127c48e73 [Follower]" leader-address= leader-id= +2026-11-29T10:28:16.857Z [INFO] server-5129c6ab-abc9-5b0c-aa3c-426a8f32ba45: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-85ba3159-d1c1-709e-7c05-26b87430a51e Address:85ba3159-d1c1-709e-7c05-26b87430a51e} {Suffrage:Voter ID:server-6b62a8e3-f973-e4b5-2b2d-b81127c48e73 Address:6b62a8e3-f973-e4b5-2b2d-b81127c48e73} {Suffrage:Voter ID:server-5129c6ab-abc9-5b0c-aa3c-426a8f32ba45 Address:5129c6ab-abc9-5b0c-aa3c-426a8f32ba45}]" +2026-11-29T10:28:16.857Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:16.857Z [INFO] server-5129c6ab-abc9-5b0c-aa3c-426a8f32ba45: entering follower state: follower="Node at 5129c6ab-abc9-5b0c-aa3c-426a8f32ba45 [Follower]" leader-address= leader-id= +2026-11-29T10:28:16.917Z [WARN] server-6b62a8e3-f973-e4b5-2b2d-b81127c48e73: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:16.917Z [INFO] server-6b62a8e3-f973-e4b5-2b2d-b81127c48e73: entering candidate state: node="Node at 6b62a8e3-f973-e4b5-2b2d-b81127c48e73 [Candidate]" term=2 +2026-11-29T10:28:16.917Z [INFO] server-6b62a8e3-f973-e4b5-2b2d-b81127c48e73: election won: term=2 tally=2 +2026-11-29T10:28:16.917Z [INFO] server-6b62a8e3-f973-e4b5-2b2d-b81127c48e73: entering leader state: leader="Node at 6b62a8e3-f973-e4b5-2b2d-b81127c48e73 [Leader]" +2026-11-29T10:28:16.917Z [INFO] server-6b62a8e3-f973-e4b5-2b2d-b81127c48e73: added peer, starting replication: peer=server-85ba3159-d1c1-709e-7c05-26b87430a51e +2026-11-29T10:28:16.917Z [INFO] server-6b62a8e3-f973-e4b5-2b2d-b81127c48e73: added peer, starting replication: peer=server-5129c6ab-abc9-5b0c-aa3c-426a8f32ba45 +2026-11-29T10:28:16.917Z [INFO] server-6b62a8e3-f973-e4b5-2b2d-b81127c48e73: pipelining replication: peer="{Voter server-5129c6ab-abc9-5b0c-aa3c-426a8f32ba45 5129c6ab-abc9-5b0c-aa3c-426a8f32ba45}" +2026-11-29T10:28:16.917Z [INFO] server-6b62a8e3-f973-e4b5-2b2d-b81127c48e73: pipelining replication: peer="{Voter server-85ba3159-d1c1-709e-7c05-26b87430a51e 85ba3159-d1c1-709e-7c05-26b87430a51e}" +2026-11-29T10:28:17.022Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:16.917622299 +0000 UTC m=+7.652676441 (1 nodes), 59.662ms from start of poll, 59.90884ms from cluster start. Timeout at 2026-11-29 10:28:17.022629059 +0000 UTC m=+7.757685521, 105.00908ms after stability +2026-11-29T10:28:17.022Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:28:17.127Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:28:17.022786859 +0000 UTC m=+7.757841001 (2 nodes), 1.4µs from start of poll, 165.0734ms from cluster start. Timeout at 2026-11-29 10:28:17.127794339 +0000 UTC m=+7.862848481, 105.00748ms after stability +2026-11-29T10:28:17.127Z [INFO] server-6b62a8e3-f973-e4b5-2b2d-b81127c48e73: aborting pipeline replication: peer="{Voter server-5129c6ab-abc9-5b0c-aa3c-426a8f32ba45 5129c6ab-abc9-5b0c-aa3c-426a8f32ba45}" +2026-11-29T10:28:17.127Z [INFO] server-6b62a8e3-f973-e4b5-2b2d-b81127c48e73: aborting pipeline replication: peer="{Voter server-85ba3159-d1c1-709e-7c05-26b87430a51e 85ba3159-d1c1-709e-7c05-26b87430a51e}" --- PASS: TestRaft_ApplyNonLeader (0.27s) === RUN TestRaft_ApplyConcurrent -2025-10-27T04:02:13.422Z [INFO] server-8a29d036-da82-88a5-bfd6-e15f3bdfb049: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8a29d036-da82-88a5-bfd6-e15f3bdfb049 Address:8a29d036-da82-88a5-bfd6-e15f3bdfb049} {Suffrage:Voter ID:server-763117e9-f97b-10a5-0fd5-91dd72498e29 Address:763117e9-f97b-10a5-0fd5-91dd72498e29} {Suffrage:Voter ID:server-0db84aca-b9ca-973e-478c-31c1fa9f9045 Address:0db84aca-b9ca-973e-478c-31c1fa9f9045}]" -2025-10-27T04:02:13.422Z [INFO] server-763117e9-f97b-10a5-0fd5-91dd72498e29: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8a29d036-da82-88a5-bfd6-e15f3bdfb049 Address:8a29d036-da82-88a5-bfd6-e15f3bdfb049} {Suffrage:Voter ID:server-763117e9-f97b-10a5-0fd5-91dd72498e29 Address:763117e9-f97b-10a5-0fd5-91dd72498e29} {Suffrage:Voter ID:server-0db84aca-b9ca-973e-478c-31c1fa9f9045 Address:0db84aca-b9ca-973e-478c-31c1fa9f9045}]" -2025-10-27T04:02:13.422Z [INFO] server-0db84aca-b9ca-973e-478c-31c1fa9f9045: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8a29d036-da82-88a5-bfd6-e15f3bdfb049 Address:8a29d036-da82-88a5-bfd6-e15f3bdfb049} {Suffrage:Voter ID:server-763117e9-f97b-10a5-0fd5-91dd72498e29 Address:763117e9-f97b-10a5-0fd5-91dd72498e29} {Suffrage:Voter ID:server-0db84aca-b9ca-973e-478c-31c1fa9f9045 Address:0db84aca-b9ca-973e-478c-31c1fa9f9045}]" -2025-10-27T04:02:13.422Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:13.422Z [INFO] server-8a29d036-da82-88a5-bfd6-e15f3bdfb049: entering follower state: follower="Node at 8a29d036-da82-88a5-bfd6-e15f3bdfb049 [Follower]" leader-address= leader-id= -2025-10-27T04:02:13.422Z [INFO] server-763117e9-f97b-10a5-0fd5-91dd72498e29: entering follower state: follower="Node at 763117e9-f97b-10a5-0fd5-91dd72498e29 [Follower]" leader-address= leader-id= -2025-10-27T04:02:13.422Z [INFO] server-0db84aca-b9ca-973e-478c-31c1fa9f9045: entering follower state: follower="Node at 0db84aca-b9ca-973e-478c-31c1fa9f9045 [Follower]" leader-address= leader-id= -2025-10-27T04:02:13.541Z [WARN] server-8a29d036-da82-88a5-bfd6-e15f3bdfb049: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:13.541Z [INFO] server-8a29d036-da82-88a5-bfd6-e15f3bdfb049: entering candidate state: node="Node at 8a29d036-da82-88a5-bfd6-e15f3bdfb049 [Candidate]" term=2 -2025-10-27T04:02:13.541Z [INFO] server-8a29d036-da82-88a5-bfd6-e15f3bdfb049: election won: term=2 tally=2 -2025-10-27T04:02:13.541Z [INFO] server-8a29d036-da82-88a5-bfd6-e15f3bdfb049: entering leader state: leader="Node at 8a29d036-da82-88a5-bfd6-e15f3bdfb049 [Leader]" -2025-10-27T04:02:13.541Z [INFO] server-8a29d036-da82-88a5-bfd6-e15f3bdfb049: added peer, starting replication: peer=server-763117e9-f97b-10a5-0fd5-91dd72498e29 -2025-10-27T04:02:13.541Z [INFO] server-8a29d036-da82-88a5-bfd6-e15f3bdfb049: added peer, starting replication: peer=server-0db84aca-b9ca-973e-478c-31c1fa9f9045 -2025-10-27T04:02:13.541Z [INFO] server-8a29d036-da82-88a5-bfd6-e15f3bdfb049: pipelining replication: peer="{Voter server-0db84aca-b9ca-973e-478c-31c1fa9f9045 0db84aca-b9ca-973e-478c-31c1fa9f9045}" -2025-10-27T04:02:13.541Z [INFO] server-8a29d036-da82-88a5-bfd6-e15f3bdfb049: pipelining replication: peer="{Voter server-763117e9-f97b-10a5-0fd5-91dd72498e29 763117e9-f97b-10a5-0fd5-91dd72498e29}" -2025-10-27T04:02:13.747Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:13.541830513 +0000 UTC m=+8.536702911 (1 nodes), 119.243135ms from start of poll, 119.500457ms from cluster start. Timeout at 2025-10-27 04:02:13.746834433 +0000 UTC m=+8.741706871, 205.00396ms after stability -2025-10-27T04:02:13.760Z [INFO] server-8a29d036-da82-88a5-bfd6-e15f3bdfb049: aborting pipeline replication: peer="{Voter server-763117e9-f97b-10a5-0fd5-91dd72498e29 763117e9-f97b-10a5-0fd5-91dd72498e29}" -2025-10-27T04:02:13.760Z [INFO] server-8a29d036-da82-88a5-bfd6-e15f3bdfb049: aborting pipeline replication: peer="{Voter server-0db84aca-b9ca-973e-478c-31c1fa9f9045 0db84aca-b9ca-973e-478c-31c1fa9f9045}" ---- PASS: TestRaft_ApplyConcurrent (0.34s) +2026-11-29T10:28:17.128Z [INFO] server-445ace8a-ee8b-43e7-6ca3-e3fc074f95bf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-445ace8a-ee8b-43e7-6ca3-e3fc074f95bf Address:445ace8a-ee8b-43e7-6ca3-e3fc074f95bf} {Suffrage:Voter ID:server-6c59e669-1ca1-0393-e6dd-3cafa82163bc Address:6c59e669-1ca1-0393-e6dd-3cafa82163bc} {Suffrage:Voter ID:server-8f854b7c-6f87-d8f1-70af-0bedde739c3b Address:8f854b7c-6f87-d8f1-70af-0bedde739c3b}]" +2026-11-29T10:28:17.128Z [INFO] server-6c59e669-1ca1-0393-e6dd-3cafa82163bc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-445ace8a-ee8b-43e7-6ca3-e3fc074f95bf Address:445ace8a-ee8b-43e7-6ca3-e3fc074f95bf} {Suffrage:Voter ID:server-6c59e669-1ca1-0393-e6dd-3cafa82163bc Address:6c59e669-1ca1-0393-e6dd-3cafa82163bc} {Suffrage:Voter ID:server-8f854b7c-6f87-d8f1-70af-0bedde739c3b Address:8f854b7c-6f87-d8f1-70af-0bedde739c3b}]" +2026-11-29T10:28:17.128Z [INFO] server-445ace8a-ee8b-43e7-6ca3-e3fc074f95bf: entering follower state: follower="Node at 445ace8a-ee8b-43e7-6ca3-e3fc074f95bf [Follower]" leader-address= leader-id= +2026-11-29T10:28:17.128Z [INFO] server-8f854b7c-6f87-d8f1-70af-0bedde739c3b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-445ace8a-ee8b-43e7-6ca3-e3fc074f95bf Address:445ace8a-ee8b-43e7-6ca3-e3fc074f95bf} {Suffrage:Voter ID:server-6c59e669-1ca1-0393-e6dd-3cafa82163bc Address:6c59e669-1ca1-0393-e6dd-3cafa82163bc} {Suffrage:Voter ID:server-8f854b7c-6f87-d8f1-70af-0bedde739c3b Address:8f854b7c-6f87-d8f1-70af-0bedde739c3b}]" +2026-11-29T10:28:17.128Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:17.128Z [INFO] server-8f854b7c-6f87-d8f1-70af-0bedde739c3b: entering follower state: follower="Node at 8f854b7c-6f87-d8f1-70af-0bedde739c3b [Follower]" leader-address= leader-id= +2026-11-29T10:28:17.128Z [INFO] server-6c59e669-1ca1-0393-e6dd-3cafa82163bc: entering follower state: follower="Node at 6c59e669-1ca1-0393-e6dd-3cafa82163bc [Follower]" leader-address= leader-id= +2026-11-29T10:28:17.240Z [WARN] server-445ace8a-ee8b-43e7-6ca3-e3fc074f95bf: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:17.240Z [INFO] server-445ace8a-ee8b-43e7-6ca3-e3fc074f95bf: entering candidate state: node="Node at 445ace8a-ee8b-43e7-6ca3-e3fc074f95bf [Candidate]" term=2 +2026-11-29T10:28:17.240Z [INFO] server-445ace8a-ee8b-43e7-6ca3-e3fc074f95bf: election won: term=2 tally=2 +2026-11-29T10:28:17.240Z [INFO] server-445ace8a-ee8b-43e7-6ca3-e3fc074f95bf: entering leader state: leader="Node at 445ace8a-ee8b-43e7-6ca3-e3fc074f95bf [Leader]" +2026-11-29T10:28:17.240Z [INFO] server-445ace8a-ee8b-43e7-6ca3-e3fc074f95bf: added peer, starting replication: peer=server-6c59e669-1ca1-0393-e6dd-3cafa82163bc +2026-11-29T10:28:17.240Z [INFO] server-445ace8a-ee8b-43e7-6ca3-e3fc074f95bf: added peer, starting replication: peer=server-8f854b7c-6f87-d8f1-70af-0bedde739c3b +2026-11-29T10:28:17.240Z [INFO] server-445ace8a-ee8b-43e7-6ca3-e3fc074f95bf: pipelining replication: peer="{Voter server-8f854b7c-6f87-d8f1-70af-0bedde739c3b 8f854b7c-6f87-d8f1-70af-0bedde739c3b}" +2026-11-29T10:28:17.240Z [INFO] server-445ace8a-ee8b-43e7-6ca3-e3fc074f95bf: pipelining replication: peer="{Voter server-6c59e669-1ca1-0393-e6dd-3cafa82163bc 6c59e669-1ca1-0393-e6dd-3cafa82163bc}" +2026-11-29T10:28:17.445Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:17.240354299 +0000 UTC m=+7.975408441 (1 nodes), 111.54572ms from start of poll, 111.79804ms from cluster start. Timeout at 2026-11-29 10:28:17.445364899 +0000 UTC m=+8.180419041, 205.0106ms after stability +2026-11-29T10:28:17.462Z [INFO] server-445ace8a-ee8b-43e7-6ca3-e3fc074f95bf: aborting pipeline replication: peer="{Voter server-8f854b7c-6f87-d8f1-70af-0bedde739c3b 8f854b7c-6f87-d8f1-70af-0bedde739c3b}" +2026-11-29T10:28:17.462Z [INFO] server-445ace8a-ee8b-43e7-6ca3-e3fc074f95bf: aborting pipeline replication: peer="{Voter server-6c59e669-1ca1-0393-e6dd-3cafa82163bc 6c59e669-1ca1-0393-e6dd-3cafa82163bc}" +--- PASS: TestRaft_ApplyConcurrent (0.33s) === RUN TestRaft_ApplyConcurrent_Timeout -2025-10-27T04:02:13.761Z [INFO] server-775c0a83-90a5-51c5-1fe6-8ca2c1b81aed: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-775c0a83-90a5-51c5-1fe6-8ca2c1b81aed Address:775c0a83-90a5-51c5-1fe6-8ca2c1b81aed}]" -2025-10-27T04:02:13.761Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:13.761Z [INFO] server-775c0a83-90a5-51c5-1fe6-8ca2c1b81aed: entering follower state: follower="Node at 775c0a83-90a5-51c5-1fe6-8ca2c1b81aed [Follower]" leader-address= leader-id= -2025-10-27T04:02:13.943Z [WARN] server-775c0a83-90a5-51c5-1fe6-8ca2c1b81aed: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:13.943Z [INFO] server-775c0a83-90a5-51c5-1fe6-8ca2c1b81aed: entering candidate state: node="Node at 775c0a83-90a5-51c5-1fe6-8ca2c1b81aed [Candidate]" term=2 -2025-10-27T04:02:13.943Z [INFO] server-775c0a83-90a5-51c5-1fe6-8ca2c1b81aed: election won: term=2 tally=1 -2025-10-27T04:02:13.943Z [INFO] server-775c0a83-90a5-51c5-1fe6-8ca2c1b81aed: entering leader state: leader="Node at 775c0a83-90a5-51c5-1fe6-8ca2c1b81aed [Leader]" -2025-10-27T04:02:14.145Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:13.943947101 +0000 UTC m=+8.938819539 (1 nodes), 182.548447ms from start of poll, 182.666168ms from cluster start. Timeout at 2025-10-27 04:02:14.144950461 +0000 UTC m=+9.139822899, 201.00336ms after stability ---- PASS: TestRaft_ApplyConcurrent_Timeout (0.39s) +2026-11-29T10:28:17.463Z [INFO] server-96b6c162-694e-5a1f-8329-eaea3f7a12df: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-96b6c162-694e-5a1f-8329-eaea3f7a12df Address:96b6c162-694e-5a1f-8329-eaea3f7a12df}]" +2026-11-29T10:28:17.463Z [INFO] server-96b6c162-694e-5a1f-8329-eaea3f7a12df: entering follower state: follower="Node at 96b6c162-694e-5a1f-8329-eaea3f7a12df [Follower]" leader-address= leader-id= +2026-11-29T10:28:17.463Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:17.614Z [WARN] server-96b6c162-694e-5a1f-8329-eaea3f7a12df: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:17.614Z [INFO] server-96b6c162-694e-5a1f-8329-eaea3f7a12df: entering candidate state: node="Node at 96b6c162-694e-5a1f-8329-eaea3f7a12df [Candidate]" term=2 +2026-11-29T10:28:17.614Z [INFO] server-96b6c162-694e-5a1f-8329-eaea3f7a12df: election won: term=2 tally=1 +2026-11-29T10:28:17.614Z [INFO] server-96b6c162-694e-5a1f-8329-eaea3f7a12df: entering leader state: leader="Node at 96b6c162-694e-5a1f-8329-eaea3f7a12df [Leader]" +2026-11-29T10:28:17.816Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:17.614975899 +0000 UTC m=+8.350030001 (1 nodes), 150.01764ms from start of poll, 151.90884ms from cluster start. Timeout at 2026-11-29 10:28:17.815976979 +0000 UTC m=+8.551031121, 201.00112ms after stability +--- PASS: TestRaft_ApplyConcurrent_Timeout (0.35s) === RUN TestRaft_JoinNode -2025-10-27T04:02:14.149Z [INFO] server-c0021f15-43e2-0739-970f-2b7604c702c7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c0021f15-43e2-0739-970f-2b7604c702c7 Address:c0021f15-43e2-0739-970f-2b7604c702c7} {Suffrage:Voter ID:server-2cc238e8-f1a3-f183-6c0f-78dd4018d9cc Address:2cc238e8-f1a3-f183-6c0f-78dd4018d9cc}]" -2025-10-27T04:02:14.149Z [INFO] server-2cc238e8-f1a3-f183-6c0f-78dd4018d9cc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c0021f15-43e2-0739-970f-2b7604c702c7 Address:c0021f15-43e2-0739-970f-2b7604c702c7} {Suffrage:Voter ID:server-2cc238e8-f1a3-f183-6c0f-78dd4018d9cc Address:2cc238e8-f1a3-f183-6c0f-78dd4018d9cc}]" -2025-10-27T04:02:14.149Z [INFO] server-c0021f15-43e2-0739-970f-2b7604c702c7: entering follower state: follower="Node at c0021f15-43e2-0739-970f-2b7604c702c7 [Follower]" leader-address= leader-id= -2025-10-27T04:02:14.149Z [INFO] server-2cc238e8-f1a3-f183-6c0f-78dd4018d9cc: entering follower state: follower="Node at 2cc238e8-f1a3-f183-6c0f-78dd4018d9cc [Follower]" leader-address= leader-id= -2025-10-27T04:02:14.149Z [INFO] server-16e72236-8c57-dac8-1b46-c425322c772a: initial configuration: index=0 servers=[] -2025-10-27T04:02:14.149Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:14.149Z [INFO] server-16e72236-8c57-dac8-1b46-c425322c772a: entering follower state: follower="Node at 16e72236-8c57-dac8-1b46-c425322c772a [Follower]" leader-address= leader-id= -2025-10-27T04:02:14.210Z [WARN] server-2cc238e8-f1a3-f183-6c0f-78dd4018d9cc: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:14.210Z [INFO] server-2cc238e8-f1a3-f183-6c0f-78dd4018d9cc: entering candidate state: node="Node at 2cc238e8-f1a3-f183-6c0f-78dd4018d9cc [Candidate]" term=2 -2025-10-27T04:02:14.210Z [INFO] server-2cc238e8-f1a3-f183-6c0f-78dd4018d9cc: election won: term=2 tally=2 -2025-10-27T04:02:14.210Z [INFO] server-2cc238e8-f1a3-f183-6c0f-78dd4018d9cc: entering leader state: leader="Node at 2cc238e8-f1a3-f183-6c0f-78dd4018d9cc [Leader]" -2025-10-27T04:02:14.210Z [INFO] server-2cc238e8-f1a3-f183-6c0f-78dd4018d9cc: added peer, starting replication: peer=server-c0021f15-43e2-0739-970f-2b7604c702c7 -2025-10-27T04:02:14.210Z [INFO] server-2cc238e8-f1a3-f183-6c0f-78dd4018d9cc: pipelining replication: peer="{Voter server-c0021f15-43e2-0739-970f-2b7604c702c7 c0021f15-43e2-0739-970f-2b7604c702c7}" -2025-10-27T04:02:14.235Z [WARN] server-16e72236-8c57-dac8-1b46-c425322c772a: no known peers, aborting election -2025-10-27T04:02:14.316Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:14.210678924 +0000 UTC m=+9.205551362 (1 nodes), 61.144738ms from start of poll, 61.662621ms from cluster start. Timeout at 2025-10-27 04:02:14.315681164 +0000 UTC m=+9.310553602, 105.00224ms after stability -2025-10-27T04:02:14.316Z [INFO] server-2cc238e8-f1a3-f183-6c0f-78dd4018d9cc: updating configuration: command=AddVoter server-id=server-16e72236-8c57-dac8-1b46-c425322c772a server-addr=16e72236-8c57-dac8-1b46-c425322c772a servers="[{Suffrage:Voter ID:server-c0021f15-43e2-0739-970f-2b7604c702c7 Address:c0021f15-43e2-0739-970f-2b7604c702c7} {Suffrage:Voter ID:server-2cc238e8-f1a3-f183-6c0f-78dd4018d9cc Address:2cc238e8-f1a3-f183-6c0f-78dd4018d9cc} {Suffrage:Voter ID:server-16e72236-8c57-dac8-1b46-c425322c772a Address:16e72236-8c57-dac8-1b46-c425322c772a}]" -2025-10-27T04:02:14.316Z [INFO] server-2cc238e8-f1a3-f183-6c0f-78dd4018d9cc: added peer, starting replication: peer=server-16e72236-8c57-dac8-1b46-c425322c772a -2025-10-27T04:02:14.316Z [WARN] server-16e72236-8c57-dac8-1b46-c425322c772a: failed to get previous log: previous-index=3 last-index=0 error="log not found" -2025-10-27T04:02:14.316Z [WARN] server-2cc238e8-f1a3-f183-6c0f-78dd4018d9cc: appendEntries rejected, sending older logs: peer="{Voter server-16e72236-8c57-dac8-1b46-c425322c772a 16e72236-8c57-dac8-1b46-c425322c772a}" next=1 -2025-10-27T04:02:14.316Z [INFO] server-2cc238e8-f1a3-f183-6c0f-78dd4018d9cc: pipelining replication: peer="{Voter server-16e72236-8c57-dac8-1b46-c425322c772a 16e72236-8c57-dac8-1b46-c425322c772a}" -2025-10-27T04:02:14.316Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:14.421Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:14.316746689 +0000 UTC m=+9.311619127 (1 nodes), 440ns from start of poll, 167.730386ms from cluster start. Timeout at 2025-10-27 04:02:14.421753609 +0000 UTC m=+9.416626047, 105.00692ms after stability -2025-10-27T04:02:14.421Z [INFO] server-2cc238e8-f1a3-f183-6c0f-78dd4018d9cc: aborting pipeline replication: peer="{Voter server-16e72236-8c57-dac8-1b46-c425322c772a 16e72236-8c57-dac8-1b46-c425322c772a}" -2025-10-27T04:02:14.421Z [INFO] server-2cc238e8-f1a3-f183-6c0f-78dd4018d9cc: aborting pipeline replication: peer="{Voter server-c0021f15-43e2-0739-970f-2b7604c702c7 c0021f15-43e2-0739-970f-2b7604c702c7}" ---- PASS: TestRaft_JoinNode (0.27s) +2026-11-29T10:28:17.817Z [INFO] server-6f9662bf-153e-12bd-f0a8-624e71d36f80: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6f9662bf-153e-12bd-f0a8-624e71d36f80 Address:6f9662bf-153e-12bd-f0a8-624e71d36f80} {Suffrage:Voter ID:server-0ce56a24-1451-04ec-3dce-edaa4ed2903f Address:0ce56a24-1451-04ec-3dce-edaa4ed2903f}]" +2026-11-29T10:28:17.817Z [INFO] server-0ce56a24-1451-04ec-3dce-edaa4ed2903f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6f9662bf-153e-12bd-f0a8-624e71d36f80 Address:6f9662bf-153e-12bd-f0a8-624e71d36f80} {Suffrage:Voter ID:server-0ce56a24-1451-04ec-3dce-edaa4ed2903f Address:0ce56a24-1451-04ec-3dce-edaa4ed2903f}]" +2026-11-29T10:28:17.817Z [INFO] server-0ce56a24-1451-04ec-3dce-edaa4ed2903f: entering follower state: follower="Node at 0ce56a24-1451-04ec-3dce-edaa4ed2903f [Follower]" leader-address= leader-id= +2026-11-29T10:28:17.817Z [INFO] server-6838007c-c268-9e1b-2b80-768ed608b5ed: initial configuration: index=0 servers=[] +2026-11-29T10:28:17.817Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:17.817Z [INFO] server-6838007c-c268-9e1b-2b80-768ed608b5ed: entering follower state: follower="Node at 6838007c-c268-9e1b-2b80-768ed608b5ed [Follower]" leader-address= leader-id= +2026-11-29T10:28:17.817Z [INFO] server-6f9662bf-153e-12bd-f0a8-624e71d36f80: entering follower state: follower="Node at 6f9662bf-153e-12bd-f0a8-624e71d36f80 [Follower]" leader-address= leader-id= +2026-11-29T10:28:17.869Z [WARN] server-0ce56a24-1451-04ec-3dce-edaa4ed2903f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:17.869Z [INFO] server-0ce56a24-1451-04ec-3dce-edaa4ed2903f: entering candidate state: node="Node at 0ce56a24-1451-04ec-3dce-edaa4ed2903f [Candidate]" term=2 +2026-11-29T10:28:17.869Z [INFO] server-0ce56a24-1451-04ec-3dce-edaa4ed2903f: election won: term=2 tally=2 +2026-11-29T10:28:17.869Z [INFO] server-0ce56a24-1451-04ec-3dce-edaa4ed2903f: entering leader state: leader="Node at 0ce56a24-1451-04ec-3dce-edaa4ed2903f [Leader]" +2026-11-29T10:28:17.869Z [INFO] server-0ce56a24-1451-04ec-3dce-edaa4ed2903f: added peer, starting replication: peer=server-6f9662bf-153e-12bd-f0a8-624e71d36f80 +2026-11-29T10:28:17.869Z [INFO] server-0ce56a24-1451-04ec-3dce-edaa4ed2903f: pipelining replication: peer="{Voter server-6f9662bf-153e-12bd-f0a8-624e71d36f80 6f9662bf-153e-12bd-f0a8-624e71d36f80}" +2026-11-29T10:28:17.890Z [WARN] server-6838007c-c268-9e1b-2b80-768ed608b5ed: no known peers, aborting election +2026-11-29T10:28:17.974Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:17.869392459 +0000 UTC m=+8.604446641 (1 nodes), 51.82284ms from start of poll, 52.2558ms from cluster start. Timeout at 2026-11-29 10:28:17.974395379 +0000 UTC m=+8.709449521, 105.00288ms after stability +2026-11-29T10:28:17.974Z [INFO] server-0ce56a24-1451-04ec-3dce-edaa4ed2903f: updating configuration: command=AddVoter server-id=server-6838007c-c268-9e1b-2b80-768ed608b5ed server-addr=6838007c-c268-9e1b-2b80-768ed608b5ed servers="[{Suffrage:Voter ID:server-6f9662bf-153e-12bd-f0a8-624e71d36f80 Address:6f9662bf-153e-12bd-f0a8-624e71d36f80} {Suffrage:Voter ID:server-0ce56a24-1451-04ec-3dce-edaa4ed2903f Address:0ce56a24-1451-04ec-3dce-edaa4ed2903f} {Suffrage:Voter ID:server-6838007c-c268-9e1b-2b80-768ed608b5ed Address:6838007c-c268-9e1b-2b80-768ed608b5ed}]" +2026-11-29T10:28:17.974Z [INFO] server-0ce56a24-1451-04ec-3dce-edaa4ed2903f: added peer, starting replication: peer=server-6838007c-c268-9e1b-2b80-768ed608b5ed +2026-11-29T10:28:17.975Z [WARN] server-6838007c-c268-9e1b-2b80-768ed608b5ed: failed to get previous log: previous-index=3 last-index=0 error="log not found" +2026-11-29T10:28:17.975Z [WARN] server-0ce56a24-1451-04ec-3dce-edaa4ed2903f: appendEntries rejected, sending older logs: peer="{Voter server-6838007c-c268-9e1b-2b80-768ed608b5ed 6838007c-c268-9e1b-2b80-768ed608b5ed}" next=1 +2026-11-29T10:28:17.975Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:17.975Z [INFO] server-0ce56a24-1451-04ec-3dce-edaa4ed2903f: pipelining replication: peer="{Voter server-6838007c-c268-9e1b-2b80-768ed608b5ed 6838007c-c268-9e1b-2b80-768ed608b5ed}" +2026-11-29T10:28:18.080Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:17.975181459 +0000 UTC m=+8.710235601 (1 nodes), 1µs from start of poll, 158.04476ms from cluster start. Timeout at 2026-11-29 10:28:18.080185979 +0000 UTC m=+8.815240121, 105.00452ms after stability +2026-11-29T10:28:18.080Z [INFO] server-0ce56a24-1451-04ec-3dce-edaa4ed2903f: aborting pipeline replication: peer="{Voter server-6f9662bf-153e-12bd-f0a8-624e71d36f80 6f9662bf-153e-12bd-f0a8-624e71d36f80}" +2026-11-29T10:28:18.080Z [INFO] server-0ce56a24-1451-04ec-3dce-edaa4ed2903f: aborting pipeline replication: peer="{Voter server-6838007c-c268-9e1b-2b80-768ed608b5ed 6838007c-c268-9e1b-2b80-768ed608b5ed}" +--- PASS: TestRaft_JoinNode (0.26s) === RUN TestRaft_JoinNode_ConfigStore -2025-10-27T04:02:14.422Z [INFO] server-13a957e2-4c01-4502-fd2a-cf5a8bf066a6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-13a957e2-4c01-4502-fd2a-cf5a8bf066a6 Address:13a957e2-4c01-4502-fd2a-cf5a8bf066a6}]" -2025-10-27T04:02:14.422Z [INFO] server-24d87b57-6259-d5b3-3da3-8a36db0e69eb: initial configuration: index=0 servers=[] -2025-10-27T04:02:14.423Z [INFO] server-7cfe05ec-6fb7-9f21-f18d-43f364272179: initial configuration: index=0 servers=[] -2025-10-27T04:02:14.423Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:14.425Z [INFO] server-13a957e2-4c01-4502-fd2a-cf5a8bf066a6: entering follower state: follower="Node at 13a957e2-4c01-4502-fd2a-cf5a8bf066a6 [Follower]" leader-address= leader-id= -2025-10-27T04:02:14.425Z [INFO] server-7cfe05ec-6fb7-9f21-f18d-43f364272179: entering follower state: follower="Node at 7cfe05ec-6fb7-9f21-f18d-43f364272179 [Follower]" leader-address= leader-id= -2025-10-27T04:02:14.425Z [INFO] server-24d87b57-6259-d5b3-3da3-8a36db0e69eb: entering follower state: follower="Node at 24d87b57-6259-d5b3-3da3-8a36db0e69eb [Follower]" leader-address= leader-id= -2025-10-27T04:02:14.491Z [WARN] server-7cfe05ec-6fb7-9f21-f18d-43f364272179: no known peers, aborting election -2025-10-27T04:02:14.505Z [WARN] server-13a957e2-4c01-4502-fd2a-cf5a8bf066a6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:14.505Z [INFO] server-13a957e2-4c01-4502-fd2a-cf5a8bf066a6: entering candidate state: node="Node at 13a957e2-4c01-4502-fd2a-cf5a8bf066a6 [Candidate]" term=2 -2025-10-27T04:02:14.505Z [INFO] server-13a957e2-4c01-4502-fd2a-cf5a8bf066a6: election won: term=2 tally=1 -2025-10-27T04:02:14.505Z [INFO] server-13a957e2-4c01-4502-fd2a-cf5a8bf066a6: entering leader state: leader="Node at 13a957e2-4c01-4502-fd2a-cf5a8bf066a6 [Leader]" -2025-10-27T04:02:14.515Z [WARN] server-24d87b57-6259-d5b3-3da3-8a36db0e69eb: no known peers, aborting election -2025-10-27T04:02:14.610Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:14.505698381 +0000 UTC m=+9.500570819 (1 nodes), 82.506164ms from start of poll, 83.233209ms from cluster start. Timeout at 2025-10-27 04:02:14.610701981 +0000 UTC m=+9.605574419, 105.0036ms after stability -2025-10-27T04:02:14.610Z [INFO] server-13a957e2-4c01-4502-fd2a-cf5a8bf066a6: updating configuration: command=AddVoter server-id=server-24d87b57-6259-d5b3-3da3-8a36db0e69eb server-addr=24d87b57-6259-d5b3-3da3-8a36db0e69eb servers="[{Suffrage:Voter ID:server-13a957e2-4c01-4502-fd2a-cf5a8bf066a6 Address:13a957e2-4c01-4502-fd2a-cf5a8bf066a6} {Suffrage:Voter ID:server-24d87b57-6259-d5b3-3da3-8a36db0e69eb Address:24d87b57-6259-d5b3-3da3-8a36db0e69eb}]" -2025-10-27T04:02:14.611Z [INFO] server-13a957e2-4c01-4502-fd2a-cf5a8bf066a6: added peer, starting replication: peer=server-24d87b57-6259-d5b3-3da3-8a36db0e69eb -2025-10-27T04:02:14.611Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:14.611Z [WARN] server-24d87b57-6259-d5b3-3da3-8a36db0e69eb: failed to get previous log: previous-index=3 last-index=0 error="log not found" -2025-10-27T04:02:14.611Z [WARN] server-13a957e2-4c01-4502-fd2a-cf5a8bf066a6: appendEntries rejected, sending older logs: peer="{Voter server-24d87b57-6259-d5b3-3da3-8a36db0e69eb 24d87b57-6259-d5b3-3da3-8a36db0e69eb}" next=1 -2025-10-27T04:02:14.611Z [INFO] server-13a957e2-4c01-4502-fd2a-cf5a8bf066a6: pipelining replication: peer="{Voter server-24d87b57-6259-d5b3-3da3-8a36db0e69eb 24d87b57-6259-d5b3-3da3-8a36db0e69eb}" -2025-10-27T04:02:14.717Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:14.611105061 +0000 UTC m=+9.605977499 (1 nodes), 640ns from start of poll, 188.639889ms from cluster start. Timeout at 2025-10-27 04:02:14.716108821 +0000 UTC m=+9.710981219, 105.00372ms after stability -2025-10-27T04:02:14.717Z [INFO] server-13a957e2-4c01-4502-fd2a-cf5a8bf066a6: updating configuration: command=AddVoter server-id=server-7cfe05ec-6fb7-9f21-f18d-43f364272179 server-addr=7cfe05ec-6fb7-9f21-f18d-43f364272179 servers="[{Suffrage:Voter ID:server-13a957e2-4c01-4502-fd2a-cf5a8bf066a6 Address:13a957e2-4c01-4502-fd2a-cf5a8bf066a6} {Suffrage:Voter ID:server-24d87b57-6259-d5b3-3da3-8a36db0e69eb Address:24d87b57-6259-d5b3-3da3-8a36db0e69eb} {Suffrage:Voter ID:server-7cfe05ec-6fb7-9f21-f18d-43f364272179 Address:7cfe05ec-6fb7-9f21-f18d-43f364272179}]" -2025-10-27T04:02:14.717Z [INFO] server-13a957e2-4c01-4502-fd2a-cf5a8bf066a6: added peer, starting replication: peer=server-7cfe05ec-6fb7-9f21-f18d-43f364272179 -2025-10-27T04:02:14.717Z [WARN] server-7cfe05ec-6fb7-9f21-f18d-43f364272179: failed to get previous log: previous-index=4 last-index=0 error="log not found" -2025-10-27T04:02:14.717Z [WARN] server-13a957e2-4c01-4502-fd2a-cf5a8bf066a6: appendEntries rejected, sending older logs: peer="{Voter server-7cfe05ec-6fb7-9f21-f18d-43f364272179 7cfe05ec-6fb7-9f21-f18d-43f364272179}" next=1 -2025-10-27T04:02:14.717Z [INFO] server-13a957e2-4c01-4502-fd2a-cf5a8bf066a6: pipelining replication: peer="{Voter server-7cfe05ec-6fb7-9f21-f18d-43f364272179 7cfe05ec-6fb7-9f21-f18d-43f364272179}" -2025-10-27T04:02:14.717Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:14.823Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:14.717631869 +0000 UTC m=+9.712504307 (1 nodes), 440ns from start of poll, 295.166697ms from cluster start. Timeout at 2025-10-27 04:02:14.822635069 +0000 UTC m=+9.817507507, 105.0032ms after stability -2025-10-27T04:02:14.823Z [INFO] server-13a957e2-4c01-4502-fd2a-cf5a8bf066a6: aborting pipeline replication: peer="{Voter server-7cfe05ec-6fb7-9f21-f18d-43f364272179 7cfe05ec-6fb7-9f21-f18d-43f364272179}" -2025-10-27T04:02:14.823Z [INFO] server-13a957e2-4c01-4502-fd2a-cf5a8bf066a6: aborting pipeline replication: peer="{Voter server-24d87b57-6259-d5b3-3da3-8a36db0e69eb 24d87b57-6259-d5b3-3da3-8a36db0e69eb}" -2025-10-27T04:02:15.324Z [ERROR] server-13a957e2-4c01-4502-fd2a-cf5a8bf066a6: failed to heartbeat to: peer=24d87b57-6259-d5b3-3da3-8a36db0e69eb backoff time=10ms error="command timed out" ---- PASS: TestRaft_JoinNode_ConfigStore (0.90s) +2026-11-29T10:28:18.081Z [INFO] server-2a597e0f-6e33-9a9b-f310-4d87107c297e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2a597e0f-6e33-9a9b-f310-4d87107c297e Address:2a597e0f-6e33-9a9b-f310-4d87107c297e}]" +2026-11-29T10:28:18.081Z [INFO] server-2a597e0f-6e33-9a9b-f310-4d87107c297e: entering follower state: follower="Node at 2a597e0f-6e33-9a9b-f310-4d87107c297e [Follower]" leader-address= leader-id= +2026-11-29T10:28:18.081Z [INFO] server-7e6cf92e-6a99-2c70-cb95-acdf2fcceb4b: initial configuration: index=0 servers=[] +2026-11-29T10:28:18.081Z [INFO] server-7e6cf92e-6a99-2c70-cb95-acdf2fcceb4b: entering follower state: follower="Node at 7e6cf92e-6a99-2c70-cb95-acdf2fcceb4b [Follower]" leader-address= leader-id= +2026-11-29T10:28:18.081Z [INFO] server-a147d87f-563c-c770-4934-1da175eb0648: initial configuration: index=0 servers=[] +2026-11-29T10:28:18.081Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:18.081Z [INFO] server-a147d87f-563c-c770-4934-1da175eb0648: entering follower state: follower="Node at a147d87f-563c-c770-4934-1da175eb0648 [Follower]" leader-address= leader-id= +2026-11-29T10:28:18.147Z [WARN] server-7e6cf92e-6a99-2c70-cb95-acdf2fcceb4b: no known peers, aborting election +2026-11-29T10:28:18.170Z [WARN] server-a147d87f-563c-c770-4934-1da175eb0648: no known peers, aborting election +2026-11-29T10:28:18.174Z [WARN] server-2a597e0f-6e33-9a9b-f310-4d87107c297e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:18.174Z [INFO] server-2a597e0f-6e33-9a9b-f310-4d87107c297e: entering candidate state: node="Node at 2a597e0f-6e33-9a9b-f310-4d87107c297e [Candidate]" term=2 +2026-11-29T10:28:18.174Z [INFO] server-2a597e0f-6e33-9a9b-f310-4d87107c297e: election won: term=2 tally=1 +2026-11-29T10:28:18.174Z [INFO] server-2a597e0f-6e33-9a9b-f310-4d87107c297e: entering leader state: leader="Node at 2a597e0f-6e33-9a9b-f310-4d87107c297e [Leader]" +2026-11-29T10:28:18.280Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:18.174732699 +0000 UTC m=+8.909786841 (1 nodes), 92.8696ms from start of poll, 93.3124ms from cluster start. Timeout at 2026-11-29 10:28:18.279735259 +0000 UTC m=+9.014789401, 105.00256ms after stability +2026-11-29T10:28:18.280Z [INFO] server-2a597e0f-6e33-9a9b-f310-4d87107c297e: updating configuration: command=AddVoter server-id=server-7e6cf92e-6a99-2c70-cb95-acdf2fcceb4b server-addr=7e6cf92e-6a99-2c70-cb95-acdf2fcceb4b servers="[{Suffrage:Voter ID:server-2a597e0f-6e33-9a9b-f310-4d87107c297e Address:2a597e0f-6e33-9a9b-f310-4d87107c297e} {Suffrage:Voter ID:server-7e6cf92e-6a99-2c70-cb95-acdf2fcceb4b Address:7e6cf92e-6a99-2c70-cb95-acdf2fcceb4b}]" +2026-11-29T10:28:18.280Z [INFO] server-2a597e0f-6e33-9a9b-f310-4d87107c297e: added peer, starting replication: peer=server-7e6cf92e-6a99-2c70-cb95-acdf2fcceb4b +2026-11-29T10:28:18.280Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:18.280Z [WARN] server-7e6cf92e-6a99-2c70-cb95-acdf2fcceb4b: failed to get previous log: previous-index=3 last-index=0 error="log not found" +2026-11-29T10:28:18.280Z [WARN] server-2a597e0f-6e33-9a9b-f310-4d87107c297e: appendEntries rejected, sending older logs: peer="{Voter server-7e6cf92e-6a99-2c70-cb95-acdf2fcceb4b 7e6cf92e-6a99-2c70-cb95-acdf2fcceb4b}" next=1 +2026-11-29T10:28:18.280Z [INFO] server-2a597e0f-6e33-9a9b-f310-4d87107c297e: pipelining replication: peer="{Voter server-7e6cf92e-6a99-2c70-cb95-acdf2fcceb4b 7e6cf92e-6a99-2c70-cb95-acdf2fcceb4b}" +2026-11-29T10:28:18.385Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:18.280681779 +0000 UTC m=+9.015735921 (1 nodes), 880ns from start of poll, 199.26148ms from cluster start. Timeout at 2026-11-29 10:28:18.385683979 +0000 UTC m=+9.120738041, 105.00212ms after stability +2026-11-29T10:28:18.386Z [INFO] server-2a597e0f-6e33-9a9b-f310-4d87107c297e: updating configuration: command=AddVoter server-id=server-a147d87f-563c-c770-4934-1da175eb0648 server-addr=a147d87f-563c-c770-4934-1da175eb0648 servers="[{Suffrage:Voter ID:server-2a597e0f-6e33-9a9b-f310-4d87107c297e Address:2a597e0f-6e33-9a9b-f310-4d87107c297e} {Suffrage:Voter ID:server-7e6cf92e-6a99-2c70-cb95-acdf2fcceb4b Address:7e6cf92e-6a99-2c70-cb95-acdf2fcceb4b} {Suffrage:Voter ID:server-a147d87f-563c-c770-4934-1da175eb0648 Address:a147d87f-563c-c770-4934-1da175eb0648}]" +2026-11-29T10:28:18.386Z [INFO] server-2a597e0f-6e33-9a9b-f310-4d87107c297e: added peer, starting replication: peer=server-a147d87f-563c-c770-4934-1da175eb0648 +2026-11-29T10:28:18.386Z [WARN] server-a147d87f-563c-c770-4934-1da175eb0648: failed to get previous log: previous-index=4 last-index=0 error="log not found" +2026-11-29T10:28:18.386Z [WARN] server-2a597e0f-6e33-9a9b-f310-4d87107c297e: appendEntries rejected, sending older logs: peer="{Voter server-a147d87f-563c-c770-4934-1da175eb0648 a147d87f-563c-c770-4934-1da175eb0648}" next=1 +2026-11-29T10:28:18.386Z [INFO] server-2a597e0f-6e33-9a9b-f310-4d87107c297e: pipelining replication: peer="{Voter server-a147d87f-563c-c770-4934-1da175eb0648 a147d87f-563c-c770-4934-1da175eb0648}" +2026-11-29T10:28:18.386Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:18.492Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:18.386383419 +0000 UTC m=+9.121437561 (1 nodes), 1µs from start of poll, 304.96312ms from cluster start. Timeout at 2026-11-29 10:28:18.491388099 +0000 UTC m=+9.226442241, 105.00468ms after stability +2026-11-29T10:28:18.492Z [INFO] server-2a597e0f-6e33-9a9b-f310-4d87107c297e: aborting pipeline replication: peer="{Voter server-a147d87f-563c-c770-4934-1da175eb0648 a147d87f-563c-c770-4934-1da175eb0648}" +2026-11-29T10:28:18.492Z [INFO] server-2a597e0f-6e33-9a9b-f310-4d87107c297e: aborting pipeline replication: peer="{Voter server-7e6cf92e-6a99-2c70-cb95-acdf2fcceb4b 7e6cf92e-6a99-2c70-cb95-acdf2fcceb4b}" +--- PASS: TestRaft_JoinNode_ConfigStore (0.41s) === RUN TestRaft_RemoveFollower -2025-10-27T04:02:15.325Z [INFO] server-14c7c257-6635-d35b-0d51-f1d2d2ad12bc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-14c7c257-6635-d35b-0d51-f1d2d2ad12bc Address:14c7c257-6635-d35b-0d51-f1d2d2ad12bc} {Suffrage:Voter ID:server-d8a5db99-c079-79d5-bdec-850e335d8b3f Address:d8a5db99-c079-79d5-bdec-850e335d8b3f} {Suffrage:Voter ID:server-3e6307d3-911a-d5fb-a439-25e7c7791c9a Address:3e6307d3-911a-d5fb-a439-25e7c7791c9a}]" -2025-10-27T04:02:15.325Z [INFO] server-d8a5db99-c079-79d5-bdec-850e335d8b3f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-14c7c257-6635-d35b-0d51-f1d2d2ad12bc Address:14c7c257-6635-d35b-0d51-f1d2d2ad12bc} {Suffrage:Voter ID:server-d8a5db99-c079-79d5-bdec-850e335d8b3f Address:d8a5db99-c079-79d5-bdec-850e335d8b3f} {Suffrage:Voter ID:server-3e6307d3-911a-d5fb-a439-25e7c7791c9a Address:3e6307d3-911a-d5fb-a439-25e7c7791c9a}]" -2025-10-27T04:02:15.325Z [INFO] server-3e6307d3-911a-d5fb-a439-25e7c7791c9a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-14c7c257-6635-d35b-0d51-f1d2d2ad12bc Address:14c7c257-6635-d35b-0d51-f1d2d2ad12bc} {Suffrage:Voter ID:server-d8a5db99-c079-79d5-bdec-850e335d8b3f Address:d8a5db99-c079-79d5-bdec-850e335d8b3f} {Suffrage:Voter ID:server-3e6307d3-911a-d5fb-a439-25e7c7791c9a Address:3e6307d3-911a-d5fb-a439-25e7c7791c9a}]" -2025-10-27T04:02:15.325Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:15.325Z [INFO] server-14c7c257-6635-d35b-0d51-f1d2d2ad12bc: entering follower state: follower="Node at 14c7c257-6635-d35b-0d51-f1d2d2ad12bc [Follower]" leader-address= leader-id= -2025-10-27T04:02:15.325Z [INFO] server-d8a5db99-c079-79d5-bdec-850e335d8b3f: entering follower state: follower="Node at d8a5db99-c079-79d5-bdec-850e335d8b3f [Follower]" leader-address= leader-id= -2025-10-27T04:02:15.325Z [INFO] server-3e6307d3-911a-d5fb-a439-25e7c7791c9a: entering follower state: follower="Node at 3e6307d3-911a-d5fb-a439-25e7c7791c9a [Follower]" leader-address= leader-id= -2025-10-27T04:02:15.385Z [WARN] server-3e6307d3-911a-d5fb-a439-25e7c7791c9a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:15.385Z [INFO] server-3e6307d3-911a-d5fb-a439-25e7c7791c9a: entering candidate state: node="Node at 3e6307d3-911a-d5fb-a439-25e7c7791c9a [Candidate]" term=2 -2025-10-27T04:02:15.385Z [INFO] server-3e6307d3-911a-d5fb-a439-25e7c7791c9a: election won: term=2 tally=2 -2025-10-27T04:02:15.386Z [INFO] server-3e6307d3-911a-d5fb-a439-25e7c7791c9a: entering leader state: leader="Node at 3e6307d3-911a-d5fb-a439-25e7c7791c9a [Leader]" -2025-10-27T04:02:15.386Z [INFO] server-3e6307d3-911a-d5fb-a439-25e7c7791c9a: added peer, starting replication: peer=server-14c7c257-6635-d35b-0d51-f1d2d2ad12bc -2025-10-27T04:02:15.386Z [INFO] server-3e6307d3-911a-d5fb-a439-25e7c7791c9a: added peer, starting replication: peer=server-d8a5db99-c079-79d5-bdec-850e335d8b3f -2025-10-27T04:02:15.386Z [INFO] server-3e6307d3-911a-d5fb-a439-25e7c7791c9a: pipelining replication: peer="{Voter server-d8a5db99-c079-79d5-bdec-850e335d8b3f d8a5db99-c079-79d5-bdec-850e335d8b3f}" -2025-10-27T04:02:15.386Z [INFO] server-3e6307d3-911a-d5fb-a439-25e7c7791c9a: pipelining replication: peer="{Voter server-14c7c257-6635-d35b-0d51-f1d2d2ad12bc 14c7c257-6635-d35b-0d51-f1d2d2ad12bc}" -2025-10-27T04:02:15.492Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:15.386217199 +0000 UTC m=+10.381089597 (1 nodes), 60.729855ms from start of poll, 60.983297ms from cluster start. Timeout at 2025-10-27 04:02:15.491219599 +0000 UTC m=+10.486092037, 105.00244ms after stability -2025-10-27T04:02:15.497Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:02:15.603Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:02:15.497747121 +0000 UTC m=+10.492619519 (2 nodes), 1.96µs from start of poll, 172.513219ms from cluster start. Timeout at 2025-10-27 04:02:15.602755641 +0000 UTC m=+10.597628079, 105.00856ms after stability -2025-10-27T04:02:15.603Z [INFO] server-3e6307d3-911a-d5fb-a439-25e7c7791c9a: updating configuration: command=RemoveServer server-id=server-14c7c257-6635-d35b-0d51-f1d2d2ad12bc server-addr= servers="[{Suffrage:Voter ID:server-d8a5db99-c079-79d5-bdec-850e335d8b3f Address:d8a5db99-c079-79d5-bdec-850e335d8b3f} {Suffrage:Voter ID:server-3e6307d3-911a-d5fb-a439-25e7c7791c9a Address:3e6307d3-911a-d5fb-a439-25e7c7791c9a}]" -2025-10-27T04:02:15.603Z [INFO] server-3e6307d3-911a-d5fb-a439-25e7c7791c9a: removed peer, stopping replication: peer=server-14c7c257-6635-d35b-0d51-f1d2d2ad12bc last-index=3 -2025-10-27T04:02:15.603Z [INFO] server-3e6307d3-911a-d5fb-a439-25e7c7791c9a: aborting pipeline replication: peer="{Voter server-14c7c257-6635-d35b-0d51-f1d2d2ad12bc 14c7c257-6635-d35b-0d51-f1d2d2ad12bc}" -2025-10-27T04:02:15.660Z [WARN] server-14c7c257-6635-d35b-0d51-f1d2d2ad12bc: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election -2025-10-27T04:02:15.709Z [INFO] server-3e6307d3-911a-d5fb-a439-25e7c7791c9a: aborting pipeline replication: peer="{Voter server-d8a5db99-c079-79d5-bdec-850e335d8b3f d8a5db99-c079-79d5-bdec-850e335d8b3f}" ---- PASS: TestRaft_RemoveFollower (0.38s) +2026-11-29T10:28:18.493Z [INFO] server-5adefda2-c0e0-7c87-33dc-eed71100f34f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5adefda2-c0e0-7c87-33dc-eed71100f34f Address:5adefda2-c0e0-7c87-33dc-eed71100f34f} {Suffrage:Voter ID:server-2ad872cb-1792-350e-1105-d0f4e0647441 Address:2ad872cb-1792-350e-1105-d0f4e0647441} {Suffrage:Voter ID:server-e8053735-69d7-109c-7119-6dfbd97b175a Address:e8053735-69d7-109c-7119-6dfbd97b175a}]" +2026-11-29T10:28:18.493Z [INFO] server-5adefda2-c0e0-7c87-33dc-eed71100f34f: entering follower state: follower="Node at 5adefda2-c0e0-7c87-33dc-eed71100f34f [Follower]" leader-address= leader-id= +2026-11-29T10:28:18.493Z [INFO] server-2ad872cb-1792-350e-1105-d0f4e0647441: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5adefda2-c0e0-7c87-33dc-eed71100f34f Address:5adefda2-c0e0-7c87-33dc-eed71100f34f} {Suffrage:Voter ID:server-2ad872cb-1792-350e-1105-d0f4e0647441 Address:2ad872cb-1792-350e-1105-d0f4e0647441} {Suffrage:Voter ID:server-e8053735-69d7-109c-7119-6dfbd97b175a Address:e8053735-69d7-109c-7119-6dfbd97b175a}]" +2026-11-29T10:28:18.493Z [INFO] server-2ad872cb-1792-350e-1105-d0f4e0647441: entering follower state: follower="Node at 2ad872cb-1792-350e-1105-d0f4e0647441 [Follower]" leader-address= leader-id= +2026-11-29T10:28:18.493Z [INFO] server-e8053735-69d7-109c-7119-6dfbd97b175a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5adefda2-c0e0-7c87-33dc-eed71100f34f Address:5adefda2-c0e0-7c87-33dc-eed71100f34f} {Suffrage:Voter ID:server-2ad872cb-1792-350e-1105-d0f4e0647441 Address:2ad872cb-1792-350e-1105-d0f4e0647441} {Suffrage:Voter ID:server-e8053735-69d7-109c-7119-6dfbd97b175a Address:e8053735-69d7-109c-7119-6dfbd97b175a}]" +2026-11-29T10:28:18.493Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:18.493Z [INFO] server-e8053735-69d7-109c-7119-6dfbd97b175a: entering follower state: follower="Node at e8053735-69d7-109c-7119-6dfbd97b175a [Follower]" leader-address= leader-id= +2026-11-29T10:28:18.566Z [WARN] server-e8053735-69d7-109c-7119-6dfbd97b175a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:18.567Z [INFO] server-e8053735-69d7-109c-7119-6dfbd97b175a: entering candidate state: node="Node at e8053735-69d7-109c-7119-6dfbd97b175a [Candidate]" term=2 +2026-11-29T10:28:18.567Z [INFO] server-e8053735-69d7-109c-7119-6dfbd97b175a: election won: term=2 tally=2 +2026-11-29T10:28:18.567Z [INFO] server-e8053735-69d7-109c-7119-6dfbd97b175a: entering leader state: leader="Node at e8053735-69d7-109c-7119-6dfbd97b175a [Leader]" +2026-11-29T10:28:18.567Z [INFO] server-e8053735-69d7-109c-7119-6dfbd97b175a: added peer, starting replication: peer=server-5adefda2-c0e0-7c87-33dc-eed71100f34f +2026-11-29T10:28:18.567Z [INFO] server-e8053735-69d7-109c-7119-6dfbd97b175a: added peer, starting replication: peer=server-2ad872cb-1792-350e-1105-d0f4e0647441 +2026-11-29T10:28:18.567Z [INFO] server-e8053735-69d7-109c-7119-6dfbd97b175a: pipelining replication: peer="{Voter server-5adefda2-c0e0-7c87-33dc-eed71100f34f 5adefda2-c0e0-7c87-33dc-eed71100f34f}" +2026-11-29T10:28:18.567Z [INFO] server-e8053735-69d7-109c-7119-6dfbd97b175a: pipelining replication: peer="{Voter server-2ad872cb-1792-350e-1105-d0f4e0647441 2ad872cb-1792-350e-1105-d0f4e0647441}" +2026-11-29T10:28:18.672Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:18.567310579 +0000 UTC m=+9.302364721 (1 nodes), 73.83952ms from start of poll, 74.2554ms from cluster start. Timeout at 2026-11-29 10:28:18.672314779 +0000 UTC m=+9.407368921, 105.0042ms after stability +2026-11-29T10:28:18.677Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:28:18.782Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:28:18.677783699 +0000 UTC m=+9.412837881 (2 nodes), 1.72µs from start of poll, 184.72856ms from cluster start. Timeout at 2026-11-29 10:28:18.782786819 +0000 UTC m=+9.517840921, 105.00304ms after stability +2026-11-29T10:28:18.783Z [INFO] server-e8053735-69d7-109c-7119-6dfbd97b175a: updating configuration: command=RemoveServer server-id=server-5adefda2-c0e0-7c87-33dc-eed71100f34f server-addr= servers="[{Suffrage:Voter ID:server-2ad872cb-1792-350e-1105-d0f4e0647441 Address:2ad872cb-1792-350e-1105-d0f4e0647441} {Suffrage:Voter ID:server-e8053735-69d7-109c-7119-6dfbd97b175a Address:e8053735-69d7-109c-7119-6dfbd97b175a}]" +2026-11-29T10:28:18.783Z [INFO] server-e8053735-69d7-109c-7119-6dfbd97b175a: removed peer, stopping replication: peer=server-5adefda2-c0e0-7c87-33dc-eed71100f34f last-index=3 +2026-11-29T10:28:18.783Z [INFO] server-e8053735-69d7-109c-7119-6dfbd97b175a: aborting pipeline replication: peer="{Voter server-5adefda2-c0e0-7c87-33dc-eed71100f34f 5adefda2-c0e0-7c87-33dc-eed71100f34f}" +2026-11-29T10:28:18.837Z [WARN] server-5adefda2-c0e0-7c87-33dc-eed71100f34f: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +2026-11-29T10:28:18.889Z [INFO] server-e8053735-69d7-109c-7119-6dfbd97b175a: aborting pipeline replication: peer="{Voter server-2ad872cb-1792-350e-1105-d0f4e0647441 2ad872cb-1792-350e-1105-d0f4e0647441}" +--- PASS: TestRaft_RemoveFollower (0.40s) === RUN TestRaft_RemoveLeader -2025-10-27T04:02:15.710Z [INFO] server-fd9b1661-ae5a-a7ea-937a-bca50c70ea97: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fd9b1661-ae5a-a7ea-937a-bca50c70ea97 Address:fd9b1661-ae5a-a7ea-937a-bca50c70ea97} {Suffrage:Voter ID:server-01106951-9acb-1eed-2254-85a3dfd57ebe Address:01106951-9acb-1eed-2254-85a3dfd57ebe} {Suffrage:Voter ID:server-2c0d7602-82c4-d1a5-98fb-80935be06217 Address:2c0d7602-82c4-d1a5-98fb-80935be06217}]" -2025-10-27T04:02:15.710Z [INFO] server-01106951-9acb-1eed-2254-85a3dfd57ebe: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fd9b1661-ae5a-a7ea-937a-bca50c70ea97 Address:fd9b1661-ae5a-a7ea-937a-bca50c70ea97} {Suffrage:Voter ID:server-01106951-9acb-1eed-2254-85a3dfd57ebe Address:01106951-9acb-1eed-2254-85a3dfd57ebe} {Suffrage:Voter ID:server-2c0d7602-82c4-d1a5-98fb-80935be06217 Address:2c0d7602-82c4-d1a5-98fb-80935be06217}]" -2025-10-27T04:02:15.710Z [INFO] server-2c0d7602-82c4-d1a5-98fb-80935be06217: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fd9b1661-ae5a-a7ea-937a-bca50c70ea97 Address:fd9b1661-ae5a-a7ea-937a-bca50c70ea97} {Suffrage:Voter ID:server-01106951-9acb-1eed-2254-85a3dfd57ebe Address:01106951-9acb-1eed-2254-85a3dfd57ebe} {Suffrage:Voter ID:server-2c0d7602-82c4-d1a5-98fb-80935be06217 Address:2c0d7602-82c4-d1a5-98fb-80935be06217}]" -2025-10-27T04:02:15.710Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:15.710Z [INFO] server-fd9b1661-ae5a-a7ea-937a-bca50c70ea97: entering follower state: follower="Node at fd9b1661-ae5a-a7ea-937a-bca50c70ea97 [Follower]" leader-address= leader-id= -2025-10-27T04:02:15.710Z [INFO] server-01106951-9acb-1eed-2254-85a3dfd57ebe: entering follower state: follower="Node at 01106951-9acb-1eed-2254-85a3dfd57ebe [Follower]" leader-address= leader-id= -2025-10-27T04:02:15.710Z [INFO] server-2c0d7602-82c4-d1a5-98fb-80935be06217: entering follower state: follower="Node at 2c0d7602-82c4-d1a5-98fb-80935be06217 [Follower]" leader-address= leader-id= -2025-10-27T04:02:15.769Z [WARN] server-fd9b1661-ae5a-a7ea-937a-bca50c70ea97: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:15.769Z [INFO] server-fd9b1661-ae5a-a7ea-937a-bca50c70ea97: entering candidate state: node="Node at fd9b1661-ae5a-a7ea-937a-bca50c70ea97 [Candidate]" term=2 -2025-10-27T04:02:15.769Z [INFO] server-fd9b1661-ae5a-a7ea-937a-bca50c70ea97: election won: term=2 tally=2 -2025-10-27T04:02:15.769Z [INFO] server-fd9b1661-ae5a-a7ea-937a-bca50c70ea97: entering leader state: leader="Node at fd9b1661-ae5a-a7ea-937a-bca50c70ea97 [Leader]" -2025-10-27T04:02:15.769Z [INFO] server-fd9b1661-ae5a-a7ea-937a-bca50c70ea97: added peer, starting replication: peer=server-01106951-9acb-1eed-2254-85a3dfd57ebe -2025-10-27T04:02:15.769Z [INFO] server-fd9b1661-ae5a-a7ea-937a-bca50c70ea97: added peer, starting replication: peer=server-2c0d7602-82c4-d1a5-98fb-80935be06217 -2025-10-27T04:02:15.769Z [INFO] server-fd9b1661-ae5a-a7ea-937a-bca50c70ea97: pipelining replication: peer="{Voter server-2c0d7602-82c4-d1a5-98fb-80935be06217 2c0d7602-82c4-d1a5-98fb-80935be06217}" -2025-10-27T04:02:15.769Z [INFO] server-fd9b1661-ae5a-a7ea-937a-bca50c70ea97: pipelining replication: peer="{Voter server-01106951-9acb-1eed-2254-85a3dfd57ebe 01106951-9acb-1eed-2254-85a3dfd57ebe}" -2025-10-27T04:02:15.874Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:15.769550059 +0000 UTC m=+10.764422497 (1 nodes), 59.349405ms from start of poll, 59.533407ms from cluster start. Timeout at 2025-10-27 04:02:15.87455426 +0000 UTC m=+10.869426698, 105.004201ms after stability -2025-10-27T04:02:15.883Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:02:15.988Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:02:15.883147836 +0000 UTC m=+10.878020274 (2 nodes), 2.2µs from start of poll, 173.131184ms from cluster start. Timeout at 2025-10-27 04:02:15.988154116 +0000 UTC m=+10.983026554, 105.00628ms after stability -2025-10-27T04:02:15.988Z [INFO] server-fd9b1661-ae5a-a7ea-937a-bca50c70ea97: updating configuration: command=RemoveServer server-id=server-fd9b1661-ae5a-a7ea-937a-bca50c70ea97 server-addr= servers="[{Suffrage:Voter ID:server-01106951-9acb-1eed-2254-85a3dfd57ebe Address:01106951-9acb-1eed-2254-85a3dfd57ebe} {Suffrage:Voter ID:server-2c0d7602-82c4-d1a5-98fb-80935be06217 Address:2c0d7602-82c4-d1a5-98fb-80935be06217}]" -2025-10-27T04:02:15.988Z [INFO] server-fd9b1661-ae5a-a7ea-937a-bca50c70ea97: removed ourself, shutting down -2025-10-27T04:02:15.988Z [INFO] server-fd9b1661-ae5a-a7ea-937a-bca50c70ea97: aborting pipeline replication: peer="{Voter server-2c0d7602-82c4-d1a5-98fb-80935be06217 2c0d7602-82c4-d1a5-98fb-80935be06217}" -2025-10-27T04:02:15.988Z [INFO] server-fd9b1661-ae5a-a7ea-937a-bca50c70ea97: aborting pipeline replication: peer="{Voter server-01106951-9acb-1eed-2254-85a3dfd57ebe 01106951-9acb-1eed-2254-85a3dfd57ebe}" -2025-10-27T04:02:16.089Z [WARN] server-01106951-9acb-1eed-2254-85a3dfd57ebe: heartbeat timeout reached, starting election: last-leader-addr=fd9b1661-ae5a-a7ea-937a-bca50c70ea97 last-leader-id=server-fd9b1661-ae5a-a7ea-937a-bca50c70ea97 -2025-10-27T04:02:16.089Z [INFO] server-01106951-9acb-1eed-2254-85a3dfd57ebe: entering candidate state: node="Node at 01106951-9acb-1eed-2254-85a3dfd57ebe [Candidate]" term=3 -2025-10-27T04:02:16.089Z [WARN] server-2c0d7602-82c4-d1a5-98fb-80935be06217: rejecting vote request since we have a leader: from=01106951-9acb-1eed-2254-85a3dfd57ebe leader=fd9b1661-ae5a-a7ea-937a-bca50c70ea97 leader-id=server-fd9b1661-ae5a-a7ea-937a-bca50c70ea97 -2025-10-27T04:02:16.096Z [WARN] server-2c0d7602-82c4-d1a5-98fb-80935be06217: heartbeat timeout reached, starting election: last-leader-addr=fd9b1661-ae5a-a7ea-937a-bca50c70ea97 last-leader-id=server-fd9b1661-ae5a-a7ea-937a-bca50c70ea97 -2025-10-27T04:02:16.096Z [INFO] server-2c0d7602-82c4-d1a5-98fb-80935be06217: entering candidate state: node="Node at 2c0d7602-82c4-d1a5-98fb-80935be06217 [Candidate]" term=3 -2025-10-27T04:02:16.096Z [INFO] server-01106951-9acb-1eed-2254-85a3dfd57ebe: duplicate requestVote for same term: term=3 -2025-10-27T04:02:16.143Z [WARN] server-01106951-9acb-1eed-2254-85a3dfd57ebe: Election timeout reached, restarting election -2025-10-27T04:02:16.143Z [INFO] server-01106951-9acb-1eed-2254-85a3dfd57ebe: entering candidate state: node="Node at 01106951-9acb-1eed-2254-85a3dfd57ebe [Candidate]" term=4 -2025-10-27T04:02:16.143Z [INFO] server-2c0d7602-82c4-d1a5-98fb-80935be06217: entering follower state: follower="Node at 2c0d7602-82c4-d1a5-98fb-80935be06217 [Follower]" leader-address= leader-id= -2025-10-27T04:02:16.143Z [INFO] server-01106951-9acb-1eed-2254-85a3dfd57ebe: election won: term=4 tally=2 -2025-10-27T04:02:16.143Z [INFO] server-01106951-9acb-1eed-2254-85a3dfd57ebe: entering leader state: leader="Node at 01106951-9acb-1eed-2254-85a3dfd57ebe [Leader]" -2025-10-27T04:02:16.143Z [INFO] server-01106951-9acb-1eed-2254-85a3dfd57ebe: added peer, starting replication: peer=server-2c0d7602-82c4-d1a5-98fb-80935be06217 -2025-10-27T04:02:16.144Z [INFO] server-01106951-9acb-1eed-2254-85a3dfd57ebe: pipelining replication: peer="{Voter server-2c0d7602-82c4-d1a5-98fb-80935be06217 2c0d7602-82c4-d1a5-98fb-80935be06217}" -2025-10-27T04:02:16.199Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:16.304Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:16.199173796 +0000 UTC m=+11.194046234 (1 nodes), 31.441µs from start of poll, 489.157144ms from cluster start. Timeout at 2025-10-27 04:02:16.304181596 +0000 UTC m=+11.299053994, 105.00776ms after stability -2025-10-27T04:02:16.304Z [INFO] server-01106951-9acb-1eed-2254-85a3dfd57ebe: aborting pipeline replication: peer="{Voter server-2c0d7602-82c4-d1a5-98fb-80935be06217 2c0d7602-82c4-d1a5-98fb-80935be06217}" ---- PASS: TestRaft_RemoveLeader (0.59s) +2026-11-29T10:28:18.890Z [INFO] server-16472ce5-78b0-3a29-0df7-eea569f99924: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-16472ce5-78b0-3a29-0df7-eea569f99924 Address:16472ce5-78b0-3a29-0df7-eea569f99924} {Suffrage:Voter ID:server-47092a52-319a-044f-2f78-1b43af3820a5 Address:47092a52-319a-044f-2f78-1b43af3820a5} {Suffrage:Voter ID:server-9fbb2287-d63b-ab23-6ae0-8af534266c3f Address:9fbb2287-d63b-ab23-6ae0-8af534266c3f}]" +2026-11-29T10:28:18.890Z [INFO] server-47092a52-319a-044f-2f78-1b43af3820a5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-16472ce5-78b0-3a29-0df7-eea569f99924 Address:16472ce5-78b0-3a29-0df7-eea569f99924} {Suffrage:Voter ID:server-47092a52-319a-044f-2f78-1b43af3820a5 Address:47092a52-319a-044f-2f78-1b43af3820a5} {Suffrage:Voter ID:server-9fbb2287-d63b-ab23-6ae0-8af534266c3f Address:9fbb2287-d63b-ab23-6ae0-8af534266c3f}]" +2026-11-29T10:28:18.890Z [INFO] server-16472ce5-78b0-3a29-0df7-eea569f99924: entering follower state: follower="Node at 16472ce5-78b0-3a29-0df7-eea569f99924 [Follower]" leader-address= leader-id= +2026-11-29T10:28:18.890Z [INFO] server-9fbb2287-d63b-ab23-6ae0-8af534266c3f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-16472ce5-78b0-3a29-0df7-eea569f99924 Address:16472ce5-78b0-3a29-0df7-eea569f99924} {Suffrage:Voter ID:server-47092a52-319a-044f-2f78-1b43af3820a5 Address:47092a52-319a-044f-2f78-1b43af3820a5} {Suffrage:Voter ID:server-9fbb2287-d63b-ab23-6ae0-8af534266c3f Address:9fbb2287-d63b-ab23-6ae0-8af534266c3f}]" +2026-11-29T10:28:18.890Z [INFO] server-47092a52-319a-044f-2f78-1b43af3820a5: entering follower state: follower="Node at 47092a52-319a-044f-2f78-1b43af3820a5 [Follower]" leader-address= leader-id= +2026-11-29T10:28:18.890Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:18.890Z [INFO] server-9fbb2287-d63b-ab23-6ae0-8af534266c3f: entering follower state: follower="Node at 9fbb2287-d63b-ab23-6ae0-8af534266c3f [Follower]" leader-address= leader-id= +2026-11-29T10:28:18.964Z [WARN] server-9fbb2287-d63b-ab23-6ae0-8af534266c3f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:18.964Z [INFO] server-9fbb2287-d63b-ab23-6ae0-8af534266c3f: entering candidate state: node="Node at 9fbb2287-d63b-ab23-6ae0-8af534266c3f [Candidate]" term=2 +2026-11-29T10:28:18.964Z [INFO] server-9fbb2287-d63b-ab23-6ae0-8af534266c3f: election won: term=2 tally=2 +2026-11-29T10:28:18.964Z [INFO] server-9fbb2287-d63b-ab23-6ae0-8af534266c3f: entering leader state: leader="Node at 9fbb2287-d63b-ab23-6ae0-8af534266c3f [Leader]" +2026-11-29T10:28:18.964Z [INFO] server-9fbb2287-d63b-ab23-6ae0-8af534266c3f: added peer, starting replication: peer=server-16472ce5-78b0-3a29-0df7-eea569f99924 +2026-11-29T10:28:18.964Z [INFO] server-9fbb2287-d63b-ab23-6ae0-8af534266c3f: added peer, starting replication: peer=server-47092a52-319a-044f-2f78-1b43af3820a5 +2026-11-29T10:28:18.964Z [INFO] server-9fbb2287-d63b-ab23-6ae0-8af534266c3f: pipelining replication: peer="{Voter server-47092a52-319a-044f-2f78-1b43af3820a5 47092a52-319a-044f-2f78-1b43af3820a5}" +2026-11-29T10:28:18.965Z [INFO] server-9fbb2287-d63b-ab23-6ae0-8af534266c3f: pipelining replication: peer="{Voter server-16472ce5-78b0-3a29-0df7-eea569f99924 16472ce5-78b0-3a29-0df7-eea569f99924}" +2026-11-29T10:28:19.070Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:18.964796779 +0000 UTC m=+9.699850961 (1 nodes), 74.39484ms from start of poll, 74.81668ms from cluster start. Timeout at 2026-11-29 10:28:19.069799779 +0000 UTC m=+9.804853881, 105.00292ms after stability +2026-11-29T10:28:19.075Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:28:19.180Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:28:19.075444499 +0000 UTC m=+9.810498601 (2 nodes), 1.2µs from start of poll, 185.46432ms from cluster start. Timeout at 2026-11-29 10:28:19.180449579 +0000 UTC m=+9.915503721, 105.00512ms after stability +2026-11-29T10:28:19.180Z [INFO] server-9fbb2287-d63b-ab23-6ae0-8af534266c3f: updating configuration: command=RemoveServer server-id=server-9fbb2287-d63b-ab23-6ae0-8af534266c3f server-addr= servers="[{Suffrage:Voter ID:server-16472ce5-78b0-3a29-0df7-eea569f99924 Address:16472ce5-78b0-3a29-0df7-eea569f99924} {Suffrage:Voter ID:server-47092a52-319a-044f-2f78-1b43af3820a5 Address:47092a52-319a-044f-2f78-1b43af3820a5}]" +2026-11-29T10:28:19.181Z [INFO] server-9fbb2287-d63b-ab23-6ae0-8af534266c3f: removed ourself, shutting down +2026-11-29T10:28:19.181Z [INFO] server-9fbb2287-d63b-ab23-6ae0-8af534266c3f: aborting pipeline replication: peer="{Voter server-47092a52-319a-044f-2f78-1b43af3820a5 47092a52-319a-044f-2f78-1b43af3820a5}" +2026-11-29T10:28:19.181Z [INFO] server-9fbb2287-d63b-ab23-6ae0-8af534266c3f: aborting pipeline replication: peer="{Voter server-16472ce5-78b0-3a29-0df7-eea569f99924 16472ce5-78b0-3a29-0df7-eea569f99924}" +2026-11-29T10:28:19.268Z [WARN] server-16472ce5-78b0-3a29-0df7-eea569f99924: heartbeat timeout reached, starting election: last-leader-addr=9fbb2287-d63b-ab23-6ae0-8af534266c3f last-leader-id=server-9fbb2287-d63b-ab23-6ae0-8af534266c3f +2026-11-29T10:28:19.268Z [INFO] server-16472ce5-78b0-3a29-0df7-eea569f99924: entering candidate state: node="Node at 16472ce5-78b0-3a29-0df7-eea569f99924 [Candidate]" term=3 +2026-11-29T10:28:19.268Z [WARN] server-47092a52-319a-044f-2f78-1b43af3820a5: rejecting vote request since we have a leader: from=16472ce5-78b0-3a29-0df7-eea569f99924 leader=9fbb2287-d63b-ab23-6ae0-8af534266c3f leader-id=server-9fbb2287-d63b-ab23-6ae0-8af534266c3f +2026-11-29T10:28:19.277Z [WARN] server-47092a52-319a-044f-2f78-1b43af3820a5: heartbeat timeout reached, starting election: last-leader-addr=9fbb2287-d63b-ab23-6ae0-8af534266c3f last-leader-id=server-9fbb2287-d63b-ab23-6ae0-8af534266c3f +2026-11-29T10:28:19.277Z [INFO] server-47092a52-319a-044f-2f78-1b43af3820a5: entering candidate state: node="Node at 47092a52-319a-044f-2f78-1b43af3820a5 [Candidate]" term=3 +2026-11-29T10:28:19.277Z [INFO] server-16472ce5-78b0-3a29-0df7-eea569f99924: duplicate requestVote for same term: term=3 +2026-11-29T10:28:19.361Z [WARN] server-16472ce5-78b0-3a29-0df7-eea569f99924: Election timeout reached, restarting election +2026-11-29T10:28:19.361Z [INFO] server-16472ce5-78b0-3a29-0df7-eea569f99924: entering candidate state: node="Node at 16472ce5-78b0-3a29-0df7-eea569f99924 [Candidate]" term=4 +2026-11-29T10:28:19.361Z [INFO] server-47092a52-319a-044f-2f78-1b43af3820a5: entering follower state: follower="Node at 47092a52-319a-044f-2f78-1b43af3820a5 [Follower]" leader-address= leader-id= +2026-11-29T10:28:19.361Z [INFO] server-16472ce5-78b0-3a29-0df7-eea569f99924: election won: term=4 tally=2 +2026-11-29T10:28:19.361Z [INFO] server-16472ce5-78b0-3a29-0df7-eea569f99924: entering leader state: leader="Node at 16472ce5-78b0-3a29-0df7-eea569f99924 [Leader]" +2026-11-29T10:28:19.361Z [INFO] server-16472ce5-78b0-3a29-0df7-eea569f99924: added peer, starting replication: peer=server-47092a52-319a-044f-2f78-1b43af3820a5 +2026-11-29T10:28:19.362Z [INFO] server-16472ce5-78b0-3a29-0df7-eea569f99924: pipelining replication: peer="{Voter server-47092a52-319a-044f-2f78-1b43af3820a5 47092a52-319a-044f-2f78-1b43af3820a5}" +2026-11-29T10:28:19.392Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:19.498Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:19.392967619 +0000 UTC m=+10.128021721 (1 nodes), 38.88µs from start of poll, 502.98744ms from cluster start. Timeout at 2026-11-29 10:28:19.497971139 +0000 UTC m=+10.233025241, 105.00352ms after stability +2026-11-29T10:28:19.499Z [INFO] server-16472ce5-78b0-3a29-0df7-eea569f99924: aborting pipeline replication: peer="{Voter server-47092a52-319a-044f-2f78-1b43af3820a5 47092a52-319a-044f-2f78-1b43af3820a5}" +--- PASS: TestRaft_RemoveLeader (0.61s) === RUN TestRaft_RemoveLeader_NoShutdown -2025-10-27T04:02:16.305Z [INFO] server-29140827-3dc5-ecd0-81cb-1ec055df109c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-29140827-3dc5-ecd0-81cb-1ec055df109c Address:29140827-3dc5-ecd0-81cb-1ec055df109c} {Suffrage:Voter ID:server-7859aeb3-38aa-02b4-fb69-384cc7946230 Address:7859aeb3-38aa-02b4-fb69-384cc7946230} {Suffrage:Voter ID:server-e1f3cd99-f8bf-e7fe-92f3-a3e1fd05a916 Address:e1f3cd99-f8bf-e7fe-92f3-a3e1fd05a916}]" -2025-10-27T04:02:16.305Z [INFO] server-7859aeb3-38aa-02b4-fb69-384cc7946230: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-29140827-3dc5-ecd0-81cb-1ec055df109c Address:29140827-3dc5-ecd0-81cb-1ec055df109c} {Suffrage:Voter ID:server-7859aeb3-38aa-02b4-fb69-384cc7946230 Address:7859aeb3-38aa-02b4-fb69-384cc7946230} {Suffrage:Voter ID:server-e1f3cd99-f8bf-e7fe-92f3-a3e1fd05a916 Address:e1f3cd99-f8bf-e7fe-92f3-a3e1fd05a916}]" -2025-10-27T04:02:16.305Z [INFO] server-e1f3cd99-f8bf-e7fe-92f3-a3e1fd05a916: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-29140827-3dc5-ecd0-81cb-1ec055df109c Address:29140827-3dc5-ecd0-81cb-1ec055df109c} {Suffrage:Voter ID:server-7859aeb3-38aa-02b4-fb69-384cc7946230 Address:7859aeb3-38aa-02b4-fb69-384cc7946230} {Suffrage:Voter ID:server-e1f3cd99-f8bf-e7fe-92f3-a3e1fd05a916 Address:e1f3cd99-f8bf-e7fe-92f3-a3e1fd05a916}]" -2025-10-27T04:02:16.305Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:02:16.305Z [INFO] server-29140827-3dc5-ecd0-81cb-1ec055df109c: entering follower state: follower="Node at 29140827-3dc5-ecd0-81cb-1ec055df109c [Follower]" leader-address= leader-id= -2025-10-27T04:02:16.305Z [INFO] server-7859aeb3-38aa-02b4-fb69-384cc7946230: entering follower state: follower="Node at 7859aeb3-38aa-02b4-fb69-384cc7946230 [Follower]" leader-address= leader-id= -2025-10-27T04:02:16.305Z [INFO] server-e1f3cd99-f8bf-e7fe-92f3-a3e1fd05a916: entering follower state: follower="Node at e1f3cd99-f8bf-e7fe-92f3-a3e1fd05a916 [Follower]" leader-address= leader-id= -2025-10-27T04:02:16.358Z [WARN] server-7859aeb3-38aa-02b4-fb69-384cc7946230: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:16.358Z [INFO] server-7859aeb3-38aa-02b4-fb69-384cc7946230: entering candidate state: node="Node at 7859aeb3-38aa-02b4-fb69-384cc7946230 [Candidate]" term=2 -2025-10-27T04:02:16.358Z [INFO] server-7859aeb3-38aa-02b4-fb69-384cc7946230: election won: term=2 tally=2 -2025-10-27T04:02:16.358Z [INFO] server-7859aeb3-38aa-02b4-fb69-384cc7946230: entering leader state: leader="Node at 7859aeb3-38aa-02b4-fb69-384cc7946230 [Leader]" -2025-10-27T04:02:16.358Z [INFO] server-7859aeb3-38aa-02b4-fb69-384cc7946230: added peer, starting replication: peer=server-29140827-3dc5-ecd0-81cb-1ec055df109c -2025-10-27T04:02:16.358Z [INFO] server-7859aeb3-38aa-02b4-fb69-384cc7946230: added peer, starting replication: peer=server-e1f3cd99-f8bf-e7fe-92f3-a3e1fd05a916 -2025-10-27T04:02:16.358Z [INFO] server-7859aeb3-38aa-02b4-fb69-384cc7946230: pipelining replication: peer="{Voter server-e1f3cd99-f8bf-e7fe-92f3-a3e1fd05a916 e1f3cd99-f8bf-e7fe-92f3-a3e1fd05a916}" -2025-10-27T04:02:16.358Z [WARN] server-e1f3cd99-f8bf-e7fe-92f3-a3e1fd05a916: rejecting vote request since our last term is greater: candidate=7859aeb3-38aa-02b4-fb69-384cc7946230 last-term=2 last-candidate-term=1 -2025-10-27T04:02:16.358Z [INFO] server-7859aeb3-38aa-02b4-fb69-384cc7946230: pipelining replication: peer="{Voter server-29140827-3dc5-ecd0-81cb-1ec055df109c 29140827-3dc5-ecd0-81cb-1ec055df109c}" -2025-10-27T04:02:16.464Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:02:16.358900127 +0000 UTC m=+11.353772565 (2 nodes), 53.680407ms from start of poll, 53.843808ms from cluster start. Timeout at 2025-10-27 04:02:16.463904407 +0000 UTC m=+11.458776845, 105.00428ms after stability -2025-10-27T04:02:16.464Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:16.569Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:16.464119087 +0000 UTC m=+11.458991525 (1 nodes), 840ns from start of poll, 159.062768ms from cluster start. Timeout at 2025-10-27 04:02:16.569127847 +0000 UTC m=+11.564000285, 105.00876ms after stability -2025-10-27T04:02:16.573Z [INFO] server-7859aeb3-38aa-02b4-fb69-384cc7946230: updating configuration: command=RemoveServer server-id=server-7859aeb3-38aa-02b4-fb69-384cc7946230 server-addr= servers="[{Suffrage:Voter ID:server-29140827-3dc5-ecd0-81cb-1ec055df109c Address:29140827-3dc5-ecd0-81cb-1ec055df109c} {Suffrage:Voter ID:server-e1f3cd99-f8bf-e7fe-92f3-a3e1fd05a916 Address:e1f3cd99-f8bf-e7fe-92f3-a3e1fd05a916}]" -2025-10-27T04:02:16.574Z [INFO] server-7859aeb3-38aa-02b4-fb69-384cc7946230: removed ourself, transitioning to follower -2025-10-27T04:02:16.574Z [INFO] server-7859aeb3-38aa-02b4-fb69-384cc7946230: entering follower state: follower="Node at 7859aeb3-38aa-02b4-fb69-384cc7946230 [Follower]" leader-address= leader-id= -2025-10-27T04:02:16.574Z [INFO] server-7859aeb3-38aa-02b4-fb69-384cc7946230: aborting pipeline replication: peer="{Voter server-e1f3cd99-f8bf-e7fe-92f3-a3e1fd05a916 e1f3cd99-f8bf-e7fe-92f3-a3e1fd05a916}" -2025-10-27T04:02:16.574Z [INFO] server-7859aeb3-38aa-02b4-fb69-384cc7946230: aborting pipeline replication: peer="{Voter server-29140827-3dc5-ecd0-81cb-1ec055df109c 29140827-3dc5-ecd0-81cb-1ec055df109c}" -2025-10-27T04:02:16.665Z [WARN] server-e1f3cd99-f8bf-e7fe-92f3-a3e1fd05a916: heartbeat timeout reached, starting election: last-leader-addr=7859aeb3-38aa-02b4-fb69-384cc7946230 last-leader-id=server-7859aeb3-38aa-02b4-fb69-384cc7946230 -2025-10-27T04:02:16.665Z [INFO] server-e1f3cd99-f8bf-e7fe-92f3-a3e1fd05a916: entering candidate state: node="Node at e1f3cd99-f8bf-e7fe-92f3-a3e1fd05a916 [Candidate]" term=3 -2025-10-27T04:02:16.665Z [WARN] server-29140827-3dc5-ecd0-81cb-1ec055df109c: rejecting vote request since we have a leader: from=e1f3cd99-f8bf-e7fe-92f3-a3e1fd05a916 leader=7859aeb3-38aa-02b4-fb69-384cc7946230 leader-id=server-7859aeb3-38aa-02b4-fb69-384cc7946230 -2025-10-27T04:02:16.666Z [WARN] server-7859aeb3-38aa-02b4-fb69-384cc7946230: not part of stable configuration, aborting election -2025-10-27T04:02:16.675Z [WARN] server-29140827-3dc5-ecd0-81cb-1ec055df109c: heartbeat timeout reached, starting election: last-leader-addr=7859aeb3-38aa-02b4-fb69-384cc7946230 last-leader-id=server-7859aeb3-38aa-02b4-fb69-384cc7946230 -2025-10-27T04:02:16.675Z [INFO] server-29140827-3dc5-ecd0-81cb-1ec055df109c: entering candidate state: node="Node at 29140827-3dc5-ecd0-81cb-1ec055df109c [Candidate]" term=3 -2025-10-27T04:02:16.676Z [INFO] server-e1f3cd99-f8bf-e7fe-92f3-a3e1fd05a916: duplicate requestVote for same term: term=3 -2025-10-27T04:02:16.680Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:16.765Z [WARN] server-e1f3cd99-f8bf-e7fe-92f3-a3e1fd05a916: Election timeout reached, restarting election -2025-10-27T04:02:16.765Z [INFO] server-e1f3cd99-f8bf-e7fe-92f3-a3e1fd05a916: entering candidate state: node="Node at e1f3cd99-f8bf-e7fe-92f3-a3e1fd05a916 [Candidate]" term=4 -2025-10-27T04:02:16.765Z [INFO] server-29140827-3dc5-ecd0-81cb-1ec055df109c: entering follower state: follower="Node at 29140827-3dc5-ecd0-81cb-1ec055df109c [Follower]" leader-address= leader-id= -2025-10-27T04:02:16.765Z [INFO] server-e1f3cd99-f8bf-e7fe-92f3-a3e1fd05a916: election won: term=4 tally=2 -2025-10-27T04:02:16.765Z [INFO] server-e1f3cd99-f8bf-e7fe-92f3-a3e1fd05a916: entering leader state: leader="Node at e1f3cd99-f8bf-e7fe-92f3-a3e1fd05a916 [Leader]" -2025-10-27T04:02:16.765Z [INFO] server-e1f3cd99-f8bf-e7fe-92f3-a3e1fd05a916: added peer, starting replication: peer=server-29140827-3dc5-ecd0-81cb-1ec055df109c -2025-10-27T04:02:16.765Z [INFO] server-e1f3cd99-f8bf-e7fe-92f3-a3e1fd05a916: pipelining replication: peer="{Voter server-29140827-3dc5-ecd0-81cb-1ec055df109c 29140827-3dc5-ecd0-81cb-1ec055df109c}" -2025-10-27T04:02:16.871Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:16.765555067 +0000 UTC m=+11.760427505 (1 nodes), 84.977261ms from start of poll, 460.498748ms from cluster start. Timeout at 2025-10-27 04:02:16.870558667 +0000 UTC m=+11.865431105, 105.0036ms after stability -2025-10-27T04:02:16.977Z [INFO] server-e1f3cd99-f8bf-e7fe-92f3-a3e1fd05a916: aborting pipeline replication: peer="{Voter server-29140827-3dc5-ecd0-81cb-1ec055df109c 29140827-3dc5-ecd0-81cb-1ec055df109c}" ---- PASS: TestRaft_RemoveLeader_NoShutdown (0.67s) +2026-11-29T10:28:19.499Z [INFO] server-f47682d2-9c76-078d-d68c-4ac5a95d7608: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f47682d2-9c76-078d-d68c-4ac5a95d7608 Address:f47682d2-9c76-078d-d68c-4ac5a95d7608} {Suffrage:Voter ID:server-21202a53-3f70-ead6-edc0-e63b9f8aba52 Address:21202a53-3f70-ead6-edc0-e63b9f8aba52} {Suffrage:Voter ID:server-0f025a60-8f33-10c6-1a02-c47c94d03561 Address:0f025a60-8f33-10c6-1a02-c47c94d03561}]" +2026-11-29T10:28:19.499Z [INFO] server-21202a53-3f70-ead6-edc0-e63b9f8aba52: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f47682d2-9c76-078d-d68c-4ac5a95d7608 Address:f47682d2-9c76-078d-d68c-4ac5a95d7608} {Suffrage:Voter ID:server-21202a53-3f70-ead6-edc0-e63b9f8aba52 Address:21202a53-3f70-ead6-edc0-e63b9f8aba52} {Suffrage:Voter ID:server-0f025a60-8f33-10c6-1a02-c47c94d03561 Address:0f025a60-8f33-10c6-1a02-c47c94d03561}]" +2026-11-29T10:28:19.499Z [INFO] server-0f025a60-8f33-10c6-1a02-c47c94d03561: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f47682d2-9c76-078d-d68c-4ac5a95d7608 Address:f47682d2-9c76-078d-d68c-4ac5a95d7608} {Suffrage:Voter ID:server-21202a53-3f70-ead6-edc0-e63b9f8aba52 Address:21202a53-3f70-ead6-edc0-e63b9f8aba52} {Suffrage:Voter ID:server-0f025a60-8f33-10c6-1a02-c47c94d03561 Address:0f025a60-8f33-10c6-1a02-c47c94d03561}]" +2026-11-29T10:28:19.499Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:28:19.499Z [INFO] server-f47682d2-9c76-078d-d68c-4ac5a95d7608: entering follower state: follower="Node at f47682d2-9c76-078d-d68c-4ac5a95d7608 [Follower]" leader-address= leader-id= +2026-11-29T10:28:19.499Z [INFO] server-0f025a60-8f33-10c6-1a02-c47c94d03561: entering follower state: follower="Node at 0f025a60-8f33-10c6-1a02-c47c94d03561 [Follower]" leader-address= leader-id= +2026-11-29T10:28:19.499Z [INFO] server-21202a53-3f70-ead6-edc0-e63b9f8aba52: entering follower state: follower="Node at 21202a53-3f70-ead6-edc0-e63b9f8aba52 [Follower]" leader-address= leader-id= +2026-11-29T10:28:19.564Z [WARN] server-f47682d2-9c76-078d-d68c-4ac5a95d7608: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:19.564Z [INFO] server-f47682d2-9c76-078d-d68c-4ac5a95d7608: entering candidate state: node="Node at f47682d2-9c76-078d-d68c-4ac5a95d7608 [Candidate]" term=2 +2026-11-29T10:28:19.564Z [INFO] server-f47682d2-9c76-078d-d68c-4ac5a95d7608: election won: term=2 tally=2 +2026-11-29T10:28:19.564Z [INFO] server-f47682d2-9c76-078d-d68c-4ac5a95d7608: entering leader state: leader="Node at f47682d2-9c76-078d-d68c-4ac5a95d7608 [Leader]" +2026-11-29T10:28:19.564Z [INFO] server-f47682d2-9c76-078d-d68c-4ac5a95d7608: added peer, starting replication: peer=server-21202a53-3f70-ead6-edc0-e63b9f8aba52 +2026-11-29T10:28:19.564Z [INFO] server-f47682d2-9c76-078d-d68c-4ac5a95d7608: added peer, starting replication: peer=server-0f025a60-8f33-10c6-1a02-c47c94d03561 +2026-11-29T10:28:19.564Z [INFO] server-f47682d2-9c76-078d-d68c-4ac5a95d7608: pipelining replication: peer="{Voter server-21202a53-3f70-ead6-edc0-e63b9f8aba52 21202a53-3f70-ead6-edc0-e63b9f8aba52}" +2026-11-29T10:28:19.564Z [INFO] server-f47682d2-9c76-078d-d68c-4ac5a95d7608: pipelining replication: peer="{Voter server-0f025a60-8f33-10c6-1a02-c47c94d03561 0f025a60-8f33-10c6-1a02-c47c94d03561}" +2026-11-29T10:28:19.670Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:28:19.564738379 +0000 UTC m=+10.299792481 (2 nodes), 64.96008ms from start of poll, 65.176ms from cluster start. Timeout at 2026-11-29 10:28:19.669740539 +0000 UTC m=+10.404794641, 105.00216ms after stability +2026-11-29T10:28:19.670Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:19.776Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:19.670835659 +0000 UTC m=+10.405889761 (1 nodes), 880ns from start of poll, 171.27328ms from cluster start. Timeout at 2026-11-29 10:28:19.775841099 +0000 UTC m=+10.510895201, 105.00544ms after stability +2026-11-29T10:28:19.779Z [INFO] server-f47682d2-9c76-078d-d68c-4ac5a95d7608: updating configuration: command=RemoveServer server-id=server-f47682d2-9c76-078d-d68c-4ac5a95d7608 server-addr= servers="[{Suffrage:Voter ID:server-21202a53-3f70-ead6-edc0-e63b9f8aba52 Address:21202a53-3f70-ead6-edc0-e63b9f8aba52} {Suffrage:Voter ID:server-0f025a60-8f33-10c6-1a02-c47c94d03561 Address:0f025a60-8f33-10c6-1a02-c47c94d03561}]" +2026-11-29T10:28:19.779Z [INFO] server-f47682d2-9c76-078d-d68c-4ac5a95d7608: removed ourself, transitioning to follower +2026-11-29T10:28:19.779Z [INFO] server-f47682d2-9c76-078d-d68c-4ac5a95d7608: entering follower state: follower="Node at f47682d2-9c76-078d-d68c-4ac5a95d7608 [Follower]" leader-address= leader-id= +2026-11-29T10:28:19.779Z [INFO] server-f47682d2-9c76-078d-d68c-4ac5a95d7608: aborting pipeline replication: peer="{Voter server-21202a53-3f70-ead6-edc0-e63b9f8aba52 21202a53-3f70-ead6-edc0-e63b9f8aba52}" +2026-11-29T10:28:19.779Z [INFO] server-f47682d2-9c76-078d-d68c-4ac5a95d7608: aborting pipeline replication: peer="{Voter server-0f025a60-8f33-10c6-1a02-c47c94d03561 0f025a60-8f33-10c6-1a02-c47c94d03561}" +2026-11-29T10:28:19.831Z [WARN] server-f47682d2-9c76-078d-d68c-4ac5a95d7608: not part of stable configuration, aborting election +2026-11-29T10:28:19.838Z [WARN] server-0f025a60-8f33-10c6-1a02-c47c94d03561: heartbeat timeout reached, starting election: last-leader-addr=f47682d2-9c76-078d-d68c-4ac5a95d7608 last-leader-id=server-f47682d2-9c76-078d-d68c-4ac5a95d7608 +2026-11-29T10:28:19.838Z [INFO] server-0f025a60-8f33-10c6-1a02-c47c94d03561: entering candidate state: node="Node at 0f025a60-8f33-10c6-1a02-c47c94d03561 [Candidate]" term=3 +2026-11-29T10:28:19.838Z [WARN] server-21202a53-3f70-ead6-edc0-e63b9f8aba52: rejecting vote request since we have a leader: from=0f025a60-8f33-10c6-1a02-c47c94d03561 leader=f47682d2-9c76-078d-d68c-4ac5a95d7608 leader-id=server-f47682d2-9c76-078d-d68c-4ac5a95d7608 +2026-11-29T10:28:19.885Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:19.909Z [WARN] server-21202a53-3f70-ead6-edc0-e63b9f8aba52: heartbeat timeout reached, starting election: last-leader-addr=f47682d2-9c76-078d-d68c-4ac5a95d7608 last-leader-id=server-f47682d2-9c76-078d-d68c-4ac5a95d7608 +2026-11-29T10:28:19.909Z [INFO] server-21202a53-3f70-ead6-edc0-e63b9f8aba52: entering candidate state: node="Node at 21202a53-3f70-ead6-edc0-e63b9f8aba52 [Candidate]" term=3 +2026-11-29T10:28:19.909Z [INFO] server-0f025a60-8f33-10c6-1a02-c47c94d03561: duplicate requestVote for same term: term=3 +2026-11-29T10:28:19.920Z [WARN] server-0f025a60-8f33-10c6-1a02-c47c94d03561: Election timeout reached, restarting election +2026-11-29T10:28:19.921Z [INFO] server-0f025a60-8f33-10c6-1a02-c47c94d03561: entering candidate state: node="Node at 0f025a60-8f33-10c6-1a02-c47c94d03561 [Candidate]" term=4 +2026-11-29T10:28:19.921Z [INFO] server-21202a53-3f70-ead6-edc0-e63b9f8aba52: entering follower state: follower="Node at 21202a53-3f70-ead6-edc0-e63b9f8aba52 [Follower]" leader-address= leader-id= +2026-11-29T10:28:19.921Z [INFO] server-0f025a60-8f33-10c6-1a02-c47c94d03561: election won: term=4 tally=2 +2026-11-29T10:28:19.921Z [INFO] server-0f025a60-8f33-10c6-1a02-c47c94d03561: entering leader state: leader="Node at 0f025a60-8f33-10c6-1a02-c47c94d03561 [Leader]" +2026-11-29T10:28:19.921Z [INFO] server-0f025a60-8f33-10c6-1a02-c47c94d03561: added peer, starting replication: peer=server-21202a53-3f70-ead6-edc0-e63b9f8aba52 +2026-11-29T10:28:19.921Z [INFO] server-0f025a60-8f33-10c6-1a02-c47c94d03561: pipelining replication: peer="{Voter server-21202a53-3f70-ead6-edc0-e63b9f8aba52 21202a53-3f70-ead6-edc0-e63b9f8aba52}" +2026-11-29T10:28:20.027Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:19.921228779 +0000 UTC m=+10.656282921 (1 nodes), 35.64652ms from start of poll, 421.66644ms from cluster start. Timeout at 2026-11-29 10:28:20.026235099 +0000 UTC m=+10.761289241, 105.00632ms after stability +2026-11-29T10:28:20.132Z [INFO] server-0f025a60-8f33-10c6-1a02-c47c94d03561: aborting pipeline replication: peer="{Voter server-21202a53-3f70-ead6-edc0-e63b9f8aba52 21202a53-3f70-ead6-edc0-e63b9f8aba52}" +--- PASS: TestRaft_RemoveLeader_NoShutdown (0.63s) === RUN TestRaft_RemoveFollower_SplitCluster -2025-10-27T04:02:16.977Z [INFO] server-6caae519-d5c7-1b36-46c3-dcd8d6db8784: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6caae519-d5c7-1b36-46c3-dcd8d6db8784 Address:6caae519-d5c7-1b36-46c3-dcd8d6db8784} {Suffrage:Voter ID:server-466b69cd-920d-ace2-a7c2-ea5948533755 Address:466b69cd-920d-ace2-a7c2-ea5948533755} {Suffrage:Voter ID:server-4820dc28-b25a-b0eb-3920-01a8b47ad7fd Address:4820dc28-b25a-b0eb-3920-01a8b47ad7fd} {Suffrage:Voter ID:server-2927baa2-ddbc-4361-7b3f-62efee85180d Address:2927baa2-ddbc-4361-7b3f-62efee85180d}]" -2025-10-27T04:02:16.977Z [INFO] server-466b69cd-920d-ace2-a7c2-ea5948533755: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6caae519-d5c7-1b36-46c3-dcd8d6db8784 Address:6caae519-d5c7-1b36-46c3-dcd8d6db8784} {Suffrage:Voter ID:server-466b69cd-920d-ace2-a7c2-ea5948533755 Address:466b69cd-920d-ace2-a7c2-ea5948533755} {Suffrage:Voter ID:server-4820dc28-b25a-b0eb-3920-01a8b47ad7fd Address:4820dc28-b25a-b0eb-3920-01a8b47ad7fd} {Suffrage:Voter ID:server-2927baa2-ddbc-4361-7b3f-62efee85180d Address:2927baa2-ddbc-4361-7b3f-62efee85180d}]" -2025-10-27T04:02:16.977Z [INFO] server-4820dc28-b25a-b0eb-3920-01a8b47ad7fd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6caae519-d5c7-1b36-46c3-dcd8d6db8784 Address:6caae519-d5c7-1b36-46c3-dcd8d6db8784} {Suffrage:Voter ID:server-466b69cd-920d-ace2-a7c2-ea5948533755 Address:466b69cd-920d-ace2-a7c2-ea5948533755} {Suffrage:Voter ID:server-4820dc28-b25a-b0eb-3920-01a8b47ad7fd Address:4820dc28-b25a-b0eb-3920-01a8b47ad7fd} {Suffrage:Voter ID:server-2927baa2-ddbc-4361-7b3f-62efee85180d Address:2927baa2-ddbc-4361-7b3f-62efee85180d}]" -2025-10-27T04:02:16.977Z [INFO] server-2927baa2-ddbc-4361-7b3f-62efee85180d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6caae519-d5c7-1b36-46c3-dcd8d6db8784 Address:6caae519-d5c7-1b36-46c3-dcd8d6db8784} {Suffrage:Voter ID:server-466b69cd-920d-ace2-a7c2-ea5948533755 Address:466b69cd-920d-ace2-a7c2-ea5948533755} {Suffrage:Voter ID:server-4820dc28-b25a-b0eb-3920-01a8b47ad7fd Address:4820dc28-b25a-b0eb-3920-01a8b47ad7fd} {Suffrage:Voter ID:server-2927baa2-ddbc-4361-7b3f-62efee85180d Address:2927baa2-ddbc-4361-7b3f-62efee85180d}]" -2025-10-27T04:02:16.977Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:16.978Z [INFO] server-6caae519-d5c7-1b36-46c3-dcd8d6db8784: entering follower state: follower="Node at 6caae519-d5c7-1b36-46c3-dcd8d6db8784 [Follower]" leader-address= leader-id= -2025-10-27T04:02:16.978Z [INFO] server-466b69cd-920d-ace2-a7c2-ea5948533755: entering follower state: follower="Node at 466b69cd-920d-ace2-a7c2-ea5948533755 [Follower]" leader-address= leader-id= -2025-10-27T04:02:16.978Z [INFO] server-4820dc28-b25a-b0eb-3920-01a8b47ad7fd: entering follower state: follower="Node at 4820dc28-b25a-b0eb-3920-01a8b47ad7fd [Follower]" leader-address= leader-id= -2025-10-27T04:02:16.978Z [INFO] server-2927baa2-ddbc-4361-7b3f-62efee85180d: entering follower state: follower="Node at 2927baa2-ddbc-4361-7b3f-62efee85180d [Follower]" leader-address= leader-id= -2025-10-27T04:02:17.029Z [WARN] server-2927baa2-ddbc-4361-7b3f-62efee85180d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:17.029Z [INFO] server-2927baa2-ddbc-4361-7b3f-62efee85180d: entering candidate state: node="Node at 2927baa2-ddbc-4361-7b3f-62efee85180d [Candidate]" term=2 -2025-10-27T04:02:17.029Z [INFO] server-2927baa2-ddbc-4361-7b3f-62efee85180d: election won: term=2 tally=3 -2025-10-27T04:02:17.029Z [INFO] server-2927baa2-ddbc-4361-7b3f-62efee85180d: entering leader state: leader="Node at 2927baa2-ddbc-4361-7b3f-62efee85180d [Leader]" -2025-10-27T04:02:17.029Z [INFO] server-2927baa2-ddbc-4361-7b3f-62efee85180d: added peer, starting replication: peer=server-6caae519-d5c7-1b36-46c3-dcd8d6db8784 -2025-10-27T04:02:17.029Z [INFO] server-2927baa2-ddbc-4361-7b3f-62efee85180d: added peer, starting replication: peer=server-466b69cd-920d-ace2-a7c2-ea5948533755 -2025-10-27T04:02:17.029Z [INFO] server-2927baa2-ddbc-4361-7b3f-62efee85180d: added peer, starting replication: peer=server-4820dc28-b25a-b0eb-3920-01a8b47ad7fd -2025-10-27T04:02:17.030Z [INFO] server-2927baa2-ddbc-4361-7b3f-62efee85180d: pipelining replication: peer="{Voter server-4820dc28-b25a-b0eb-3920-01a8b47ad7fd 4820dc28-b25a-b0eb-3920-01a8b47ad7fd}" -2025-10-27T04:02:17.030Z [INFO] server-2927baa2-ddbc-4361-7b3f-62efee85180d: pipelining replication: peer="{Voter server-6caae519-d5c7-1b36-46c3-dcd8d6db8784 6caae519-d5c7-1b36-46c3-dcd8d6db8784}" -2025-10-27T04:02:17.030Z [INFO] server-2927baa2-ddbc-4361-7b3f-62efee85180d: pipelining replication: peer="{Voter server-466b69cd-920d-ace2-a7c2-ea5948533755 466b69cd-920d-ace2-a7c2-ea5948533755}" -2025-10-27T04:02:17.135Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:17.030005834 +0000 UTC m=+12.024878232 (1 nodes), 52.009355ms from start of poll, 52.238237ms from cluster start. Timeout at 2025-10-27 04:02:17.135007554 +0000 UTC m=+12.129880032, 105.0018ms after stability -2025-10-27T04:02:17.241Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:02:17.352Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:02:17.2415588 +0000 UTC m=+12.236431238 (3 nodes), 2.24µs from start of poll, 263.791243ms from cluster start. Timeout at 2025-10-27 04:02:17.3465644 +0000 UTC m=+12.341436838, 105.0056ms after stability -2025-10-27T04:02:17.352Z [INFO] server-2927baa2-ddbc-4361-7b3f-62efee85180d: updating configuration: command=RemoveServer server-id=server-4820dc28-b25a-b0eb-3920-01a8b47ad7fd server-addr= servers="[{Suffrage:Voter ID:server-6caae519-d5c7-1b36-46c3-dcd8d6db8784 Address:6caae519-d5c7-1b36-46c3-dcd8d6db8784} {Suffrage:Voter ID:server-466b69cd-920d-ace2-a7c2-ea5948533755 Address:466b69cd-920d-ace2-a7c2-ea5948533755} {Suffrage:Voter ID:server-2927baa2-ddbc-4361-7b3f-62efee85180d Address:2927baa2-ddbc-4361-7b3f-62efee85180d}]" -2025-10-27T04:02:17.353Z [INFO] server-2927baa2-ddbc-4361-7b3f-62efee85180d: removed peer, stopping replication: peer=server-4820dc28-b25a-b0eb-3920-01a8b47ad7fd last-index=3 -2025-10-27T04:02:17.353Z [ERROR] server-2927baa2-ddbc-4361-7b3f-62efee85180d: failed to pipeline appendEntries: peer="{Voter server-6caae519-d5c7-1b36-46c3-dcd8d6db8784 6caae519-d5c7-1b36-46c3-dcd8d6db8784}" error="append pipeline closed" -2025-10-27T04:02:17.353Z [INFO] server-2927baa2-ddbc-4361-7b3f-62efee85180d: aborting pipeline replication: peer="{Voter server-6caae519-d5c7-1b36-46c3-dcd8d6db8784 6caae519-d5c7-1b36-46c3-dcd8d6db8784}" -2025-10-27T04:02:17.353Z [ERROR] server-2927baa2-ddbc-4361-7b3f-62efee85180d: failed to pipeline appendEntries: peer="{Voter server-466b69cd-920d-ace2-a7c2-ea5948533755 466b69cd-920d-ace2-a7c2-ea5948533755}" error="append pipeline closed" -2025-10-27T04:02:17.353Z [INFO] server-2927baa2-ddbc-4361-7b3f-62efee85180d: aborting pipeline replication: peer="{Voter server-466b69cd-920d-ace2-a7c2-ea5948533755 466b69cd-920d-ace2-a7c2-ea5948533755}" -2025-10-27T04:02:17.353Z [INFO] server-2927baa2-ddbc-4361-7b3f-62efee85180d: aborting pipeline replication: peer="{Voter server-4820dc28-b25a-b0eb-3920-01a8b47ad7fd 4820dc28-b25a-b0eb-3920-01a8b47ad7fd}" -2025-10-27T04:02:17.359Z [ERROR] server-2927baa2-ddbc-4361-7b3f-62efee85180d: failed to heartbeat to: peer=6caae519-d5c7-1b36-46c3-dcd8d6db8784 backoff time=10ms error="failed to connect to peer: 6caae519-d5c7-1b36-46c3-dcd8d6db8784" -2025-10-27T04:02:17.361Z [ERROR] server-2927baa2-ddbc-4361-7b3f-62efee85180d: failed to appendEntries to: peer="{Voter server-466b69cd-920d-ace2-a7c2-ea5948533755 466b69cd-920d-ace2-a7c2-ea5948533755}" error="failed to connect to peer: 466b69cd-920d-ace2-a7c2-ea5948533755" -2025-10-27T04:02:17.361Z [ERROR] server-2927baa2-ddbc-4361-7b3f-62efee85180d: failed to heartbeat to: peer=466b69cd-920d-ace2-a7c2-ea5948533755 backoff time=10ms error="failed to connect to peer: 466b69cd-920d-ace2-a7c2-ea5948533755" -2025-10-27T04:02:17.361Z [ERROR] server-2927baa2-ddbc-4361-7b3f-62efee85180d: failed to appendEntries to: peer="{Voter server-6caae519-d5c7-1b36-46c3-dcd8d6db8784 6caae519-d5c7-1b36-46c3-dcd8d6db8784}" error="failed to connect to peer: 6caae519-d5c7-1b36-46c3-dcd8d6db8784" -2025-10-27T04:02:17.376Z [ERROR] server-2927baa2-ddbc-4361-7b3f-62efee85180d: failed to heartbeat to: peer=6caae519-d5c7-1b36-46c3-dcd8d6db8784 backoff time=10ms error="failed to connect to peer: 6caae519-d5c7-1b36-46c3-dcd8d6db8784" -2025-10-27T04:02:17.381Z [ERROR] server-2927baa2-ddbc-4361-7b3f-62efee85180d: failed to heartbeat to: peer=466b69cd-920d-ace2-a7c2-ea5948533755 backoff time=10ms error="failed to connect to peer: 466b69cd-920d-ace2-a7c2-ea5948533755" -2025-10-27T04:02:17.382Z [ERROR] server-2927baa2-ddbc-4361-7b3f-62efee85180d: failed to appendEntries to: peer="{Voter server-466b69cd-920d-ace2-a7c2-ea5948533755 466b69cd-920d-ace2-a7c2-ea5948533755}" error="failed to connect to peer: 466b69cd-920d-ace2-a7c2-ea5948533755" -2025-10-27T04:02:17.382Z [ERROR] server-2927baa2-ddbc-4361-7b3f-62efee85180d: failed to appendEntries to: peer="{Voter server-6caae519-d5c7-1b36-46c3-dcd8d6db8784 6caae519-d5c7-1b36-46c3-dcd8d6db8784}" error="failed to connect to peer: 6caae519-d5c7-1b36-46c3-dcd8d6db8784" -2025-10-27T04:02:17.394Z [ERROR] server-2927baa2-ddbc-4361-7b3f-62efee85180d: failed to heartbeat to: peer=6caae519-d5c7-1b36-46c3-dcd8d6db8784 backoff time=10ms error="failed to connect to peer: 6caae519-d5c7-1b36-46c3-dcd8d6db8784" -2025-10-27T04:02:17.402Z [WARN] server-2927baa2-ddbc-4361-7b3f-62efee85180d: failed to contact: server-id=server-6caae519-d5c7-1b36-46c3-dcd8d6db8784 time=50.920348ms -2025-10-27T04:02:17.402Z [WARN] server-2927baa2-ddbc-4361-7b3f-62efee85180d: failed to contact: server-id=server-466b69cd-920d-ace2-a7c2-ea5948533755 time=50.959108ms -2025-10-27T04:02:17.402Z [WARN] server-2927baa2-ddbc-4361-7b3f-62efee85180d: failed to contact quorum of nodes, stepping down -2025-10-27T04:02:17.403Z [INFO] server-2927baa2-ddbc-4361-7b3f-62efee85180d: entering follower state: follower="Node at 2927baa2-ddbc-4361-7b3f-62efee85180d [Follower]" leader-address= leader-id= -2025-10-27T04:02:17.403Z [ERROR] server-2927baa2-ddbc-4361-7b3f-62efee85180d: failed to appendEntries to: peer="{Voter server-6caae519-d5c7-1b36-46c3-dcd8d6db8784 6caae519-d5c7-1b36-46c3-dcd8d6db8784}" error="failed to connect to peer: 6caae519-d5c7-1b36-46c3-dcd8d6db8784" -2025-10-27T04:02:17.403Z [ERROR] server-2927baa2-ddbc-4361-7b3f-62efee85180d: failed to appendEntries to: peer="{Voter server-466b69cd-920d-ace2-a7c2-ea5948533755 466b69cd-920d-ace2-a7c2-ea5948533755}" error="failed to connect to peer: 466b69cd-920d-ace2-a7c2-ea5948533755" +2026-11-29T10:28:20.134Z [INFO] server-0dcf0e54-f30e-e4a1-a418-29c49c7c5f85: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0dcf0e54-f30e-e4a1-a418-29c49c7c5f85 Address:0dcf0e54-f30e-e4a1-a418-29c49c7c5f85} {Suffrage:Voter ID:server-743a9d18-f55d-08cd-22dc-a01953d881c1 Address:743a9d18-f55d-08cd-22dc-a01953d881c1} {Suffrage:Voter ID:server-bd512ca1-b2dd-16cf-a108-597e48327211 Address:bd512ca1-b2dd-16cf-a108-597e48327211} {Suffrage:Voter ID:server-4c9344db-bdca-45c4-041b-38a0d1f08793 Address:4c9344db-bdca-45c4-041b-38a0d1f08793}]" +2026-11-29T10:28:20.135Z [INFO] server-743a9d18-f55d-08cd-22dc-a01953d881c1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0dcf0e54-f30e-e4a1-a418-29c49c7c5f85 Address:0dcf0e54-f30e-e4a1-a418-29c49c7c5f85} {Suffrage:Voter ID:server-743a9d18-f55d-08cd-22dc-a01953d881c1 Address:743a9d18-f55d-08cd-22dc-a01953d881c1} {Suffrage:Voter ID:server-bd512ca1-b2dd-16cf-a108-597e48327211 Address:bd512ca1-b2dd-16cf-a108-597e48327211} {Suffrage:Voter ID:server-4c9344db-bdca-45c4-041b-38a0d1f08793 Address:4c9344db-bdca-45c4-041b-38a0d1f08793}]" +2026-11-29T10:28:20.135Z [INFO] server-0dcf0e54-f30e-e4a1-a418-29c49c7c5f85: entering follower state: follower="Node at 0dcf0e54-f30e-e4a1-a418-29c49c7c5f85 [Follower]" leader-address= leader-id= +2026-11-29T10:28:20.135Z [INFO] server-743a9d18-f55d-08cd-22dc-a01953d881c1: entering follower state: follower="Node at 743a9d18-f55d-08cd-22dc-a01953d881c1 [Follower]" leader-address= leader-id= +2026-11-29T10:28:20.135Z [INFO] server-bd512ca1-b2dd-16cf-a108-597e48327211: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0dcf0e54-f30e-e4a1-a418-29c49c7c5f85 Address:0dcf0e54-f30e-e4a1-a418-29c49c7c5f85} {Suffrage:Voter ID:server-743a9d18-f55d-08cd-22dc-a01953d881c1 Address:743a9d18-f55d-08cd-22dc-a01953d881c1} {Suffrage:Voter ID:server-bd512ca1-b2dd-16cf-a108-597e48327211 Address:bd512ca1-b2dd-16cf-a108-597e48327211} {Suffrage:Voter ID:server-4c9344db-bdca-45c4-041b-38a0d1f08793 Address:4c9344db-bdca-45c4-041b-38a0d1f08793}]" +2026-11-29T10:28:20.135Z [INFO] server-bd512ca1-b2dd-16cf-a108-597e48327211: entering follower state: follower="Node at bd512ca1-b2dd-16cf-a108-597e48327211 [Follower]" leader-address= leader-id= +2026-11-29T10:28:20.135Z [INFO] server-4c9344db-bdca-45c4-041b-38a0d1f08793: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0dcf0e54-f30e-e4a1-a418-29c49c7c5f85 Address:0dcf0e54-f30e-e4a1-a418-29c49c7c5f85} {Suffrage:Voter ID:server-743a9d18-f55d-08cd-22dc-a01953d881c1 Address:743a9d18-f55d-08cd-22dc-a01953d881c1} {Suffrage:Voter ID:server-bd512ca1-b2dd-16cf-a108-597e48327211 Address:bd512ca1-b2dd-16cf-a108-597e48327211} {Suffrage:Voter ID:server-4c9344db-bdca-45c4-041b-38a0d1f08793 Address:4c9344db-bdca-45c4-041b-38a0d1f08793}]" +2026-11-29T10:28:20.135Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:20.135Z [INFO] server-4c9344db-bdca-45c4-041b-38a0d1f08793: entering follower state: follower="Node at 4c9344db-bdca-45c4-041b-38a0d1f08793 [Follower]" leader-address= leader-id= +2026-11-29T10:28:20.186Z [WARN] server-743a9d18-f55d-08cd-22dc-a01953d881c1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:20.186Z [INFO] server-743a9d18-f55d-08cd-22dc-a01953d881c1: entering candidate state: node="Node at 743a9d18-f55d-08cd-22dc-a01953d881c1 [Candidate]" term=2 +2026-11-29T10:28:20.186Z [INFO] server-743a9d18-f55d-08cd-22dc-a01953d881c1: election won: term=2 tally=3 +2026-11-29T10:28:20.186Z [INFO] server-743a9d18-f55d-08cd-22dc-a01953d881c1: entering leader state: leader="Node at 743a9d18-f55d-08cd-22dc-a01953d881c1 [Leader]" +2026-11-29T10:28:20.186Z [INFO] server-743a9d18-f55d-08cd-22dc-a01953d881c1: added peer, starting replication: peer=server-0dcf0e54-f30e-e4a1-a418-29c49c7c5f85 +2026-11-29T10:28:20.186Z [INFO] server-743a9d18-f55d-08cd-22dc-a01953d881c1: added peer, starting replication: peer=server-bd512ca1-b2dd-16cf-a108-597e48327211 +2026-11-29T10:28:20.186Z [INFO] server-743a9d18-f55d-08cd-22dc-a01953d881c1: added peer, starting replication: peer=server-4c9344db-bdca-45c4-041b-38a0d1f08793 +2026-11-29T10:28:20.186Z [INFO] server-743a9d18-f55d-08cd-22dc-a01953d881c1: pipelining replication: peer="{Voter server-0dcf0e54-f30e-e4a1-a418-29c49c7c5f85 0dcf0e54-f30e-e4a1-a418-29c49c7c5f85}" +2026-11-29T10:28:20.186Z [INFO] server-743a9d18-f55d-08cd-22dc-a01953d881c1: pipelining replication: peer="{Voter server-4c9344db-bdca-45c4-041b-38a0d1f08793 4c9344db-bdca-45c4-041b-38a0d1f08793}" +2026-11-29T10:28:20.186Z [INFO] server-743a9d18-f55d-08cd-22dc-a01953d881c1: pipelining replication: peer="{Voter server-bd512ca1-b2dd-16cf-a108-597e48327211 bd512ca1-b2dd-16cf-a108-597e48327211}" +2026-11-29T10:28:20.291Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:20.186408379 +0000 UTC m=+10.921462521 (1 nodes), 50.93768ms from start of poll, 51.6082ms from cluster start. Timeout at 2026-11-29 10:28:20.291411539 +0000 UTC m=+11.026465721, 105.0032ms after stability +2026-11-29T10:28:20.396Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:28:20.502Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:28:20.396951659 +0000 UTC m=+11.132005841 (3 nodes), 1.6µs from start of poll, 262.15152ms from cluster start. Timeout at 2026-11-29 10:28:20.501959059 +0000 UTC m=+11.237013201, 105.00736ms after stability +2026-11-29T10:28:20.503Z [INFO] server-743a9d18-f55d-08cd-22dc-a01953d881c1: updating configuration: command=RemoveServer server-id=server-4c9344db-bdca-45c4-041b-38a0d1f08793 server-addr= servers="[{Suffrage:Voter ID:server-0dcf0e54-f30e-e4a1-a418-29c49c7c5f85 Address:0dcf0e54-f30e-e4a1-a418-29c49c7c5f85} {Suffrage:Voter ID:server-743a9d18-f55d-08cd-22dc-a01953d881c1 Address:743a9d18-f55d-08cd-22dc-a01953d881c1} {Suffrage:Voter ID:server-bd512ca1-b2dd-16cf-a108-597e48327211 Address:bd512ca1-b2dd-16cf-a108-597e48327211}]" +2026-11-29T10:28:20.503Z [INFO] server-743a9d18-f55d-08cd-22dc-a01953d881c1: removed peer, stopping replication: peer=server-4c9344db-bdca-45c4-041b-38a0d1f08793 last-index=3 +2026-11-29T10:28:20.503Z [ERROR] server-743a9d18-f55d-08cd-22dc-a01953d881c1: failed to pipeline appendEntries: peer="{Voter server-0dcf0e54-f30e-e4a1-a418-29c49c7c5f85 0dcf0e54-f30e-e4a1-a418-29c49c7c5f85}" error="append pipeline closed" +2026-11-29T10:28:20.503Z [INFO] server-743a9d18-f55d-08cd-22dc-a01953d881c1: aborting pipeline replication: peer="{Voter server-0dcf0e54-f30e-e4a1-a418-29c49c7c5f85 0dcf0e54-f30e-e4a1-a418-29c49c7c5f85}" +2026-11-29T10:28:20.503Z [INFO] server-743a9d18-f55d-08cd-22dc-a01953d881c1: aborting pipeline replication: peer="{Voter server-4c9344db-bdca-45c4-041b-38a0d1f08793 4c9344db-bdca-45c4-041b-38a0d1f08793}" +2026-11-29T10:28:20.503Z [ERROR] server-743a9d18-f55d-08cd-22dc-a01953d881c1: failed to pipeline appendEntries: peer="{Voter server-bd512ca1-b2dd-16cf-a108-597e48327211 bd512ca1-b2dd-16cf-a108-597e48327211}" error="append pipeline closed" +2026-11-29T10:28:20.503Z [INFO] server-743a9d18-f55d-08cd-22dc-a01953d881c1: aborting pipeline replication: peer="{Voter server-bd512ca1-b2dd-16cf-a108-597e48327211 bd512ca1-b2dd-16cf-a108-597e48327211}" +2026-11-29T10:28:20.505Z [ERROR] server-743a9d18-f55d-08cd-22dc-a01953d881c1: failed to heartbeat to: peer=0dcf0e54-f30e-e4a1-a418-29c49c7c5f85 backoff time=10ms error="failed to connect to peer: 0dcf0e54-f30e-e4a1-a418-29c49c7c5f85" +2026-11-29T10:28:20.506Z [ERROR] server-743a9d18-f55d-08cd-22dc-a01953d881c1: failed to heartbeat to: peer=bd512ca1-b2dd-16cf-a108-597e48327211 backoff time=10ms error="failed to connect to peer: bd512ca1-b2dd-16cf-a108-597e48327211" +2026-11-29T10:28:20.511Z [ERROR] server-743a9d18-f55d-08cd-22dc-a01953d881c1: failed to appendEntries to: peer="{Voter server-0dcf0e54-f30e-e4a1-a418-29c49c7c5f85 0dcf0e54-f30e-e4a1-a418-29c49c7c5f85}" error="failed to connect to peer: 0dcf0e54-f30e-e4a1-a418-29c49c7c5f85" +2026-11-29T10:28:20.511Z [ERROR] server-743a9d18-f55d-08cd-22dc-a01953d881c1: failed to appendEntries to: peer="{Voter server-bd512ca1-b2dd-16cf-a108-597e48327211 bd512ca1-b2dd-16cf-a108-597e48327211}" error="failed to connect to peer: bd512ca1-b2dd-16cf-a108-597e48327211" +2026-11-29T10:28:20.524Z [ERROR] server-743a9d18-f55d-08cd-22dc-a01953d881c1: failed to heartbeat to: peer=0dcf0e54-f30e-e4a1-a418-29c49c7c5f85 backoff time=10ms error="failed to connect to peer: 0dcf0e54-f30e-e4a1-a418-29c49c7c5f85" +2026-11-29T10:28:20.526Z [ERROR] server-743a9d18-f55d-08cd-22dc-a01953d881c1: failed to heartbeat to: peer=bd512ca1-b2dd-16cf-a108-597e48327211 backoff time=10ms error="failed to connect to peer: bd512ca1-b2dd-16cf-a108-597e48327211" +2026-11-29T10:28:20.527Z [ERROR] server-743a9d18-f55d-08cd-22dc-a01953d881c1: failed to appendEntries to: peer="{Voter server-0dcf0e54-f30e-e4a1-a418-29c49c7c5f85 0dcf0e54-f30e-e4a1-a418-29c49c7c5f85}" error="failed to connect to peer: 0dcf0e54-f30e-e4a1-a418-29c49c7c5f85" +2026-11-29T10:28:20.529Z [ERROR] server-743a9d18-f55d-08cd-22dc-a01953d881c1: failed to appendEntries to: peer="{Voter server-bd512ca1-b2dd-16cf-a108-597e48327211 bd512ca1-b2dd-16cf-a108-597e48327211}" error="failed to connect to peer: bd512ca1-b2dd-16cf-a108-597e48327211" +2026-11-29T10:28:20.541Z [ERROR] server-743a9d18-f55d-08cd-22dc-a01953d881c1: failed to heartbeat to: peer=0dcf0e54-f30e-e4a1-a418-29c49c7c5f85 backoff time=10ms error="failed to connect to peer: 0dcf0e54-f30e-e4a1-a418-29c49c7c5f85" +2026-11-29T10:28:20.544Z [ERROR] server-743a9d18-f55d-08cd-22dc-a01953d881c1: failed to heartbeat to: peer=bd512ca1-b2dd-16cf-a108-597e48327211 backoff time=10ms error="failed to connect to peer: bd512ca1-b2dd-16cf-a108-597e48327211" +2026-11-29T10:28:20.544Z [ERROR] server-743a9d18-f55d-08cd-22dc-a01953d881c1: failed to appendEntries to: peer="{Voter server-0dcf0e54-f30e-e4a1-a418-29c49c7c5f85 0dcf0e54-f30e-e4a1-a418-29c49c7c5f85}" error="failed to connect to peer: 0dcf0e54-f30e-e4a1-a418-29c49c7c5f85" +2026-11-29T10:28:20.547Z [WARN] server-743a9d18-f55d-08cd-22dc-a01953d881c1: failed to contact: server-id=server-0dcf0e54-f30e-e4a1-a418-29c49c7c5f85 time=51.04032ms +2026-11-29T10:28:20.547Z [ERROR] server-743a9d18-f55d-08cd-22dc-a01953d881c1: failed to appendEntries to: peer="{Voter server-bd512ca1-b2dd-16cf-a108-597e48327211 bd512ca1-b2dd-16cf-a108-597e48327211}" error="failed to connect to peer: bd512ca1-b2dd-16cf-a108-597e48327211" +2026-11-29T10:28:20.558Z [WARN] server-743a9d18-f55d-08cd-22dc-a01953d881c1: failed to contact: server-id=server-0dcf0e54-f30e-e4a1-a418-29c49c7c5f85 time=62.04168ms +2026-11-29T10:28:20.558Z [WARN] server-743a9d18-f55d-08cd-22dc-a01953d881c1: failed to contact: server-id=server-bd512ca1-b2dd-16cf-a108-597e48327211 time=60.83184ms +2026-11-29T10:28:20.558Z [WARN] server-743a9d18-f55d-08cd-22dc-a01953d881c1: failed to contact quorum of nodes, stepping down +2026-11-29T10:28:20.558Z [INFO] server-743a9d18-f55d-08cd-22dc-a01953d881c1: entering follower state: follower="Node at 743a9d18-f55d-08cd-22dc-a01953d881c1 [Follower]" leader-address= leader-id= +2026-11-29T10:28:20.558Z [ERROR] server-743a9d18-f55d-08cd-22dc-a01953d881c1: failed to appendEntries to: peer="{Voter server-bd512ca1-b2dd-16cf-a108-597e48327211 bd512ca1-b2dd-16cf-a108-597e48327211}" error="failed to connect to peer: bd512ca1-b2dd-16cf-a108-597e48327211" +2026-11-29T10:28:20.558Z [ERROR] server-743a9d18-f55d-08cd-22dc-a01953d881c1: failed to appendEntries to: peer="{Voter server-0dcf0e54-f30e-e4a1-a418-29c49c7c5f85 0dcf0e54-f30e-e4a1-a418-29c49c7c5f85}" error="failed to connect to peer: 0dcf0e54-f30e-e4a1-a418-29c49c7c5f85" --- PASS: TestRaft_RemoveFollower_SplitCluster (0.43s) === RUN TestRaft_AddKnownPeer -2025-10-27T04:02:17.403Z [INFO] server-00b60ca9-cb7c-7243-94c1-cd3bb3493b1f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-00b60ca9-cb7c-7243-94c1-cd3bb3493b1f Address:00b60ca9-cb7c-7243-94c1-cd3bb3493b1f} {Suffrage:Voter ID:server-9f5b5aff-27b6-4b90-51e9-055be529a6b1 Address:9f5b5aff-27b6-4b90-51e9-055be529a6b1} {Suffrage:Voter ID:server-01c804d8-018b-0b41-0878-7c050b1a2b54 Address:01c804d8-018b-0b41-0878-7c050b1a2b54}]" -2025-10-27T04:02:17.403Z [INFO] server-9f5b5aff-27b6-4b90-51e9-055be529a6b1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-00b60ca9-cb7c-7243-94c1-cd3bb3493b1f Address:00b60ca9-cb7c-7243-94c1-cd3bb3493b1f} {Suffrage:Voter ID:server-9f5b5aff-27b6-4b90-51e9-055be529a6b1 Address:9f5b5aff-27b6-4b90-51e9-055be529a6b1} {Suffrage:Voter ID:server-01c804d8-018b-0b41-0878-7c050b1a2b54 Address:01c804d8-018b-0b41-0878-7c050b1a2b54}]" -2025-10-27T04:02:17.403Z [INFO] server-01c804d8-018b-0b41-0878-7c050b1a2b54: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-00b60ca9-cb7c-7243-94c1-cd3bb3493b1f Address:00b60ca9-cb7c-7243-94c1-cd3bb3493b1f} {Suffrage:Voter ID:server-9f5b5aff-27b6-4b90-51e9-055be529a6b1 Address:9f5b5aff-27b6-4b90-51e9-055be529a6b1} {Suffrage:Voter ID:server-01c804d8-018b-0b41-0878-7c050b1a2b54 Address:01c804d8-018b-0b41-0878-7c050b1a2b54}]" -2025-10-27T04:02:17.403Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:17.403Z [INFO] server-00b60ca9-cb7c-7243-94c1-cd3bb3493b1f: entering follower state: follower="Node at 00b60ca9-cb7c-7243-94c1-cd3bb3493b1f [Follower]" leader-address= leader-id= -2025-10-27T04:02:17.403Z [INFO] server-9f5b5aff-27b6-4b90-51e9-055be529a6b1: entering follower state: follower="Node at 9f5b5aff-27b6-4b90-51e9-055be529a6b1 [Follower]" leader-address= leader-id= -2025-10-27T04:02:17.403Z [INFO] server-01c804d8-018b-0b41-0878-7c050b1a2b54: entering follower state: follower="Node at 01c804d8-018b-0b41-0878-7c050b1a2b54 [Follower]" leader-address= leader-id= -2025-10-27T04:02:17.458Z [WARN] server-00b60ca9-cb7c-7243-94c1-cd3bb3493b1f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:17.458Z [INFO] server-00b60ca9-cb7c-7243-94c1-cd3bb3493b1f: entering candidate state: node="Node at 00b60ca9-cb7c-7243-94c1-cd3bb3493b1f [Candidate]" term=2 -2025-10-27T04:02:17.458Z [INFO] server-00b60ca9-cb7c-7243-94c1-cd3bb3493b1f: election won: term=2 tally=2 -2025-10-27T04:02:17.458Z [INFO] server-00b60ca9-cb7c-7243-94c1-cd3bb3493b1f: entering leader state: leader="Node at 00b60ca9-cb7c-7243-94c1-cd3bb3493b1f [Leader]" -2025-10-27T04:02:17.458Z [INFO] server-00b60ca9-cb7c-7243-94c1-cd3bb3493b1f: added peer, starting replication: peer=server-9f5b5aff-27b6-4b90-51e9-055be529a6b1 -2025-10-27T04:02:17.458Z [INFO] server-00b60ca9-cb7c-7243-94c1-cd3bb3493b1f: added peer, starting replication: peer=server-01c804d8-018b-0b41-0878-7c050b1a2b54 -2025-10-27T04:02:17.458Z [INFO] server-00b60ca9-cb7c-7243-94c1-cd3bb3493b1f: pipelining replication: peer="{Voter server-01c804d8-018b-0b41-0878-7c050b1a2b54 01c804d8-018b-0b41-0878-7c050b1a2b54}" -2025-10-27T04:02:17.458Z [INFO] server-00b60ca9-cb7c-7243-94c1-cd3bb3493b1f: pipelining replication: peer="{Voter server-9f5b5aff-27b6-4b90-51e9-055be529a6b1 9f5b5aff-27b6-4b90-51e9-055be529a6b1}" -2025-10-27T04:02:17.564Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:17.458798565 +0000 UTC m=+12.453671003 (1 nodes), 55.110737ms from start of poll, 55.302258ms from cluster start. Timeout at 2025-10-27 04:02:17.563800565 +0000 UTC m=+12.558672963, 105.00196ms after stability -2025-10-27T04:02:17.564Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:02:17.670Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:02:17.564326246 +0000 UTC m=+12.559198684 (2 nodes), 1.72µs from start of poll, 160.829939ms from cluster start. Timeout at 2025-10-27 04:02:17.669329926 +0000 UTC m=+12.664202364, 105.00368ms after stability -2025-10-27T04:02:17.670Z [INFO] server-00b60ca9-cb7c-7243-94c1-cd3bb3493b1f: updating configuration: command=AddVoter server-id=server-9f5b5aff-27b6-4b90-51e9-055be529a6b1 server-addr=9f5b5aff-27b6-4b90-51e9-055be529a6b1 servers="[{Suffrage:Voter ID:server-00b60ca9-cb7c-7243-94c1-cd3bb3493b1f Address:00b60ca9-cb7c-7243-94c1-cd3bb3493b1f} {Suffrage:Voter ID:server-9f5b5aff-27b6-4b90-51e9-055be529a6b1 Address:9f5b5aff-27b6-4b90-51e9-055be529a6b1} {Suffrage:Voter ID:server-01c804d8-018b-0b41-0878-7c050b1a2b54 Address:01c804d8-018b-0b41-0878-7c050b1a2b54}]" -2025-10-27T04:02:17.670Z [INFO] server-00b60ca9-cb7c-7243-94c1-cd3bb3493b1f: aborting pipeline replication: peer="{Voter server-01c804d8-018b-0b41-0878-7c050b1a2b54 01c804d8-018b-0b41-0878-7c050b1a2b54}" -2025-10-27T04:02:17.670Z [INFO] server-00b60ca9-cb7c-7243-94c1-cd3bb3493b1f: aborting pipeline replication: peer="{Voter server-9f5b5aff-27b6-4b90-51e9-055be529a6b1 9f5b5aff-27b6-4b90-51e9-055be529a6b1}" +2026-11-29T10:28:20.559Z [INFO] server-e3d90b4f-89dc-1d32-55fc-46300d04f34b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e3d90b4f-89dc-1d32-55fc-46300d04f34b Address:e3d90b4f-89dc-1d32-55fc-46300d04f34b} {Suffrage:Voter ID:server-001e6e5b-506c-e596-a00f-8d61b21572d1 Address:001e6e5b-506c-e596-a00f-8d61b21572d1} {Suffrage:Voter ID:server-1237459b-2923-6a82-3a67-a6d3a71e4702 Address:1237459b-2923-6a82-3a67-a6d3a71e4702}]" +2026-11-29T10:28:20.559Z [INFO] server-e3d90b4f-89dc-1d32-55fc-46300d04f34b: entering follower state: follower="Node at e3d90b4f-89dc-1d32-55fc-46300d04f34b [Follower]" leader-address= leader-id= +2026-11-29T10:28:20.559Z [INFO] server-001e6e5b-506c-e596-a00f-8d61b21572d1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e3d90b4f-89dc-1d32-55fc-46300d04f34b Address:e3d90b4f-89dc-1d32-55fc-46300d04f34b} {Suffrage:Voter ID:server-001e6e5b-506c-e596-a00f-8d61b21572d1 Address:001e6e5b-506c-e596-a00f-8d61b21572d1} {Suffrage:Voter ID:server-1237459b-2923-6a82-3a67-a6d3a71e4702 Address:1237459b-2923-6a82-3a67-a6d3a71e4702}]" +2026-11-29T10:28:20.559Z [INFO] server-001e6e5b-506c-e596-a00f-8d61b21572d1: entering follower state: follower="Node at 001e6e5b-506c-e596-a00f-8d61b21572d1 [Follower]" leader-address= leader-id= +2026-11-29T10:28:20.559Z [INFO] server-1237459b-2923-6a82-3a67-a6d3a71e4702: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e3d90b4f-89dc-1d32-55fc-46300d04f34b Address:e3d90b4f-89dc-1d32-55fc-46300d04f34b} {Suffrage:Voter ID:server-001e6e5b-506c-e596-a00f-8d61b21572d1 Address:001e6e5b-506c-e596-a00f-8d61b21572d1} {Suffrage:Voter ID:server-1237459b-2923-6a82-3a67-a6d3a71e4702 Address:1237459b-2923-6a82-3a67-a6d3a71e4702}]" +2026-11-29T10:28:20.559Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:20.559Z [INFO] server-1237459b-2923-6a82-3a67-a6d3a71e4702: entering follower state: follower="Node at 1237459b-2923-6a82-3a67-a6d3a71e4702 [Follower]" leader-address= leader-id= +2026-11-29T10:28:20.619Z [WARN] server-1237459b-2923-6a82-3a67-a6d3a71e4702: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:20.619Z [INFO] server-1237459b-2923-6a82-3a67-a6d3a71e4702: entering candidate state: node="Node at 1237459b-2923-6a82-3a67-a6d3a71e4702 [Candidate]" term=2 +2026-11-29T10:28:20.619Z [INFO] server-1237459b-2923-6a82-3a67-a6d3a71e4702: election won: term=2 tally=2 +2026-11-29T10:28:20.619Z [INFO] server-1237459b-2923-6a82-3a67-a6d3a71e4702: entering leader state: leader="Node at 1237459b-2923-6a82-3a67-a6d3a71e4702 [Leader]" +2026-11-29T10:28:20.619Z [INFO] server-1237459b-2923-6a82-3a67-a6d3a71e4702: added peer, starting replication: peer=server-e3d90b4f-89dc-1d32-55fc-46300d04f34b +2026-11-29T10:28:20.619Z [INFO] server-1237459b-2923-6a82-3a67-a6d3a71e4702: added peer, starting replication: peer=server-001e6e5b-506c-e596-a00f-8d61b21572d1 +2026-11-29T10:28:20.619Z [INFO] server-1237459b-2923-6a82-3a67-a6d3a71e4702: pipelining replication: peer="{Voter server-001e6e5b-506c-e596-a00f-8d61b21572d1 001e6e5b-506c-e596-a00f-8d61b21572d1}" +2026-11-29T10:28:20.619Z [INFO] server-1237459b-2923-6a82-3a67-a6d3a71e4702: pipelining replication: peer="{Voter server-e3d90b4f-89dc-1d32-55fc-46300d04f34b e3d90b4f-89dc-1d32-55fc-46300d04f34b}" +2026-11-29T10:28:20.725Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:20.619470979 +0000 UTC m=+11.354525121 (1 nodes), 59.67112ms from start of poll, 60.1886ms from cluster start. Timeout at 2026-11-29 10:28:20.724472819 +0000 UTC m=+11.459526921, 105.0018ms after stability +2026-11-29T10:28:20.725Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:28:20.831Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:28:20.725520379 +0000 UTC m=+11.460574481 (2 nodes), 1.92µs from start of poll, 166.23796ms from cluster start. Timeout at 2026-11-29 10:28:20.830526299 +0000 UTC m=+11.565580401, 105.00592ms after stability +2026-11-29T10:28:20.831Z [INFO] server-1237459b-2923-6a82-3a67-a6d3a71e4702: updating configuration: command=AddVoter server-id=server-e3d90b4f-89dc-1d32-55fc-46300d04f34b server-addr=e3d90b4f-89dc-1d32-55fc-46300d04f34b servers="[{Suffrage:Voter ID:server-e3d90b4f-89dc-1d32-55fc-46300d04f34b Address:e3d90b4f-89dc-1d32-55fc-46300d04f34b} {Suffrage:Voter ID:server-001e6e5b-506c-e596-a00f-8d61b21572d1 Address:001e6e5b-506c-e596-a00f-8d61b21572d1} {Suffrage:Voter ID:server-1237459b-2923-6a82-3a67-a6d3a71e4702 Address:1237459b-2923-6a82-3a67-a6d3a71e4702}]" +2026-11-29T10:28:20.831Z [INFO] server-1237459b-2923-6a82-3a67-a6d3a71e4702: aborting pipeline replication: peer="{Voter server-e3d90b4f-89dc-1d32-55fc-46300d04f34b e3d90b4f-89dc-1d32-55fc-46300d04f34b}" +2026-11-29T10:28:20.831Z [INFO] server-1237459b-2923-6a82-3a67-a6d3a71e4702: aborting pipeline replication: peer="{Voter server-001e6e5b-506c-e596-a00f-8d61b21572d1 001e6e5b-506c-e596-a00f-8d61b21572d1}" --- PASS: TestRaft_AddKnownPeer (0.27s) === RUN TestRaft_RemoveUnknownPeer -2025-10-27T04:02:17.670Z [INFO] server-ffc64619-ecac-7d00-44d4-937f1b873f51: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ffc64619-ecac-7d00-44d4-937f1b873f51 Address:ffc64619-ecac-7d00-44d4-937f1b873f51} {Suffrage:Voter ID:server-7e96d8f4-5b2d-516a-09a7-f0040e820964 Address:7e96d8f4-5b2d-516a-09a7-f0040e820964} {Suffrage:Voter ID:server-ba63f00c-73af-3095-7e7f-101e93b07ee3 Address:ba63f00c-73af-3095-7e7f-101e93b07ee3}]" -2025-10-27T04:02:17.670Z [INFO] server-7e96d8f4-5b2d-516a-09a7-f0040e820964: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ffc64619-ecac-7d00-44d4-937f1b873f51 Address:ffc64619-ecac-7d00-44d4-937f1b873f51} {Suffrage:Voter ID:server-7e96d8f4-5b2d-516a-09a7-f0040e820964 Address:7e96d8f4-5b2d-516a-09a7-f0040e820964} {Suffrage:Voter ID:server-ba63f00c-73af-3095-7e7f-101e93b07ee3 Address:ba63f00c-73af-3095-7e7f-101e93b07ee3}]" -2025-10-27T04:02:17.670Z [INFO] server-ba63f00c-73af-3095-7e7f-101e93b07ee3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ffc64619-ecac-7d00-44d4-937f1b873f51 Address:ffc64619-ecac-7d00-44d4-937f1b873f51} {Suffrage:Voter ID:server-7e96d8f4-5b2d-516a-09a7-f0040e820964 Address:7e96d8f4-5b2d-516a-09a7-f0040e820964} {Suffrage:Voter ID:server-ba63f00c-73af-3095-7e7f-101e93b07ee3 Address:ba63f00c-73af-3095-7e7f-101e93b07ee3}]" -2025-10-27T04:02:17.670Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:17.670Z [INFO] server-ffc64619-ecac-7d00-44d4-937f1b873f51: entering follower state: follower="Node at ffc64619-ecac-7d00-44d4-937f1b873f51 [Follower]" leader-address= leader-id= -2025-10-27T04:02:17.670Z [INFO] server-7e96d8f4-5b2d-516a-09a7-f0040e820964: entering follower state: follower="Node at 7e96d8f4-5b2d-516a-09a7-f0040e820964 [Follower]" leader-address= leader-id= -2025-10-27T04:02:17.670Z [INFO] server-ba63f00c-73af-3095-7e7f-101e93b07ee3: entering follower state: follower="Node at ba63f00c-73af-3095-7e7f-101e93b07ee3 [Follower]" leader-address= leader-id= -2025-10-27T04:02:17.733Z [WARN] server-7e96d8f4-5b2d-516a-09a7-f0040e820964: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:17.733Z [INFO] server-7e96d8f4-5b2d-516a-09a7-f0040e820964: entering candidate state: node="Node at 7e96d8f4-5b2d-516a-09a7-f0040e820964 [Candidate]" term=2 -2025-10-27T04:02:17.733Z [INFO] server-7e96d8f4-5b2d-516a-09a7-f0040e820964: election won: term=2 tally=2 -2025-10-27T04:02:17.733Z [INFO] server-7e96d8f4-5b2d-516a-09a7-f0040e820964: entering leader state: leader="Node at 7e96d8f4-5b2d-516a-09a7-f0040e820964 [Leader]" -2025-10-27T04:02:17.733Z [INFO] server-7e96d8f4-5b2d-516a-09a7-f0040e820964: added peer, starting replication: peer=server-ffc64619-ecac-7d00-44d4-937f1b873f51 -2025-10-27T04:02:17.733Z [INFO] server-7e96d8f4-5b2d-516a-09a7-f0040e820964: added peer, starting replication: peer=server-ba63f00c-73af-3095-7e7f-101e93b07ee3 -2025-10-27T04:02:17.733Z [INFO] server-7e96d8f4-5b2d-516a-09a7-f0040e820964: pipelining replication: peer="{Voter server-ba63f00c-73af-3095-7e7f-101e93b07ee3 ba63f00c-73af-3095-7e7f-101e93b07ee3}" -2025-10-27T04:02:17.734Z [INFO] server-7e96d8f4-5b2d-516a-09a7-f0040e820964: pipelining replication: peer="{Voter server-ffc64619-ecac-7d00-44d4-937f1b873f51 ffc64619-ecac-7d00-44d4-937f1b873f51}" -2025-10-27T04:02:17.734Z [WARN] server-ffc64619-ecac-7d00-44d4-937f1b873f51: rejecting vote request since our last term is greater: candidate=7e96d8f4-5b2d-516a-09a7-f0040e820964 last-term=2 last-candidate-term=1 -2025-10-27T04:02:17.839Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:17.734085767 +0000 UTC m=+12.728958165 (1 nodes), 63.337313ms from start of poll, 63.473194ms from cluster start. Timeout at 2025-10-27 04:02:17.839088287 +0000 UTC m=+12.833960725, 105.00256ms after stability -2025-10-27T04:02:17.839Z [INFO] server-7e96d8f4-5b2d-516a-09a7-f0040e820964: updating configuration: command=RemoveServer server-id=8e6969c4-baad-8f2b-8c24-7936dba90eb8 server-addr= servers="[{Suffrage:Voter ID:server-ffc64619-ecac-7d00-44d4-937f1b873f51 Address:ffc64619-ecac-7d00-44d4-937f1b873f51} {Suffrage:Voter ID:server-7e96d8f4-5b2d-516a-09a7-f0040e820964 Address:7e96d8f4-5b2d-516a-09a7-f0040e820964} {Suffrage:Voter ID:server-ba63f00c-73af-3095-7e7f-101e93b07ee3 Address:ba63f00c-73af-3095-7e7f-101e93b07ee3}]" -2025-10-27T04:02:17.839Z [INFO] server-7e96d8f4-5b2d-516a-09a7-f0040e820964: aborting pipeline replication: peer="{Voter server-ba63f00c-73af-3095-7e7f-101e93b07ee3 ba63f00c-73af-3095-7e7f-101e93b07ee3}" -2025-10-27T04:02:17.839Z [INFO] server-7e96d8f4-5b2d-516a-09a7-f0040e820964: aborting pipeline replication: peer="{Voter server-ffc64619-ecac-7d00-44d4-937f1b873f51 ffc64619-ecac-7d00-44d4-937f1b873f51}" ---- PASS: TestRaft_RemoveUnknownPeer (0.17s) +2026-11-29T10:28:20.832Z [INFO] server-faade221-d241-96c3-3674-f08bbe7e2e9e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-faade221-d241-96c3-3674-f08bbe7e2e9e Address:faade221-d241-96c3-3674-f08bbe7e2e9e} {Suffrage:Voter ID:server-1e2c1360-9e6e-b8c8-897f-2022603fc571 Address:1e2c1360-9e6e-b8c8-897f-2022603fc571} {Suffrage:Voter ID:server-6f2f817b-3d6d-cfdb-1f6d-a694578a115e Address:6f2f817b-3d6d-cfdb-1f6d-a694578a115e}]" +2026-11-29T10:28:20.832Z [INFO] server-faade221-d241-96c3-3674-f08bbe7e2e9e: entering follower state: follower="Node at faade221-d241-96c3-3674-f08bbe7e2e9e [Follower]" leader-address= leader-id= +2026-11-29T10:28:20.832Z [INFO] server-1e2c1360-9e6e-b8c8-897f-2022603fc571: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-faade221-d241-96c3-3674-f08bbe7e2e9e Address:faade221-d241-96c3-3674-f08bbe7e2e9e} {Suffrage:Voter ID:server-1e2c1360-9e6e-b8c8-897f-2022603fc571 Address:1e2c1360-9e6e-b8c8-897f-2022603fc571} {Suffrage:Voter ID:server-6f2f817b-3d6d-cfdb-1f6d-a694578a115e Address:6f2f817b-3d6d-cfdb-1f6d-a694578a115e}]" +2026-11-29T10:28:20.832Z [INFO] server-1e2c1360-9e6e-b8c8-897f-2022603fc571: entering follower state: follower="Node at 1e2c1360-9e6e-b8c8-897f-2022603fc571 [Follower]" leader-address= leader-id= +2026-11-29T10:28:20.832Z [INFO] server-6f2f817b-3d6d-cfdb-1f6d-a694578a115e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-faade221-d241-96c3-3674-f08bbe7e2e9e Address:faade221-d241-96c3-3674-f08bbe7e2e9e} {Suffrage:Voter ID:server-1e2c1360-9e6e-b8c8-897f-2022603fc571 Address:1e2c1360-9e6e-b8c8-897f-2022603fc571} {Suffrage:Voter ID:server-6f2f817b-3d6d-cfdb-1f6d-a694578a115e Address:6f2f817b-3d6d-cfdb-1f6d-a694578a115e}]" +2026-11-29T10:28:20.832Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:20.832Z [INFO] server-6f2f817b-3d6d-cfdb-1f6d-a694578a115e: entering follower state: follower="Node at 6f2f817b-3d6d-cfdb-1f6d-a694578a115e [Follower]" leader-address= leader-id= +2026-11-29T10:28:20.885Z [WARN] server-faade221-d241-96c3-3674-f08bbe7e2e9e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:20.885Z [INFO] server-faade221-d241-96c3-3674-f08bbe7e2e9e: entering candidate state: node="Node at faade221-d241-96c3-3674-f08bbe7e2e9e [Candidate]" term=2 +2026-11-29T10:28:20.885Z [INFO] server-faade221-d241-96c3-3674-f08bbe7e2e9e: election won: term=2 tally=2 +2026-11-29T10:28:20.885Z [INFO] server-faade221-d241-96c3-3674-f08bbe7e2e9e: entering leader state: leader="Node at faade221-d241-96c3-3674-f08bbe7e2e9e [Leader]" +2026-11-29T10:28:20.885Z [INFO] server-faade221-d241-96c3-3674-f08bbe7e2e9e: added peer, starting replication: peer=server-1e2c1360-9e6e-b8c8-897f-2022603fc571 +2026-11-29T10:28:20.885Z [INFO] server-faade221-d241-96c3-3674-f08bbe7e2e9e: added peer, starting replication: peer=server-6f2f817b-3d6d-cfdb-1f6d-a694578a115e +2026-11-29T10:28:20.885Z [INFO] server-faade221-d241-96c3-3674-f08bbe7e2e9e: pipelining replication: peer="{Voter server-6f2f817b-3d6d-cfdb-1f6d-a694578a115e 6f2f817b-3d6d-cfdb-1f6d-a694578a115e}" +2026-11-29T10:28:20.885Z [INFO] server-faade221-d241-96c3-3674-f08bbe7e2e9e: pipelining replication: peer="{Voter server-1e2c1360-9e6e-b8c8-897f-2022603fc571 1e2c1360-9e6e-b8c8-897f-2022603fc571}" +2026-11-29T10:28:20.991Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:20.885656859 +0000 UTC m=+11.620711001 (1 nodes), 53.12612ms from start of poll, 53.65716ms from cluster start. Timeout at 2026-11-29 10:28:20.990660539 +0000 UTC m=+11.725714681, 105.00368ms after stability +2026-11-29T10:28:20.991Z [INFO] server-faade221-d241-96c3-3674-f08bbe7e2e9e: updating configuration: command=RemoveServer server-id=a0dd3ba0-34bc-b0ce-c4a9-061191a01700 server-addr= servers="[{Suffrage:Voter ID:server-faade221-d241-96c3-3674-f08bbe7e2e9e Address:faade221-d241-96c3-3674-f08bbe7e2e9e} {Suffrage:Voter ID:server-1e2c1360-9e6e-b8c8-897f-2022603fc571 Address:1e2c1360-9e6e-b8c8-897f-2022603fc571} {Suffrage:Voter ID:server-6f2f817b-3d6d-cfdb-1f6d-a694578a115e Address:6f2f817b-3d6d-cfdb-1f6d-a694578a115e}]" +2026-11-29T10:28:20.991Z [INFO] server-faade221-d241-96c3-3674-f08bbe7e2e9e: aborting pipeline replication: peer="{Voter server-6f2f817b-3d6d-cfdb-1f6d-a694578a115e 6f2f817b-3d6d-cfdb-1f6d-a694578a115e}" +2026-11-29T10:28:20.991Z [INFO] server-faade221-d241-96c3-3674-f08bbe7e2e9e: aborting pipeline replication: peer="{Voter server-1e2c1360-9e6e-b8c8-897f-2022603fc571 1e2c1360-9e6e-b8c8-897f-2022603fc571}" +--- PASS: TestRaft_RemoveUnknownPeer (0.16s) === RUN TestRaft_SnapshotRestore -2025-10-27T04:02:17.839Z [INFO] server-8f75ac5b-dded-75ee-14f5-283bca33453c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8f75ac5b-dded-75ee-14f5-283bca33453c Address:8f75ac5b-dded-75ee-14f5-283bca33453c}]" -2025-10-27T04:02:17.839Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:17.840Z [INFO] server-8f75ac5b-dded-75ee-14f5-283bca33453c: entering follower state: follower="Node at 8f75ac5b-dded-75ee-14f5-283bca33453c [Follower]" leader-address= leader-id= -2025-10-27T04:02:17.933Z [WARN] server-8f75ac5b-dded-75ee-14f5-283bca33453c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:17.933Z [INFO] server-8f75ac5b-dded-75ee-14f5-283bca33453c: entering candidate state: node="Node at 8f75ac5b-dded-75ee-14f5-283bca33453c [Candidate]" term=2 -2025-10-27T04:02:17.933Z [INFO] server-8f75ac5b-dded-75ee-14f5-283bca33453c: election won: term=2 tally=1 -2025-10-27T04:02:17.933Z [INFO] server-8f75ac5b-dded-75ee-14f5-283bca33453c: entering leader state: leader="Node at 8f75ac5b-dded-75ee-14f5-283bca33453c [Leader]" -2025-10-27T04:02:18.039Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:17.934024533 +0000 UTC m=+12.928896971 (1 nodes), 94.139443ms from start of poll, 94.352685ms from cluster start. Timeout at 2025-10-27 04:02:18.039026813 +0000 UTC m=+13.033899211, 105.00224ms after stability -2025-10-27T04:02:18.041Z [INFO] server-8f75ac5b-dded-75ee-14f5-283bca33453c: starting snapshot up to: index=102 -2025-10-27T04:02:18.041Z [INFO] creating new snapshot: path=/tmp/raft3928701263/snapshots/2-102-1761537738041.tmp -2025-10-27T04:02:18.041Z [INFO] server-8f75ac5b-dded-75ee-14f5-283bca33453c: compacting logs: from=1 to=92 -2025-10-27T04:02:18.041Z [INFO] server-8f75ac5b-dded-75ee-14f5-283bca33453c: snapshot complete up to: index=102 -2025-10-27T04:02:18.041Z [INFO] server-8f75ac5b-dded-75ee-14f5-283bca33453c: starting restore from snapshot: id=2-102-1761537738041 last-index=102 last-term=2 size-in-bytes=693 -2025-10-27T04:02:18.041Z [INFO] server-8f75ac5b-dded-75ee-14f5-283bca33453c: snapshot restore progress: id=2-102-1761537738041 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% -2025-10-27T04:02:18.041Z [INFO] server-8f75ac5b-dded-75ee-14f5-283bca33453c: restored from snapshot: id=2-102-1761537738041 last-index=102 last-term=2 size-in-bytes=693 -2025-10-27T04:02:18.041Z [INFO] server-8f75ac5b-dded-75ee-14f5-283bca33453c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8f75ac5b-dded-75ee-14f5-283bca33453c Address:8f75ac5b-dded-75ee-14f5-283bca33453c}]" -2025-10-27T04:02:18.041Z [INFO] server-8f75ac5b-dded-75ee-14f5-283bca33453c: entering follower state: follower="Node at 8f75ac5b-dded-75ee-14f5-283bca33453c [Follower]" leader-address= leader-id= ---- PASS: TestRaft_SnapshotRestore (0.20s) +2026-11-29T10:28:20.991Z [INFO] server-1132bc9e-5197-c753-bea1-34270d7d3076: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1132bc9e-5197-c753-bea1-34270d7d3076 Address:1132bc9e-5197-c753-bea1-34270d7d3076}]" +2026-11-29T10:28:20.991Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:20.991Z [INFO] server-1132bc9e-5197-c753-bea1-34270d7d3076: entering follower state: follower="Node at 1132bc9e-5197-c753-bea1-34270d7d3076 [Follower]" leader-address= leader-id= +2026-11-29T10:28:21.089Z [WARN] server-1132bc9e-5197-c753-bea1-34270d7d3076: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:21.089Z [INFO] server-1132bc9e-5197-c753-bea1-34270d7d3076: entering candidate state: node="Node at 1132bc9e-5197-c753-bea1-34270d7d3076 [Candidate]" term=2 +2026-11-29T10:28:21.089Z [INFO] server-1132bc9e-5197-c753-bea1-34270d7d3076: election won: term=2 tally=1 +2026-11-29T10:28:21.089Z [INFO] server-1132bc9e-5197-c753-bea1-34270d7d3076: entering leader state: leader="Node at 1132bc9e-5197-c753-bea1-34270d7d3076 [Leader]" +2026-11-29T10:28:21.195Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:21.090048659 +0000 UTC m=+11.825102801 (1 nodes), 98.1846ms from start of poll, 98.27852ms from cluster start. Timeout at 2026-11-29 10:28:21.195050059 +0000 UTC m=+11.930104241, 105.00144ms after stability +2026-11-29T10:28:21.198Z [INFO] server-1132bc9e-5197-c753-bea1-34270d7d3076: starting snapshot up to: index=102 +2026-11-29T10:28:21.198Z [INFO] creating new snapshot: path=/tmp/raft195244542/snapshots/2-102-1795948101198.tmp +2026-11-29T10:28:21.199Z [INFO] server-1132bc9e-5197-c753-bea1-34270d7d3076: compacting logs: from=1 to=92 +2026-11-29T10:28:21.199Z [INFO] server-1132bc9e-5197-c753-bea1-34270d7d3076: snapshot complete up to: index=102 +2026-11-29T10:28:21.199Z [INFO] server-1132bc9e-5197-c753-bea1-34270d7d3076: starting restore from snapshot: id=2-102-1795948101198 last-index=102 last-term=2 size-in-bytes=693 +2026-11-29T10:28:21.199Z [INFO] server-1132bc9e-5197-c753-bea1-34270d7d3076: snapshot restore progress: id=2-102-1795948101198 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% +2026-11-29T10:28:21.199Z [INFO] server-1132bc9e-5197-c753-bea1-34270d7d3076: restored from snapshot: id=2-102-1795948101198 last-index=102 last-term=2 size-in-bytes=693 +2026-11-29T10:28:21.199Z [INFO] server-1132bc9e-5197-c753-bea1-34270d7d3076: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1132bc9e-5197-c753-bea1-34270d7d3076 Address:1132bc9e-5197-c753-bea1-34270d7d3076}]" +--- PASS: TestRaft_SnapshotRestore (0.21s) === RUN TestRaft_RestoreSnapshotOnStartup_Monotonic -2025-10-27T04:02:18.042Z [INFO] server-c3404f47-b96a-c516-5ed0-0cd7732e9744: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c3404f47-b96a-c516-5ed0-0cd7732e9744 Address:c3404f47-b96a-c516-5ed0-0cd7732e9744}]" -2025-10-27T04:02:18.042Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:18.042Z [INFO] server-c3404f47-b96a-c516-5ed0-0cd7732e9744: entering follower state: follower="Node at c3404f47-b96a-c516-5ed0-0cd7732e9744 [Follower]" leader-address= leader-id= -2025-10-27T04:02:18.099Z [WARN] server-c3404f47-b96a-c516-5ed0-0cd7732e9744: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:18.099Z [INFO] server-c3404f47-b96a-c516-5ed0-0cd7732e9744: entering candidate state: node="Node at c3404f47-b96a-c516-5ed0-0cd7732e9744 [Candidate]" term=2 -2025-10-27T04:02:18.099Z [INFO] server-c3404f47-b96a-c516-5ed0-0cd7732e9744: election won: term=2 tally=1 -2025-10-27T04:02:18.099Z [INFO] server-c3404f47-b96a-c516-5ed0-0cd7732e9744: entering leader state: leader="Node at c3404f47-b96a-c516-5ed0-0cd7732e9744 [Leader]" -2025-10-27T04:02:18.205Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:18.099544944 +0000 UTC m=+13.094417382 (1 nodes), 57.249311ms from start of poll, 57.393672ms from cluster start. Timeout at 2025-10-27 04:02:18.204548544 +0000 UTC m=+13.199420982, 105.0036ms after stability -2025-10-27T04:02:18.206Z [INFO] server-c3404f47-b96a-c516-5ed0-0cd7732e9744: starting snapshot up to: index=102 -2025-10-27T04:02:18.206Z [INFO] creating new snapshot: path=/tmp/raft2743952510/snapshots/2-102-1761537738206.tmp -2025-10-27T04:02:18.206Z [INFO] server-c3404f47-b96a-c516-5ed0-0cd7732e9744: compacting logs: from=1 to=92 -2025-10-27T04:02:18.206Z [INFO] server-c3404f47-b96a-c516-5ed0-0cd7732e9744: snapshot complete up to: index=102 -2025-10-27T04:02:18.207Z [INFO] server-c3404f47-b96a-c516-5ed0-0cd7732e9744: starting restore from snapshot: id=2-102-1761537738206 last-index=102 last-term=2 size-in-bytes=693 -2025-10-27T04:02:18.207Z [INFO] server-c3404f47-b96a-c516-5ed0-0cd7732e9744: snapshot restore progress: id=2-102-1761537738206 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% -2025-10-27T04:02:18.207Z [INFO] server-c3404f47-b96a-c516-5ed0-0cd7732e9744: restored from snapshot: id=2-102-1761537738206 last-index=102 last-term=2 size-in-bytes=693 -2025-10-27T04:02:18.207Z [INFO] server-c3404f47-b96a-c516-5ed0-0cd7732e9744: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c3404f47-b96a-c516-5ed0-0cd7732e9744 Address:c3404f47-b96a-c516-5ed0-0cd7732e9744}]" +2026-11-29T10:28:21.200Z [INFO] server-e6c3a8d3-fe21-e3c3-e15f-facb7965493e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e6c3a8d3-fe21-e3c3-e15f-facb7965493e Address:e6c3a8d3-fe21-e3c3-e15f-facb7965493e}]" +2026-11-29T10:28:21.200Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:21.200Z [INFO] server-e6c3a8d3-fe21-e3c3-e15f-facb7965493e: entering follower state: follower="Node at e6c3a8d3-fe21-e3c3-e15f-facb7965493e [Follower]" leader-address= leader-id= +2026-11-29T10:28:21.258Z [WARN] server-e6c3a8d3-fe21-e3c3-e15f-facb7965493e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:21.258Z [INFO] server-e6c3a8d3-fe21-e3c3-e15f-facb7965493e: entering candidate state: node="Node at e6c3a8d3-fe21-e3c3-e15f-facb7965493e [Candidate]" term=2 +2026-11-29T10:28:21.258Z [INFO] server-e6c3a8d3-fe21-e3c3-e15f-facb7965493e: election won: term=2 tally=1 +2026-11-29T10:28:21.258Z [INFO] server-e6c3a8d3-fe21-e3c3-e15f-facb7965493e: entering leader state: leader="Node at e6c3a8d3-fe21-e3c3-e15f-facb7965493e [Leader]" +2026-11-29T10:28:21.364Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:21.258325019 +0000 UTC m=+11.993379161 (1 nodes), 57.40744ms from start of poll, 57.94488ms from cluster start. Timeout at 2026-11-29 10:28:21.363329259 +0000 UTC m=+12.098383401, 105.00424ms after stability +2026-11-29T10:28:21.366Z [INFO] server-e6c3a8d3-fe21-e3c3-e15f-facb7965493e: starting snapshot up to: index=102 +2026-11-29T10:28:21.366Z [INFO] creating new snapshot: path=/tmp/raft414104660/snapshots/2-102-1795948101366.tmp +2026-11-29T10:28:21.367Z [INFO] server-e6c3a8d3-fe21-e3c3-e15f-facb7965493e: compacting logs: from=1 to=92 +2026-11-29T10:28:21.367Z [INFO] server-e6c3a8d3-fe21-e3c3-e15f-facb7965493e: snapshot complete up to: index=102 +2026-11-29T10:28:21.367Z [INFO] server-e6c3a8d3-fe21-e3c3-e15f-facb7965493e: starting restore from snapshot: id=2-102-1795948101366 last-index=102 last-term=2 size-in-bytes=693 +2026-11-29T10:28:21.367Z [INFO] server-e6c3a8d3-fe21-e3c3-e15f-facb7965493e: snapshot restore progress: id=2-102-1795948101366 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% +2026-11-29T10:28:21.367Z [INFO] server-e6c3a8d3-fe21-e3c3-e15f-facb7965493e: restored from snapshot: id=2-102-1795948101366 last-index=102 last-term=2 size-in-bytes=693 +2026-11-29T10:28:21.367Z [INFO] server-e6c3a8d3-fe21-e3c3-e15f-facb7965493e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e6c3a8d3-fe21-e3c3-e15f-facb7965493e Address:e6c3a8d3-fe21-e3c3-e15f-facb7965493e}]" --- PASS: TestRaft_RestoreSnapshotOnStartup_Monotonic (0.17s) === RUN TestRaft_SnapshotRestore_Progress -2025-10-27T04:02:18.207Z [INFO] server-68173df7-1a52-969b-3f4a-29f949eb7d9c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-68173df7-1a52-969b-3f4a-29f949eb7d9c Address:68173df7-1a52-969b-3f4a-29f949eb7d9c}]" -2025-10-27T04:02:18.207Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:18.207Z [INFO] server-68173df7-1a52-969b-3f4a-29f949eb7d9c: entering follower state: follower="Node at 68173df7-1a52-969b-3f4a-29f949eb7d9c [Follower]" leader-address= leader-id= -2025-10-27T04:02:18.305Z [WARN] server-68173df7-1a52-969b-3f4a-29f949eb7d9c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:18.305Z [INFO] server-68173df7-1a52-969b-3f4a-29f949eb7d9c: entering candidate state: node="Node at 68173df7-1a52-969b-3f4a-29f949eb7d9c [Candidate]" term=2 -2025-10-27T04:02:18.305Z [INFO] server-68173df7-1a52-969b-3f4a-29f949eb7d9c: election won: term=2 tally=1 -2025-10-27T04:02:18.305Z [INFO] server-68173df7-1a52-969b-3f4a-29f949eb7d9c: entering leader state: leader="Node at 68173df7-1a52-969b-3f4a-29f949eb7d9c [Leader]" -2025-10-27T04:02:18.411Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:18.305441672 +0000 UTC m=+13.300314070 (1 nodes), 98.00267ms from start of poll, 98.072711ms from cluster start. Timeout at 2025-10-27 04:02:18.410443912 +0000 UTC m=+13.405316310, 105.00224ms after stability -2025-10-27T04:02:18.412Z [INFO] server-68173df7-1a52-969b-3f4a-29f949eb7d9c: starting snapshot up to: index=102 -2025-10-27T04:02:18.412Z [INFO] creating new snapshot: path=/tmp/raft1469382618/snapshots/2-102-1761537738412.tmp -2025-10-27T04:02:18.413Z [INFO] server-68173df7-1a52-969b-3f4a-29f949eb7d9c: compacting logs: from=1 to=92 -2025-10-27T04:02:18.413Z [INFO] server-68173df7-1a52-969b-3f4a-29f949eb7d9c: snapshot complete up to: index=102 ---- PASS: TestRaft_SnapshotRestore_Progress (0.21s) +2026-11-29T10:28:21.368Z [INFO] server-d246adda-6286-628e-dbed-978f83b6fbad: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d246adda-6286-628e-dbed-978f83b6fbad Address:d246adda-6286-628e-dbed-978f83b6fbad}]" +2026-11-29T10:28:21.368Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:21.368Z [INFO] server-d246adda-6286-628e-dbed-978f83b6fbad: entering follower state: follower="Node at d246adda-6286-628e-dbed-978f83b6fbad [Follower]" leader-address= leader-id= +2026-11-29T10:28:21.434Z [WARN] server-d246adda-6286-628e-dbed-978f83b6fbad: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:21.434Z [INFO] server-d246adda-6286-628e-dbed-978f83b6fbad: entering candidate state: node="Node at d246adda-6286-628e-dbed-978f83b6fbad [Candidate]" term=2 +2026-11-29T10:28:21.434Z [INFO] server-d246adda-6286-628e-dbed-978f83b6fbad: election won: term=2 tally=1 +2026-11-29T10:28:21.434Z [INFO] server-d246adda-6286-628e-dbed-978f83b6fbad: entering leader state: leader="Node at d246adda-6286-628e-dbed-978f83b6fbad [Leader]" +2026-11-29T10:28:21.539Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:21.434378099 +0000 UTC m=+12.169432241 (1 nodes), 65.8944ms from start of poll, 66.0764ms from cluster start. Timeout at 2026-11-29 10:28:21.539381499 +0000 UTC m=+12.274435641, 105.0034ms after stability +2026-11-29T10:28:21.541Z [INFO] server-d246adda-6286-628e-dbed-978f83b6fbad: starting snapshot up to: index=102 +2026-11-29T10:28:21.541Z [INFO] creating new snapshot: path=/tmp/raft453429786/snapshots/2-102-1795948101541.tmp +2026-11-29T10:28:21.542Z [INFO] server-d246adda-6286-628e-dbed-978f83b6fbad: compacting logs: from=1 to=92 +2026-11-29T10:28:21.542Z [INFO] server-d246adda-6286-628e-dbed-978f83b6fbad: snapshot complete up to: index=102 +--- PASS: TestRaft_SnapshotRestore_Progress (0.18s) === RUN TestRaft_NoRestoreOnStart -2025-10-27T04:02:18.414Z [INFO] server-286fcb7a-0de4-df53-aa4d-282c7e95ea57: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-286fcb7a-0de4-df53-aa4d-282c7e95ea57 Address:286fcb7a-0de4-df53-aa4d-282c7e95ea57}]" -2025-10-27T04:02:18.414Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:18.414Z [INFO] server-286fcb7a-0de4-df53-aa4d-282c7e95ea57: entering follower state: follower="Node at 286fcb7a-0de4-df53-aa4d-282c7e95ea57 [Follower]" leader-address= leader-id= -2025-10-27T04:02:18.505Z [WARN] server-286fcb7a-0de4-df53-aa4d-282c7e95ea57: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:18.505Z [INFO] server-286fcb7a-0de4-df53-aa4d-282c7e95ea57: entering candidate state: node="Node at 286fcb7a-0de4-df53-aa4d-282c7e95ea57 [Candidate]" term=2 -2025-10-27T04:02:18.505Z [INFO] server-286fcb7a-0de4-df53-aa4d-282c7e95ea57: election won: term=2 tally=1 -2025-10-27T04:02:18.505Z [INFO] server-286fcb7a-0de4-df53-aa4d-282c7e95ea57: entering leader state: leader="Node at 286fcb7a-0de4-df53-aa4d-282c7e95ea57 [Leader]" -2025-10-27T04:02:18.611Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:18.505293598 +0000 UTC m=+13.500166036 (1 nodes), 91.079303ms from start of poll, 91.211504ms from cluster start. Timeout at 2025-10-27 04:02:18.610296878 +0000 UTC m=+13.605169316, 105.00328ms after stability -2025-10-27T04:02:18.612Z [INFO] server-286fcb7a-0de4-df53-aa4d-282c7e95ea57: starting snapshot up to: index=102 -2025-10-27T04:02:18.612Z [INFO] creating new snapshot: path=/tmp/raft2692537034/snapshots/2-102-1761537738612.tmp -2025-10-27T04:02:18.613Z [INFO] server-286fcb7a-0de4-df53-aa4d-282c7e95ea57: compacting logs: from=1 to=92 -2025-10-27T04:02:18.613Z [INFO] server-286fcb7a-0de4-df53-aa4d-282c7e95ea57: snapshot complete up to: index=102 -2025-10-27T04:02:18.613Z [INFO] server-286fcb7a-0de4-df53-aa4d-282c7e95ea57: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-286fcb7a-0de4-df53-aa4d-282c7e95ea57 Address:286fcb7a-0de4-df53-aa4d-282c7e95ea57}]" +2026-11-29T10:28:21.543Z [INFO] server-a96148dd-f679-e686-a7d9-7ef1643df1ca: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a96148dd-f679-e686-a7d9-7ef1643df1ca Address:a96148dd-f679-e686-a7d9-7ef1643df1ca}]" +2026-11-29T10:28:21.543Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:21.543Z [INFO] server-a96148dd-f679-e686-a7d9-7ef1643df1ca: entering follower state: follower="Node at a96148dd-f679-e686-a7d9-7ef1643df1ca [Follower]" leader-address= leader-id= +2026-11-29T10:28:21.637Z [WARN] server-a96148dd-f679-e686-a7d9-7ef1643df1ca: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:21.637Z [INFO] server-a96148dd-f679-e686-a7d9-7ef1643df1ca: entering candidate state: node="Node at a96148dd-f679-e686-a7d9-7ef1643df1ca [Candidate]" term=2 +2026-11-29T10:28:21.637Z [INFO] server-a96148dd-f679-e686-a7d9-7ef1643df1ca: election won: term=2 tally=1 +2026-11-29T10:28:21.637Z [INFO] server-a96148dd-f679-e686-a7d9-7ef1643df1ca: entering leader state: leader="Node at a96148dd-f679-e686-a7d9-7ef1643df1ca [Leader]" +2026-11-29T10:28:21.743Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:21.637728659 +0000 UTC m=+12.372782761 (1 nodes), 93.97212ms from start of poll, 94.14348ms from cluster start. Timeout at 2026-11-29 10:28:21.742730659 +0000 UTC m=+12.477784801, 105.00204ms after stability +2026-11-29T10:28:21.745Z [INFO] server-a96148dd-f679-e686-a7d9-7ef1643df1ca: starting snapshot up to: index=102 +2026-11-29T10:28:21.745Z [INFO] creating new snapshot: path=/tmp/raft662738068/snapshots/2-102-1795948101745.tmp +2026-11-29T10:28:21.746Z [INFO] server-a96148dd-f679-e686-a7d9-7ef1643df1ca: compacting logs: from=1 to=92 +2026-11-29T10:28:21.746Z [INFO] server-a96148dd-f679-e686-a7d9-7ef1643df1ca: snapshot complete up to: index=102 +2026-11-29T10:28:21.746Z [INFO] server-a96148dd-f679-e686-a7d9-7ef1643df1ca: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a96148dd-f679-e686-a7d9-7ef1643df1ca Address:a96148dd-f679-e686-a7d9-7ef1643df1ca}]" --- PASS: TestRaft_NoRestoreOnStart (0.20s) === RUN TestRaft_SnapshotRestore_PeerChange +2026-11-29T10:28:21.746Z [INFO] server-a96148dd-f679-e686-a7d9-7ef1643df1ca: entering follower state: follower="Node at a96148dd-f679-e686-a7d9-7ef1643df1ca [Follower]" leader-address= leader-id= raft_test.go:1266: Skipping unreliable test --- SKIP: TestRaft_SnapshotRestore_PeerChange (0.00s) === RUN TestRaft_AutoSnapshot -2025-10-27T04:02:18.613Z [INFO] server-f522c7eb-397e-6bfe-f0fd-5725f7c41f77: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f522c7eb-397e-6bfe-f0fd-5725f7c41f77 Address:f522c7eb-397e-6bfe-f0fd-5725f7c41f77}]" -2025-10-27T04:02:18.613Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:18.613Z [INFO] server-286fcb7a-0de4-df53-aa4d-282c7e95ea57: entering follower state: follower="Node at 286fcb7a-0de4-df53-aa4d-282c7e95ea57 [Follower]" leader-address= leader-id= -2025-10-27T04:02:18.613Z [INFO] server-f522c7eb-397e-6bfe-f0fd-5725f7c41f77: entering follower state: follower="Node at f522c7eb-397e-6bfe-f0fd-5725f7c41f77 [Follower]" leader-address= leader-id= -2025-10-27T04:02:18.689Z [WARN] server-286fcb7a-0de4-df53-aa4d-282c7e95ea57: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:18.689Z [INFO] server-286fcb7a-0de4-df53-aa4d-282c7e95ea57: entering candidate state: node="Node at 286fcb7a-0de4-df53-aa4d-282c7e95ea57 [Candidate]" term=3 -2025-10-27T04:02:18.689Z [INFO] server-286fcb7a-0de4-df53-aa4d-282c7e95ea57: election won: term=3 tally=1 -2025-10-27T04:02:18.689Z [INFO] server-286fcb7a-0de4-df53-aa4d-282c7e95ea57: entering leader state: leader="Node at 286fcb7a-0de4-df53-aa4d-282c7e95ea57 [Leader]" -2025-10-27T04:02:18.708Z [WARN] server-f522c7eb-397e-6bfe-f0fd-5725f7c41f77: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:18.709Z [INFO] server-f522c7eb-397e-6bfe-f0fd-5725f7c41f77: entering candidate state: node="Node at f522c7eb-397e-6bfe-f0fd-5725f7c41f77 [Candidate]" term=2 -2025-10-27T04:02:18.709Z [INFO] server-f522c7eb-397e-6bfe-f0fd-5725f7c41f77: election won: term=2 tally=1 -2025-10-27T04:02:18.709Z [INFO] server-f522c7eb-397e-6bfe-f0fd-5725f7c41f77: entering leader state: leader="Node at f522c7eb-397e-6bfe-f0fd-5725f7c41f77 [Leader]" -2025-10-27T04:02:18.814Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:18.709200671 +0000 UTC m=+13.704073109 (1 nodes), 95.481932ms from start of poll, 95.573453ms from cluster start. Timeout at 2025-10-27 04:02:18.814202751 +0000 UTC m=+13.809075189, 105.00208ms after stability -2025-10-27T04:02:18.820Z [INFO] server-f522c7eb-397e-6bfe-f0fd-5725f7c41f77: starting snapshot up to: index=102 -2025-10-27T04:02:18.820Z [INFO] creating new snapshot: path=/tmp/raft3998009171/snapshots/2-102-1761537738820.tmp -2025-10-27T04:02:18.821Z [INFO] server-f522c7eb-397e-6bfe-f0fd-5725f7c41f77: compacting logs: from=1 to=92 -2025-10-27T04:02:18.821Z [INFO] server-f522c7eb-397e-6bfe-f0fd-5725f7c41f77: snapshot complete up to: index=102 ---- PASS: TestRaft_AutoSnapshot (0.31s) +2026-11-29T10:28:21.746Z [INFO] server-3188b808-bfd5-75d8-ac10-8374ca7fe0ed: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3188b808-bfd5-75d8-ac10-8374ca7fe0ed Address:3188b808-bfd5-75d8-ac10-8374ca7fe0ed}]" +2026-11-29T10:28:21.746Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:21.746Z [INFO] server-3188b808-bfd5-75d8-ac10-8374ca7fe0ed: entering follower state: follower="Node at 3188b808-bfd5-75d8-ac10-8374ca7fe0ed [Follower]" leader-address= leader-id= +2026-11-29T10:28:21.828Z [WARN] server-3188b808-bfd5-75d8-ac10-8374ca7fe0ed: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:21.828Z [INFO] server-3188b808-bfd5-75d8-ac10-8374ca7fe0ed: entering candidate state: node="Node at 3188b808-bfd5-75d8-ac10-8374ca7fe0ed [Candidate]" term=2 +2026-11-29T10:28:21.828Z [INFO] server-3188b808-bfd5-75d8-ac10-8374ca7fe0ed: election won: term=2 tally=1 +2026-11-29T10:28:21.828Z [INFO] server-3188b808-bfd5-75d8-ac10-8374ca7fe0ed: entering leader state: leader="Node at 3188b808-bfd5-75d8-ac10-8374ca7fe0ed [Leader]" +2026-11-29T10:28:21.831Z [WARN] server-a96148dd-f679-e686-a7d9-7ef1643df1ca: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:21.831Z [INFO] server-a96148dd-f679-e686-a7d9-7ef1643df1ca: entering candidate state: node="Node at a96148dd-f679-e686-a7d9-7ef1643df1ca [Candidate]" term=3 +2026-11-29T10:28:21.831Z [INFO] server-a96148dd-f679-e686-a7d9-7ef1643df1ca: election won: term=3 tally=1 +2026-11-29T10:28:21.831Z [INFO] server-a96148dd-f679-e686-a7d9-7ef1643df1ca: entering leader state: leader="Node at a96148dd-f679-e686-a7d9-7ef1643df1ca [Leader]" +2026-11-29T10:28:21.934Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:21.828297419 +0000 UTC m=+12.563351561 (1 nodes), 81.5436ms from start of poll, 81.69236ms from cluster start. Timeout at 2026-11-29 10:28:21.933300339 +0000 UTC m=+12.668354481, 105.00292ms after stability +2026-11-29T10:28:21.952Z [INFO] server-3188b808-bfd5-75d8-ac10-8374ca7fe0ed: starting snapshot up to: index=102 +2026-11-29T10:28:21.952Z [INFO] creating new snapshot: path=/tmp/raft490902439/snapshots/2-102-1795948101952.tmp +2026-11-29T10:28:21.952Z [INFO] server-3188b808-bfd5-75d8-ac10-8374ca7fe0ed: compacting logs: from=1 to=92 +2026-11-29T10:28:21.952Z [INFO] server-3188b808-bfd5-75d8-ac10-8374ca7fe0ed: snapshot complete up to: index=102 +--- PASS: TestRaft_AutoSnapshot (0.30s) === RUN TestRaft_UserSnapshot -2025-10-27T04:02:18.922Z [INFO] server-68d9b8c9-de07-11de-b26a-cb2dc6760c23: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-68d9b8c9-de07-11de-b26a-cb2dc6760c23 Address:68d9b8c9-de07-11de-b26a-cb2dc6760c23}]" -2025-10-27T04:02:18.923Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:18.923Z [INFO] server-68d9b8c9-de07-11de-b26a-cb2dc6760c23: entering follower state: follower="Node at 68d9b8c9-de07-11de-b26a-cb2dc6760c23 [Follower]" leader-address= leader-id= -2025-10-27T04:02:18.986Z [WARN] server-68d9b8c9-de07-11de-b26a-cb2dc6760c23: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:18.986Z [INFO] server-68d9b8c9-de07-11de-b26a-cb2dc6760c23: entering candidate state: node="Node at 68d9b8c9-de07-11de-b26a-cb2dc6760c23 [Candidate]" term=2 -2025-10-27T04:02:18.986Z [INFO] server-68d9b8c9-de07-11de-b26a-cb2dc6760c23: election won: term=2 tally=1 -2025-10-27T04:02:18.986Z [INFO] server-68d9b8c9-de07-11de-b26a-cb2dc6760c23: entering leader state: leader="Node at 68d9b8c9-de07-11de-b26a-cb2dc6760c23 [Leader]" -2025-10-27T04:02:19.092Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:18.986261125 +0000 UTC m=+13.981133643 (1 nodes), 63.206872ms from start of poll, 63.477434ms from cluster start. Timeout at 2025-10-27 04:02:19.091265405 +0000 UTC m=+14.086137883, 105.00424ms after stability -2025-10-27T04:02:19.092Z [ERROR] server-68d9b8c9-de07-11de-b26a-cb2dc6760c23: failed to take snapshot: error="nothing new to snapshot" -2025-10-27T04:02:19.092Z [INFO] server-68d9b8c9-de07-11de-b26a-cb2dc6760c23: starting snapshot up to: index=12 -2025-10-27T04:02:19.092Z [INFO] creating new snapshot: path=/tmp/raft4146332625/snapshots/2-12-1761537739092.tmp -2025-10-27T04:02:19.093Z [INFO] server-68d9b8c9-de07-11de-b26a-cb2dc6760c23: compacting logs: from=1 to=2 -2025-10-27T04:02:19.093Z [INFO] server-68d9b8c9-de07-11de-b26a-cb2dc6760c23: snapshot complete up to: index=12 +2026-11-29T10:28:22.042Z [INFO] server-da3a2bf2-4a06-0335-e970-b1ff2f567c61: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-da3a2bf2-4a06-0335-e970-b1ff2f567c61 Address:da3a2bf2-4a06-0335-e970-b1ff2f567c61}]" +2026-11-29T10:28:22.042Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:22.042Z [INFO] server-da3a2bf2-4a06-0335-e970-b1ff2f567c61: entering follower state: follower="Node at da3a2bf2-4a06-0335-e970-b1ff2f567c61 [Follower]" leader-address= leader-id= +2026-11-29T10:28:22.109Z [WARN] server-da3a2bf2-4a06-0335-e970-b1ff2f567c61: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:22.109Z [INFO] server-da3a2bf2-4a06-0335-e970-b1ff2f567c61: entering candidate state: node="Node at da3a2bf2-4a06-0335-e970-b1ff2f567c61 [Candidate]" term=2 +2026-11-29T10:28:22.109Z [INFO] server-da3a2bf2-4a06-0335-e970-b1ff2f567c61: election won: term=2 tally=1 +2026-11-29T10:28:22.109Z [INFO] server-da3a2bf2-4a06-0335-e970-b1ff2f567c61: entering leader state: leader="Node at da3a2bf2-4a06-0335-e970-b1ff2f567c61 [Leader]" +2026-11-29T10:28:22.215Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:22.109252539 +0000 UTC m=+12.844306641 (1 nodes), 66.30212ms from start of poll, 66.49736ms from cluster start. Timeout at 2026-11-29 10:28:22.214254899 +0000 UTC m=+12.949309081, 105.00244ms after stability +2026-11-29T10:28:22.215Z [ERROR] server-da3a2bf2-4a06-0335-e970-b1ff2f567c61: failed to take snapshot: error="nothing new to snapshot" +2026-11-29T10:28:22.215Z [INFO] server-da3a2bf2-4a06-0335-e970-b1ff2f567c61: starting snapshot up to: index=12 +2026-11-29T10:28:22.215Z [INFO] creating new snapshot: path=/tmp/raft755936155/snapshots/2-12-1795948102215.tmp +2026-11-29T10:28:22.216Z [INFO] server-da3a2bf2-4a06-0335-e970-b1ff2f567c61: compacting logs: from=1 to=2 +2026-11-29T10:28:22.216Z [INFO] server-da3a2bf2-4a06-0335-e970-b1ff2f567c61: snapshot complete up to: index=12 --- PASS: TestRaft_UserSnapshot (0.17s) === RUN TestRaft_UserRestore === RUN TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_false -2025-10-27T04:02:19.093Z [INFO] server-5bff919d-e43a-2c6d-a50a-dcacbedd3bd9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5bff919d-e43a-2c6d-a50a-dcacbedd3bd9 Address:5bff919d-e43a-2c6d-a50a-dcacbedd3bd9} {Suffrage:Voter ID:server-f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312 Address:f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312} {Suffrage:Voter ID:server-eacbc843-9a90-1508-27fc-ba3f16e5688c Address:eacbc843-9a90-1508-27fc-ba3f16e5688c}]" -2025-10-27T04:02:19.093Z [INFO] server-f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5bff919d-e43a-2c6d-a50a-dcacbedd3bd9 Address:5bff919d-e43a-2c6d-a50a-dcacbedd3bd9} {Suffrage:Voter ID:server-f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312 Address:f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312} {Suffrage:Voter ID:server-eacbc843-9a90-1508-27fc-ba3f16e5688c Address:eacbc843-9a90-1508-27fc-ba3f16e5688c}]" -2025-10-27T04:02:19.093Z [INFO] server-eacbc843-9a90-1508-27fc-ba3f16e5688c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5bff919d-e43a-2c6d-a50a-dcacbedd3bd9 Address:5bff919d-e43a-2c6d-a50a-dcacbedd3bd9} {Suffrage:Voter ID:server-f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312 Address:f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312} {Suffrage:Voter ID:server-eacbc843-9a90-1508-27fc-ba3f16e5688c Address:eacbc843-9a90-1508-27fc-ba3f16e5688c}]" -2025-10-27T04:02:19.093Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:19.093Z [INFO] server-5bff919d-e43a-2c6d-a50a-dcacbedd3bd9: entering follower state: follower="Node at 5bff919d-e43a-2c6d-a50a-dcacbedd3bd9 [Follower]" leader-address= leader-id= -2025-10-27T04:02:19.094Z [INFO] server-eacbc843-9a90-1508-27fc-ba3f16e5688c: entering follower state: follower="Node at eacbc843-9a90-1508-27fc-ba3f16e5688c [Follower]" leader-address= leader-id= -2025-10-27T04:02:19.093Z [INFO] server-f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312: entering follower state: follower="Node at f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312 [Follower]" leader-address= leader-id= -2025-10-27T04:02:19.632Z [WARN] server-f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:19.632Z [INFO] server-f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312: entering candidate state: node="Node at f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312 [Candidate]" term=2 -2025-10-27T04:02:19.632Z [INFO] server-f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312: election won: term=2 tally=2 -2025-10-27T04:02:19.632Z [INFO] server-f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312: entering leader state: leader="Node at f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312 [Leader]" -2025-10-27T04:02:19.632Z [INFO] server-f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312: added peer, starting replication: peer=server-5bff919d-e43a-2c6d-a50a-dcacbedd3bd9 -2025-10-27T04:02:19.632Z [INFO] server-f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312: added peer, starting replication: peer=server-eacbc843-9a90-1508-27fc-ba3f16e5688c -2025-10-27T04:02:19.632Z [INFO] server-f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312: pipelining replication: peer="{Voter server-eacbc843-9a90-1508-27fc-ba3f16e5688c eacbc843-9a90-1508-27fc-ba3f16e5688c}" -2025-10-27T04:02:19.632Z [INFO] server-f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312: pipelining replication: peer="{Voter server-5bff919d-e43a-2c6d-a50a-dcacbedd3bd9 5bff919d-e43a-2c6d-a50a-dcacbedd3bd9}" -2025-10-27T04:02:20.637Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:19.632623863 +0000 UTC m=+14.627496301 (1 nodes), 538.631082ms from start of poll, 538.816523ms from cluster start. Timeout at 2025-10-27 04:02:20.637626023 +0000 UTC m=+15.632498461, 1.00500216s after stability -2025-10-27T04:02:20.638Z [INFO] server-f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312: starting snapshot up to: index=12 -2025-10-27T04:02:20.638Z [INFO] creating new snapshot: path=/tmp/raft1104166886/snapshots/2-12-1761537740638.tmp -2025-10-27T04:02:20.638Z [INFO] server-f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312: snapshot complete up to: index=12 -2025-10-27T04:02:20.638Z [INFO] creating new snapshot: path=/tmp/raft1104166886/snapshots/2-23-1761537740638.tmp -2025-10-27T04:02:20.638Z [INFO] server-f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312: copied to local snapshot: bytes=71 -2025-10-27T04:02:20.639Z [INFO] server-f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312: snapshot restore progress: id=2-23-1761537740638 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:20.639Z [INFO] server-f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312: restored user snapshot: index=23 -2025-10-27T04:02:20.639Z [ERROR] server-f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312: failed to get log: index=23 error="log not found" -2025-10-27T04:02:20.639Z [INFO] server-f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312: aborting pipeline replication: peer="{Voter server-5bff919d-e43a-2c6d-a50a-dcacbedd3bd9 5bff919d-e43a-2c6d-a50a-dcacbedd3bd9}" -2025-10-27T04:02:20.639Z [ERROR] server-f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312: failed to get log: index=23 error="log not found" -2025-10-27T04:02:20.639Z [INFO] server-f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312: aborting pipeline replication: peer="{Voter server-eacbc843-9a90-1508-27fc-ba3f16e5688c eacbc843-9a90-1508-27fc-ba3f16e5688c}" -2025-10-27T04:02:20.646Z [ERROR] server-f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312: failed to get log: index=23 error="log not found" -2025-10-27T04:02:20.646Z [INFO] creating new snapshot: path=/tmp/raft2037138896/snapshots/2-23-1761537740646.tmp -2025-10-27T04:02:20.646Z [INFO] server-eacbc843-9a90-1508-27fc-ba3f16e5688c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:20.646Z [INFO] server-eacbc843-9a90-1508-27fc-ba3f16e5688c: copied to local snapshot: bytes=71 -2025-10-27T04:02:20.646Z [INFO] server-eacbc843-9a90-1508-27fc-ba3f16e5688c: snapshot restore progress: id=2-23-1761537740646 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:20.646Z [INFO] server-eacbc843-9a90-1508-27fc-ba3f16e5688c: Installed remote snapshot -2025-10-27T04:02:20.646Z [WARN] server-eacbc843-9a90-1508-27fc-ba3f16e5688c: skipping application of old log: index=22 -2025-10-27T04:02:20.647Z [INFO] server-f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312: pipelining replication: peer="{Voter server-eacbc843-9a90-1508-27fc-ba3f16e5688c eacbc843-9a90-1508-27fc-ba3f16e5688c}" -2025-10-27T04:02:20.647Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:02:20.648Z [ERROR] server-f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312: failed to get log: index=23 error="log not found" -2025-10-27T04:02:20.648Z [INFO] creating new snapshot: path=/tmp/raft1746001341/snapshots/2-23-1761537740648.tmp -2025-10-27T04:02:20.648Z [INFO] server-5bff919d-e43a-2c6d-a50a-dcacbedd3bd9: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:20.648Z [INFO] server-5bff919d-e43a-2c6d-a50a-dcacbedd3bd9: copied to local snapshot: bytes=71 -2025-10-27T04:02:20.649Z [INFO] server-5bff919d-e43a-2c6d-a50a-dcacbedd3bd9: snapshot restore progress: id=2-23-1761537740648 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:20.649Z [INFO] server-5bff919d-e43a-2c6d-a50a-dcacbedd3bd9: Installed remote snapshot -2025-10-27T04:02:20.650Z [INFO] server-f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312: pipelining replication: peer="{Voter server-5bff919d-e43a-2c6d-a50a-dcacbedd3bd9 5bff919d-e43a-2c6d-a50a-dcacbedd3bd9}" -2025-10-27T04:02:21.652Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:02:20.647043796 +0000 UTC m=+15.641916234 (2 nodes), 400ns from start of poll, 1.553236456s from cluster start. Timeout at 2025-10-27 04:02:21.652056596 +0000 UTC m=+16.646929114, 1.00501288s after stability -2025-10-27T04:02:21.664Z [INFO] server-f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312: aborting pipeline replication: peer="{Voter server-eacbc843-9a90-1508-27fc-ba3f16e5688c eacbc843-9a90-1508-27fc-ba3f16e5688c}" -2025-10-27T04:02:21.664Z [INFO] server-f8f57b4b-4fc7-edcf-1bdf-b30cbc3d4312: aborting pipeline replication: peer="{Voter server-5bff919d-e43a-2c6d-a50a-dcacbedd3bd9 5bff919d-e43a-2c6d-a50a-dcacbedd3bd9}" +2026-11-29T10:28:22.217Z [INFO] server-94257ef2-662d-5b68-9b4f-c5b55f5a0764: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-94257ef2-662d-5b68-9b4f-c5b55f5a0764 Address:94257ef2-662d-5b68-9b4f-c5b55f5a0764} {Suffrage:Voter ID:server-560dd12e-abd8-2d1b-7822-2468e952c049 Address:560dd12e-abd8-2d1b-7822-2468e952c049} {Suffrage:Voter ID:server-e17580e3-f9ff-9536-b13a-1bed706fd07c Address:e17580e3-f9ff-9536-b13a-1bed706fd07c}]" +2026-11-29T10:28:22.217Z [INFO] server-560dd12e-abd8-2d1b-7822-2468e952c049: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-94257ef2-662d-5b68-9b4f-c5b55f5a0764 Address:94257ef2-662d-5b68-9b4f-c5b55f5a0764} {Suffrage:Voter ID:server-560dd12e-abd8-2d1b-7822-2468e952c049 Address:560dd12e-abd8-2d1b-7822-2468e952c049} {Suffrage:Voter ID:server-e17580e3-f9ff-9536-b13a-1bed706fd07c Address:e17580e3-f9ff-9536-b13a-1bed706fd07c}]" +2026-11-29T10:28:22.217Z [INFO] server-94257ef2-662d-5b68-9b4f-c5b55f5a0764: entering follower state: follower="Node at 94257ef2-662d-5b68-9b4f-c5b55f5a0764 [Follower]" leader-address= leader-id= +2026-11-29T10:28:22.217Z [INFO] server-e17580e3-f9ff-9536-b13a-1bed706fd07c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-94257ef2-662d-5b68-9b4f-c5b55f5a0764 Address:94257ef2-662d-5b68-9b4f-c5b55f5a0764} {Suffrage:Voter ID:server-560dd12e-abd8-2d1b-7822-2468e952c049 Address:560dd12e-abd8-2d1b-7822-2468e952c049} {Suffrage:Voter ID:server-e17580e3-f9ff-9536-b13a-1bed706fd07c Address:e17580e3-f9ff-9536-b13a-1bed706fd07c}]" +2026-11-29T10:28:22.217Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:22.217Z [INFO] server-560dd12e-abd8-2d1b-7822-2468e952c049: entering follower state: follower="Node at 560dd12e-abd8-2d1b-7822-2468e952c049 [Follower]" leader-address= leader-id= +2026-11-29T10:28:22.217Z [INFO] server-e17580e3-f9ff-9536-b13a-1bed706fd07c: entering follower state: follower="Node at e17580e3-f9ff-9536-b13a-1bed706fd07c [Follower]" leader-address= leader-id= +2026-11-29T10:28:22.880Z [WARN] server-560dd12e-abd8-2d1b-7822-2468e952c049: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:22.880Z [INFO] server-560dd12e-abd8-2d1b-7822-2468e952c049: entering candidate state: node="Node at 560dd12e-abd8-2d1b-7822-2468e952c049 [Candidate]" term=2 +2026-11-29T10:28:22.880Z [INFO] server-560dd12e-abd8-2d1b-7822-2468e952c049: election won: term=2 tally=2 +2026-11-29T10:28:22.880Z [INFO] server-560dd12e-abd8-2d1b-7822-2468e952c049: entering leader state: leader="Node at 560dd12e-abd8-2d1b-7822-2468e952c049 [Leader]" +2026-11-29T10:28:22.880Z [INFO] server-560dd12e-abd8-2d1b-7822-2468e952c049: added peer, starting replication: peer=server-94257ef2-662d-5b68-9b4f-c5b55f5a0764 +2026-11-29T10:28:22.880Z [INFO] server-560dd12e-abd8-2d1b-7822-2468e952c049: added peer, starting replication: peer=server-e17580e3-f9ff-9536-b13a-1bed706fd07c +2026-11-29T10:28:22.880Z [INFO] server-560dd12e-abd8-2d1b-7822-2468e952c049: pipelining replication: peer="{Voter server-e17580e3-f9ff-9536-b13a-1bed706fd07c e17580e3-f9ff-9536-b13a-1bed706fd07c}" +2026-11-29T10:28:22.880Z [INFO] server-560dd12e-abd8-2d1b-7822-2468e952c049: pipelining replication: peer="{Voter server-94257ef2-662d-5b68-9b4f-c5b55f5a0764 94257ef2-662d-5b68-9b4f-c5b55f5a0764}" +2026-11-29T10:28:23.886Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:22.880765179 +0000 UTC m=+13.615819281 (1 nodes), 663.27112ms from start of poll, 663.70468ms from cluster start. Timeout at 2026-11-29 10:28:23.885768059 +0000 UTC m=+14.620822201, 1.00500292s after stability +2026-11-29T10:28:23.886Z [INFO] server-560dd12e-abd8-2d1b-7822-2468e952c049: starting snapshot up to: index=12 +2026-11-29T10:28:23.886Z [INFO] creating new snapshot: path=/tmp/raft3705095706/snapshots/2-12-1795948103886.tmp +2026-11-29T10:28:23.887Z [INFO] server-560dd12e-abd8-2d1b-7822-2468e952c049: snapshot complete up to: index=12 +2026-11-29T10:28:23.888Z [INFO] creating new snapshot: path=/tmp/raft3705095706/snapshots/2-23-1795948103888.tmp +2026-11-29T10:28:23.888Z [INFO] server-560dd12e-abd8-2d1b-7822-2468e952c049: copied to local snapshot: bytes=71 +2026-11-29T10:28:23.888Z [INFO] server-560dd12e-abd8-2d1b-7822-2468e952c049: snapshot restore progress: id=2-23-1795948103888 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:23.888Z [INFO] server-560dd12e-abd8-2d1b-7822-2468e952c049: restored user snapshot: index=23 +2026-11-29T10:28:23.888Z [ERROR] server-560dd12e-abd8-2d1b-7822-2468e952c049: failed to get log: index=23 error="log not found" +2026-11-29T10:28:23.888Z [INFO] server-560dd12e-abd8-2d1b-7822-2468e952c049: aborting pipeline replication: peer="{Voter server-e17580e3-f9ff-9536-b13a-1bed706fd07c e17580e3-f9ff-9536-b13a-1bed706fd07c}" +2026-11-29T10:28:23.888Z [ERROR] server-560dd12e-abd8-2d1b-7822-2468e952c049: failed to get log: index=23 error="log not found" +2026-11-29T10:28:23.889Z [INFO] server-560dd12e-abd8-2d1b-7822-2468e952c049: aborting pipeline replication: peer="{Voter server-94257ef2-662d-5b68-9b4f-c5b55f5a0764 94257ef2-662d-5b68-9b4f-c5b55f5a0764}" +2026-11-29T10:28:23.895Z [ERROR] server-560dd12e-abd8-2d1b-7822-2468e952c049: failed to get log: index=23 error="log not found" +2026-11-29T10:28:23.895Z [ERROR] server-560dd12e-abd8-2d1b-7822-2468e952c049: failed to get log: index=23 error="log not found" +2026-11-29T10:28:23.896Z [INFO] creating new snapshot: path=/tmp/raft3130607880/snapshots/2-23-1795948103896.tmp +2026-11-29T10:28:23.896Z [INFO] server-94257ef2-662d-5b68-9b4f-c5b55f5a0764: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:23.896Z [INFO] creating new snapshot: path=/tmp/raft3448851477/snapshots/2-23-1795948103896.tmp +2026-11-29T10:28:23.896Z [INFO] server-94257ef2-662d-5b68-9b4f-c5b55f5a0764: copied to local snapshot: bytes=71 +2026-11-29T10:28:23.896Z [INFO] server-94257ef2-662d-5b68-9b4f-c5b55f5a0764: snapshot restore progress: id=2-23-1795948103896 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:23.896Z [INFO] server-94257ef2-662d-5b68-9b4f-c5b55f5a0764: Installed remote snapshot +2026-11-29T10:28:23.896Z [INFO] server-e17580e3-f9ff-9536-b13a-1bed706fd07c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:23.896Z [WARN] server-94257ef2-662d-5b68-9b4f-c5b55f5a0764: skipping application of old log: index=22 +2026-11-29T10:28:23.896Z [INFO] server-560dd12e-abd8-2d1b-7822-2468e952c049: pipelining replication: peer="{Voter server-94257ef2-662d-5b68-9b4f-c5b55f5a0764 94257ef2-662d-5b68-9b4f-c5b55f5a0764}" +2026-11-29T10:28:23.896Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:28:23.896Z [INFO] server-e17580e3-f9ff-9536-b13a-1bed706fd07c: copied to local snapshot: bytes=71 +2026-11-29T10:28:23.896Z [INFO] server-e17580e3-f9ff-9536-b13a-1bed706fd07c: snapshot restore progress: id=2-23-1795948103896 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:23.896Z [INFO] server-e17580e3-f9ff-9536-b13a-1bed706fd07c: Installed remote snapshot +2026-11-29T10:28:23.897Z [INFO] server-560dd12e-abd8-2d1b-7822-2468e952c049: pipelining replication: peer="{Voter server-e17580e3-f9ff-9536-b13a-1bed706fd07c e17580e3-f9ff-9536-b13a-1bed706fd07c}" +2026-11-29T10:28:24.901Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:28:23.896601819 +0000 UTC m=+14.631655921 (2 nodes), 480ns from start of poll, 1.67954132s from cluster start. Timeout at 2026-11-29 10:28:24.901607259 +0000 UTC m=+15.636661321, 1.0050054s after stability +2026-11-29T10:28:24.914Z [INFO] server-560dd12e-abd8-2d1b-7822-2468e952c049: aborting pipeline replication: peer="{Voter server-94257ef2-662d-5b68-9b4f-c5b55f5a0764 94257ef2-662d-5b68-9b4f-c5b55f5a0764}" +2026-11-29T10:28:24.914Z [INFO] server-560dd12e-abd8-2d1b-7822-2468e952c049: aborting pipeline replication: peer="{Voter server-e17580e3-f9ff-9536-b13a-1bed706fd07c e17580e3-f9ff-9536-b13a-1bed706fd07c}" === RUN TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_false -2025-10-27T04:02:21.665Z [INFO] server-04d98493-1784-f1c6-0403-a3c3010bde91: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-04d98493-1784-f1c6-0403-a3c3010bde91 Address:04d98493-1784-f1c6-0403-a3c3010bde91} {Suffrage:Voter ID:server-05bd16d1-76c5-6c09-d0ac-a908d724246b Address:05bd16d1-76c5-6c09-d0ac-a908d724246b} {Suffrage:Voter ID:server-0ab45ca4-205a-7e0a-d141-be264de81edf Address:0ab45ca4-205a-7e0a-d141-be264de81edf}]" -2025-10-27T04:02:21.665Z [INFO] server-05bd16d1-76c5-6c09-d0ac-a908d724246b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-04d98493-1784-f1c6-0403-a3c3010bde91 Address:04d98493-1784-f1c6-0403-a3c3010bde91} {Suffrage:Voter ID:server-05bd16d1-76c5-6c09-d0ac-a908d724246b Address:05bd16d1-76c5-6c09-d0ac-a908d724246b} {Suffrage:Voter ID:server-0ab45ca4-205a-7e0a-d141-be264de81edf Address:0ab45ca4-205a-7e0a-d141-be264de81edf}]" -2025-10-27T04:02:21.665Z [INFO] server-04d98493-1784-f1c6-0403-a3c3010bde91: entering follower state: follower="Node at 04d98493-1784-f1c6-0403-a3c3010bde91 [Follower]" leader-address= leader-id= -2025-10-27T04:02:21.665Z [INFO] server-05bd16d1-76c5-6c09-d0ac-a908d724246b: entering follower state: follower="Node at 05bd16d1-76c5-6c09-d0ac-a908d724246b [Follower]" leader-address= leader-id= -2025-10-27T04:02:21.665Z [INFO] server-0ab45ca4-205a-7e0a-d141-be264de81edf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-04d98493-1784-f1c6-0403-a3c3010bde91 Address:04d98493-1784-f1c6-0403-a3c3010bde91} {Suffrage:Voter ID:server-05bd16d1-76c5-6c09-d0ac-a908d724246b Address:05bd16d1-76c5-6c09-d0ac-a908d724246b} {Suffrage:Voter ID:server-0ab45ca4-205a-7e0a-d141-be264de81edf Address:0ab45ca4-205a-7e0a-d141-be264de81edf}]" -2025-10-27T04:02:21.665Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:21.665Z [INFO] server-0ab45ca4-205a-7e0a-d141-be264de81edf: entering follower state: follower="Node at 0ab45ca4-205a-7e0a-d141-be264de81edf [Follower]" leader-address= leader-id= -2025-10-27T04:02:22.418Z [WARN] server-05bd16d1-76c5-6c09-d0ac-a908d724246b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:22.418Z [INFO] server-05bd16d1-76c5-6c09-d0ac-a908d724246b: entering candidate state: node="Node at 05bd16d1-76c5-6c09-d0ac-a908d724246b [Candidate]" term=2 -2025-10-27T04:02:22.418Z [INFO] server-05bd16d1-76c5-6c09-d0ac-a908d724246b: election won: term=2 tally=2 -2025-10-27T04:02:22.418Z [INFO] server-05bd16d1-76c5-6c09-d0ac-a908d724246b: entering leader state: leader="Node at 05bd16d1-76c5-6c09-d0ac-a908d724246b [Leader]" -2025-10-27T04:02:22.418Z [INFO] server-05bd16d1-76c5-6c09-d0ac-a908d724246b: added peer, starting replication: peer=server-04d98493-1784-f1c6-0403-a3c3010bde91 -2025-10-27T04:02:22.418Z [INFO] server-05bd16d1-76c5-6c09-d0ac-a908d724246b: added peer, starting replication: peer=server-0ab45ca4-205a-7e0a-d141-be264de81edf -2025-10-27T04:02:22.419Z [INFO] server-05bd16d1-76c5-6c09-d0ac-a908d724246b: pipelining replication: peer="{Voter server-0ab45ca4-205a-7e0a-d141-be264de81edf 0ab45ca4-205a-7e0a-d141-be264de81edf}" -2025-10-27T04:02:22.419Z [INFO] server-05bd16d1-76c5-6c09-d0ac-a908d724246b: pipelining replication: peer="{Voter server-04d98493-1784-f1c6-0403-a3c3010bde91 04d98493-1784-f1c6-0403-a3c3010bde91}" -2025-10-27T04:02:23.424Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:22.419029747 +0000 UTC m=+17.413902225 (1 nodes), 753.362149ms from start of poll, 753.663951ms from cluster start. Timeout at 2025-10-27 04:02:23.424031507 +0000 UTC m=+18.418904025, 1.0050018s after stability -2025-10-27T04:02:23.424Z [INFO] server-05bd16d1-76c5-6c09-d0ac-a908d724246b: starting snapshot up to: index=12 -2025-10-27T04:02:23.424Z [INFO] creating new snapshot: path=/tmp/raft3115061309/snapshots/2-12-1761537743424.tmp -2025-10-27T04:02:23.425Z [INFO] server-05bd16d1-76c5-6c09-d0ac-a908d724246b: snapshot complete up to: index=12 -2025-10-27T04:02:23.426Z [INFO] creating new snapshot: path=/tmp/raft3115061309/snapshots/2-23-1761537743426.tmp -2025-10-27T04:02:23.427Z [INFO] server-05bd16d1-76c5-6c09-d0ac-a908d724246b: copied to local snapshot: bytes=71 -2025-10-27T04:02:23.428Z [INFO] server-05bd16d1-76c5-6c09-d0ac-a908d724246b: snapshot restore progress: id=2-23-1761537743426 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:23.428Z [INFO] server-05bd16d1-76c5-6c09-d0ac-a908d724246b: removing all old logs from log store -2025-10-27T04:02:23.428Z [INFO] server-05bd16d1-76c5-6c09-d0ac-a908d724246b: compacting logs: from=1 to=22 -2025-10-27T04:02:23.428Z [INFO] server-05bd16d1-76c5-6c09-d0ac-a908d724246b: restored user snapshot: index=23 -2025-10-27T04:02:23.428Z [ERROR] server-05bd16d1-76c5-6c09-d0ac-a908d724246b: failed to get log: index=22 error="log not found" -2025-10-27T04:02:23.428Z [ERROR] server-05bd16d1-76c5-6c09-d0ac-a908d724246b: failed to get log: index=22 error="log not found" -2025-10-27T04:02:23.428Z [INFO] server-05bd16d1-76c5-6c09-d0ac-a908d724246b: aborting pipeline replication: peer="{Voter server-04d98493-1784-f1c6-0403-a3c3010bde91 04d98493-1784-f1c6-0403-a3c3010bde91}" -2025-10-27T04:02:23.428Z [INFO] server-05bd16d1-76c5-6c09-d0ac-a908d724246b: aborting pipeline replication: peer="{Voter server-0ab45ca4-205a-7e0a-d141-be264de81edf 0ab45ca4-205a-7e0a-d141-be264de81edf}" -2025-10-27T04:02:23.437Z [ERROR] server-05bd16d1-76c5-6c09-d0ac-a908d724246b: failed to get log: index=22 error="log not found" -2025-10-27T04:02:23.438Z [INFO] creating new snapshot: path=/tmp/raft1965725414/snapshots/2-23-1761537743438.tmp -2025-10-27T04:02:23.438Z [ERROR] server-05bd16d1-76c5-6c09-d0ac-a908d724246b: failed to get log: index=22 error="log not found" -2025-10-27T04:02:23.438Z [INFO] server-04d98493-1784-f1c6-0403-a3c3010bde91: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:23.438Z [INFO] server-04d98493-1784-f1c6-0403-a3c3010bde91: copied to local snapshot: bytes=71 -2025-10-27T04:02:23.438Z [INFO] creating new snapshot: path=/tmp/raft3004960850/snapshots/2-23-1761537743438.tmp -2025-10-27T04:02:23.439Z [INFO] server-04d98493-1784-f1c6-0403-a3c3010bde91: snapshot restore progress: id=2-23-1761537743438 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:23.439Z [INFO] server-04d98493-1784-f1c6-0403-a3c3010bde91: removing all old logs from log store -2025-10-27T04:02:23.439Z [INFO] server-04d98493-1784-f1c6-0403-a3c3010bde91: compacting logs: from=1 to=22 -2025-10-27T04:02:23.439Z [INFO] server-04d98493-1784-f1c6-0403-a3c3010bde91: Installed remote snapshot -2025-10-27T04:02:23.439Z [INFO] server-0ab45ca4-205a-7e0a-d141-be264de81edf: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:23.439Z [INFO] server-05bd16d1-76c5-6c09-d0ac-a908d724246b: pipelining replication: peer="{Voter server-04d98493-1784-f1c6-0403-a3c3010bde91 04d98493-1784-f1c6-0403-a3c3010bde91}" -2025-10-27T04:02:23.439Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:02:23.439Z [INFO] server-0ab45ca4-205a-7e0a-d141-be264de81edf: copied to local snapshot: bytes=71 -2025-10-27T04:02:23.439Z [INFO] server-0ab45ca4-205a-7e0a-d141-be264de81edf: snapshot restore progress: id=2-23-1761537743438 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:23.439Z [INFO] server-0ab45ca4-205a-7e0a-d141-be264de81edf: removing all old logs from log store -2025-10-27T04:02:23.439Z [INFO] server-0ab45ca4-205a-7e0a-d141-be264de81edf: compacting logs: from=1 to=22 -2025-10-27T04:02:23.439Z [INFO] server-0ab45ca4-205a-7e0a-d141-be264de81edf: Installed remote snapshot -2025-10-27T04:02:23.439Z [INFO] server-05bd16d1-76c5-6c09-d0ac-a908d724246b: pipelining replication: peer="{Voter server-0ab45ca4-205a-7e0a-d141-be264de81edf 0ab45ca4-205a-7e0a-d141-be264de81edf}" -2025-10-27T04:02:24.445Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:02:23.43928604 +0000 UTC m=+18.434158518 (2 nodes), 1.2µs from start of poll, 1.773920244s from cluster start. Timeout at 2025-10-27 04:02:24.4442922 +0000 UTC m=+19.439164598, 1.00500608s after stability -2025-10-27T04:02:24.457Z [INFO] server-05bd16d1-76c5-6c09-d0ac-a908d724246b: aborting pipeline replication: peer="{Voter server-0ab45ca4-205a-7e0a-d141-be264de81edf 0ab45ca4-205a-7e0a-d141-be264de81edf}" -2025-10-27T04:02:24.457Z [INFO] server-05bd16d1-76c5-6c09-d0ac-a908d724246b: aborting pipeline replication: peer="{Voter server-04d98493-1784-f1c6-0403-a3c3010bde91 04d98493-1784-f1c6-0403-a3c3010bde91}" +2026-11-29T10:28:24.915Z [INFO] server-91c1d931-9ad5-b373-3434-734706994147: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-91c1d931-9ad5-b373-3434-734706994147 Address:91c1d931-9ad5-b373-3434-734706994147} {Suffrage:Voter ID:server-93c54bac-0b0b-05e6-e910-e4e760e7e855 Address:93c54bac-0b0b-05e6-e910-e4e760e7e855} {Suffrage:Voter ID:server-b9b94111-ee01-e6d3-0f41-685a37718231 Address:b9b94111-ee01-e6d3-0f41-685a37718231}]" +2026-11-29T10:28:24.915Z [INFO] server-93c54bac-0b0b-05e6-e910-e4e760e7e855: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-91c1d931-9ad5-b373-3434-734706994147 Address:91c1d931-9ad5-b373-3434-734706994147} {Suffrage:Voter ID:server-93c54bac-0b0b-05e6-e910-e4e760e7e855 Address:93c54bac-0b0b-05e6-e910-e4e760e7e855} {Suffrage:Voter ID:server-b9b94111-ee01-e6d3-0f41-685a37718231 Address:b9b94111-ee01-e6d3-0f41-685a37718231}]" +2026-11-29T10:28:24.915Z [INFO] server-91c1d931-9ad5-b373-3434-734706994147: entering follower state: follower="Node at 91c1d931-9ad5-b373-3434-734706994147 [Follower]" leader-address= leader-id= +2026-11-29T10:28:24.915Z [INFO] server-b9b94111-ee01-e6d3-0f41-685a37718231: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-91c1d931-9ad5-b373-3434-734706994147 Address:91c1d931-9ad5-b373-3434-734706994147} {Suffrage:Voter ID:server-93c54bac-0b0b-05e6-e910-e4e760e7e855 Address:93c54bac-0b0b-05e6-e910-e4e760e7e855} {Suffrage:Voter ID:server-b9b94111-ee01-e6d3-0f41-685a37718231 Address:b9b94111-ee01-e6d3-0f41-685a37718231}]" +2026-11-29T10:28:24.915Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:24.915Z [INFO] server-93c54bac-0b0b-05e6-e910-e4e760e7e855: entering follower state: follower="Node at 93c54bac-0b0b-05e6-e910-e4e760e7e855 [Follower]" leader-address= leader-id= +2026-11-29T10:28:24.915Z [INFO] server-b9b94111-ee01-e6d3-0f41-685a37718231: entering follower state: follower="Node at b9b94111-ee01-e6d3-0f41-685a37718231 [Follower]" leader-address= leader-id= +2026-11-29T10:28:25.495Z [WARN] server-91c1d931-9ad5-b373-3434-734706994147: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:25.503Z [INFO] server-91c1d931-9ad5-b373-3434-734706994147: entering candidate state: node="Node at 91c1d931-9ad5-b373-3434-734706994147 [Candidate]" term=2 +2026-11-29T10:28:25.503Z [INFO] server-91c1d931-9ad5-b373-3434-734706994147: election won: term=2 tally=2 +2026-11-29T10:28:25.503Z [INFO] server-91c1d931-9ad5-b373-3434-734706994147: entering leader state: leader="Node at 91c1d931-9ad5-b373-3434-734706994147 [Leader]" +2026-11-29T10:28:25.503Z [INFO] server-91c1d931-9ad5-b373-3434-734706994147: added peer, starting replication: peer=server-93c54bac-0b0b-05e6-e910-e4e760e7e855 +2026-11-29T10:28:25.503Z [INFO] server-91c1d931-9ad5-b373-3434-734706994147: added peer, starting replication: peer=server-b9b94111-ee01-e6d3-0f41-685a37718231 +2026-11-29T10:28:25.503Z [INFO] server-91c1d931-9ad5-b373-3434-734706994147: pipelining replication: peer="{Voter server-b9b94111-ee01-e6d3-0f41-685a37718231 b9b94111-ee01-e6d3-0f41-685a37718231}" +2026-11-29T10:28:25.503Z [WARN] server-93c54bac-0b0b-05e6-e910-e4e760e7e855: rejecting vote request since our last term is greater: candidate=91c1d931-9ad5-b373-3434-734706994147 last-term=2 last-candidate-term=1 +2026-11-29T10:28:25.503Z [INFO] server-91c1d931-9ad5-b373-3434-734706994147: pipelining replication: peer="{Voter server-93c54bac-0b0b-05e6-e910-e4e760e7e855 93c54bac-0b0b-05e6-e910-e4e760e7e855}" +2026-11-29T10:28:26.509Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:25.503803819 +0000 UTC m=+16.238857961 (1 nodes), 588.34828ms from start of poll, 588.7422ms from cluster start. Timeout at 2026-11-29 10:28:26.508811899 +0000 UTC m=+17.243866041, 1.00500808s after stability +2026-11-29T10:28:26.510Z [INFO] server-91c1d931-9ad5-b373-3434-734706994147: starting snapshot up to: index=12 +2026-11-29T10:28:26.510Z [INFO] creating new snapshot: path=/tmp/raft3740959165/snapshots/2-12-1795948106510.tmp +2026-11-29T10:28:26.510Z [INFO] server-91c1d931-9ad5-b373-3434-734706994147: snapshot complete up to: index=12 +2026-11-29T10:28:26.511Z [INFO] creating new snapshot: path=/tmp/raft3740959165/snapshots/2-23-1795948106511.tmp +2026-11-29T10:28:26.511Z [INFO] server-91c1d931-9ad5-b373-3434-734706994147: copied to local snapshot: bytes=71 +2026-11-29T10:28:26.512Z [INFO] server-91c1d931-9ad5-b373-3434-734706994147: snapshot restore progress: id=2-23-1795948106511 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:26.512Z [INFO] server-91c1d931-9ad5-b373-3434-734706994147: removing all old logs from log store +2026-11-29T10:28:26.512Z [INFO] server-91c1d931-9ad5-b373-3434-734706994147: compacting logs: from=1 to=22 +2026-11-29T10:28:26.512Z [INFO] server-91c1d931-9ad5-b373-3434-734706994147: restored user snapshot: index=23 +2026-11-29T10:28:26.512Z [ERROR] server-91c1d931-9ad5-b373-3434-734706994147: failed to get log: index=22 error="log not found" +2026-11-29T10:28:26.512Z [INFO] server-91c1d931-9ad5-b373-3434-734706994147: aborting pipeline replication: peer="{Voter server-93c54bac-0b0b-05e6-e910-e4e760e7e855 93c54bac-0b0b-05e6-e910-e4e760e7e855}" +2026-11-29T10:28:26.512Z [ERROR] server-91c1d931-9ad5-b373-3434-734706994147: failed to get log: index=22 error="log not found" +2026-11-29T10:28:26.512Z [INFO] server-91c1d931-9ad5-b373-3434-734706994147: aborting pipeline replication: peer="{Voter server-b9b94111-ee01-e6d3-0f41-685a37718231 b9b94111-ee01-e6d3-0f41-685a37718231}" +2026-11-29T10:28:26.520Z [ERROR] server-91c1d931-9ad5-b373-3434-734706994147: failed to get log: index=22 error="log not found" +2026-11-29T10:28:26.521Z [ERROR] server-91c1d931-9ad5-b373-3434-734706994147: failed to get log: index=22 error="log not found" +2026-11-29T10:28:26.521Z [INFO] creating new snapshot: path=/tmp/raft1553425721/snapshots/2-23-1795948106521.tmp +2026-11-29T10:28:26.521Z [INFO] server-b9b94111-ee01-e6d3-0f41-685a37718231: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:26.521Z [INFO] creating new snapshot: path=/tmp/raft1519407855/snapshots/2-23-1795948106521.tmp +2026-11-29T10:28:26.521Z [INFO] server-b9b94111-ee01-e6d3-0f41-685a37718231: copied to local snapshot: bytes=71 +2026-11-29T10:28:26.521Z [INFO] server-93c54bac-0b0b-05e6-e910-e4e760e7e855: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:26.522Z [INFO] server-b9b94111-ee01-e6d3-0f41-685a37718231: snapshot restore progress: id=2-23-1795948106521 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:26.522Z [INFO] server-93c54bac-0b0b-05e6-e910-e4e760e7e855: copied to local snapshot: bytes=71 +2026-11-29T10:28:26.522Z [INFO] server-b9b94111-ee01-e6d3-0f41-685a37718231: removing all old logs from log store +2026-11-29T10:28:26.522Z [INFO] server-b9b94111-ee01-e6d3-0f41-685a37718231: compacting logs: from=1 to=22 +2026-11-29T10:28:26.522Z [INFO] server-b9b94111-ee01-e6d3-0f41-685a37718231: Installed remote snapshot +2026-11-29T10:28:26.522Z [WARN] server-b9b94111-ee01-e6d3-0f41-685a37718231: skipping application of old log: index=22 +2026-11-29T10:28:26.522Z [INFO] server-91c1d931-9ad5-b373-3434-734706994147: pipelining replication: peer="{Voter server-b9b94111-ee01-e6d3-0f41-685a37718231 b9b94111-ee01-e6d3-0f41-685a37718231}" +2026-11-29T10:28:26.522Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:28:26.522Z [INFO] server-93c54bac-0b0b-05e6-e910-e4e760e7e855: snapshot restore progress: id=2-23-1795948106521 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:26.522Z [INFO] server-93c54bac-0b0b-05e6-e910-e4e760e7e855: removing all old logs from log store +2026-11-29T10:28:26.522Z [INFO] server-93c54bac-0b0b-05e6-e910-e4e760e7e855: compacting logs: from=1 to=22 +2026-11-29T10:28:26.522Z [INFO] server-93c54bac-0b0b-05e6-e910-e4e760e7e855: Installed remote snapshot +2026-11-29T10:28:26.522Z [INFO] server-91c1d931-9ad5-b373-3434-734706994147: pipelining replication: peer="{Voter server-93c54bac-0b0b-05e6-e910-e4e760e7e855 93c54bac-0b0b-05e6-e910-e4e760e7e855}" +2026-11-29T10:28:27.528Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:28:26.522403299 +0000 UTC m=+17.257457441 (2 nodes), 760ns from start of poll, 1.60734168s from cluster start. Timeout at 2026-11-29 10:28:27.527413259 +0000 UTC m=+18.262467441, 1.00501s after stability +2026-11-29T10:28:27.540Z [INFO] server-91c1d931-9ad5-b373-3434-734706994147: aborting pipeline replication: peer="{Voter server-b9b94111-ee01-e6d3-0f41-685a37718231 b9b94111-ee01-e6d3-0f41-685a37718231}" +2026-11-29T10:28:27.540Z [INFO] server-91c1d931-9ad5-b373-3434-734706994147: aborting pipeline replication: peer="{Voter server-93c54bac-0b0b-05e6-e910-e4e760e7e855 93c54bac-0b0b-05e6-e910-e4e760e7e855}" === RUN TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_true -2025-10-27T04:02:24.458Z [INFO] server-ca69c3df-ea39-3913-06ec-86b598e94c27: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ca69c3df-ea39-3913-06ec-86b598e94c27 Address:ca69c3df-ea39-3913-06ec-86b598e94c27} {Suffrage:Voter ID:server-7ce29438-e738-c1ce-c543-f42c93713715 Address:7ce29438-e738-c1ce-c543-f42c93713715} {Suffrage:Voter ID:server-412fcf48-d80a-7d91-e5c0-de8ba4af6053 Address:412fcf48-d80a-7d91-e5c0-de8ba4af6053}]" -2025-10-27T04:02:24.458Z [INFO] server-7ce29438-e738-c1ce-c543-f42c93713715: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ca69c3df-ea39-3913-06ec-86b598e94c27 Address:ca69c3df-ea39-3913-06ec-86b598e94c27} {Suffrage:Voter ID:server-7ce29438-e738-c1ce-c543-f42c93713715 Address:7ce29438-e738-c1ce-c543-f42c93713715} {Suffrage:Voter ID:server-412fcf48-d80a-7d91-e5c0-de8ba4af6053 Address:412fcf48-d80a-7d91-e5c0-de8ba4af6053}]" -2025-10-27T04:02:24.458Z [INFO] server-ca69c3df-ea39-3913-06ec-86b598e94c27: entering follower state: follower="Node at ca69c3df-ea39-3913-06ec-86b598e94c27 [Follower]" leader-address= leader-id= -2025-10-27T04:02:24.458Z [INFO] server-412fcf48-d80a-7d91-e5c0-de8ba4af6053: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ca69c3df-ea39-3913-06ec-86b598e94c27 Address:ca69c3df-ea39-3913-06ec-86b598e94c27} {Suffrage:Voter ID:server-7ce29438-e738-c1ce-c543-f42c93713715 Address:7ce29438-e738-c1ce-c543-f42c93713715} {Suffrage:Voter ID:server-412fcf48-d80a-7d91-e5c0-de8ba4af6053 Address:412fcf48-d80a-7d91-e5c0-de8ba4af6053}]" -2025-10-27T04:02:24.458Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:24.458Z [INFO] server-7ce29438-e738-c1ce-c543-f42c93713715: entering follower state: follower="Node at 7ce29438-e738-c1ce-c543-f42c93713715 [Follower]" leader-address= leader-id= -2025-10-27T04:02:24.458Z [INFO] server-412fcf48-d80a-7d91-e5c0-de8ba4af6053: entering follower state: follower="Node at 412fcf48-d80a-7d91-e5c0-de8ba4af6053 [Follower]" leader-address= leader-id= -2025-10-27T04:02:25.029Z [WARN] server-7ce29438-e738-c1ce-c543-f42c93713715: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:25.029Z [INFO] server-7ce29438-e738-c1ce-c543-f42c93713715: entering candidate state: node="Node at 7ce29438-e738-c1ce-c543-f42c93713715 [Candidate]" term=2 -2025-10-27T04:02:25.029Z [INFO] server-7ce29438-e738-c1ce-c543-f42c93713715: election won: term=2 tally=2 -2025-10-27T04:02:25.029Z [INFO] server-7ce29438-e738-c1ce-c543-f42c93713715: entering leader state: leader="Node at 7ce29438-e738-c1ce-c543-f42c93713715 [Leader]" -2025-10-27T04:02:25.029Z [INFO] server-7ce29438-e738-c1ce-c543-f42c93713715: added peer, starting replication: peer=server-ca69c3df-ea39-3913-06ec-86b598e94c27 -2025-10-27T04:02:25.029Z [INFO] server-7ce29438-e738-c1ce-c543-f42c93713715: added peer, starting replication: peer=server-412fcf48-d80a-7d91-e5c0-de8ba4af6053 -2025-10-27T04:02:25.029Z [INFO] server-7ce29438-e738-c1ce-c543-f42c93713715: pipelining replication: peer="{Voter server-412fcf48-d80a-7d91-e5c0-de8ba4af6053 412fcf48-d80a-7d91-e5c0-de8ba4af6053}" -2025-10-27T04:02:25.029Z [INFO] server-7ce29438-e738-c1ce-c543-f42c93713715: pipelining replication: peer="{Voter server-ca69c3df-ea39-3913-06ec-86b598e94c27 ca69c3df-ea39-3913-06ec-86b598e94c27}" -2025-10-27T04:02:26.034Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:25.029443468 +0000 UTC m=+20.024315906 (1 nodes), 571.230424ms from start of poll, 571.482546ms from cluster start. Timeout at 2025-10-27 04:02:26.034444828 +0000 UTC m=+21.029317266, 1.00500136s after stability -2025-10-27T04:02:26.035Z [INFO] server-7ce29438-e738-c1ce-c543-f42c93713715: starting snapshot up to: index=12 -2025-10-27T04:02:26.035Z [INFO] creating new snapshot: path=/tmp/raft2264117913/snapshots/2-12-1761537746035.tmp -2025-10-27T04:02:26.035Z [INFO] server-7ce29438-e738-c1ce-c543-f42c93713715: snapshot complete up to: index=12 -2025-10-27T04:02:26.036Z [INFO] server-22fe3eee-18f3-1bb2-2bff-f15ddbb11561: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-22fe3eee-18f3-1bb2-2bff-f15ddbb11561 Address:22fe3eee-18f3-1bb2-2bff-f15ddbb11561} {Suffrage:Voter ID:server-edeb4b77-ed1d-d2b5-e747-2902099c697e Address:edeb4b77-ed1d-d2b5-e747-2902099c697e} {Suffrage:Voter ID:server-ca7093f0-3407-a770-2c0d-015092d4f9ef Address:ca7093f0-3407-a770-2c0d-015092d4f9ef}]" -2025-10-27T04:02:26.036Z [INFO] server-edeb4b77-ed1d-d2b5-e747-2902099c697e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-22fe3eee-18f3-1bb2-2bff-f15ddbb11561 Address:22fe3eee-18f3-1bb2-2bff-f15ddbb11561} {Suffrage:Voter ID:server-edeb4b77-ed1d-d2b5-e747-2902099c697e Address:edeb4b77-ed1d-d2b5-e747-2902099c697e} {Suffrage:Voter ID:server-ca7093f0-3407-a770-2c0d-015092d4f9ef Address:ca7093f0-3407-a770-2c0d-015092d4f9ef}]" -2025-10-27T04:02:26.036Z [INFO] server-ca7093f0-3407-a770-2c0d-015092d4f9ef: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-22fe3eee-18f3-1bb2-2bff-f15ddbb11561 Address:22fe3eee-18f3-1bb2-2bff-f15ddbb11561} {Suffrage:Voter ID:server-edeb4b77-ed1d-d2b5-e747-2902099c697e Address:edeb4b77-ed1d-d2b5-e747-2902099c697e} {Suffrage:Voter ID:server-ca7093f0-3407-a770-2c0d-015092d4f9ef Address:ca7093f0-3407-a770-2c0d-015092d4f9ef}]" -2025-10-27T04:02:26.036Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:26.036Z [INFO] server-22fe3eee-18f3-1bb2-2bff-f15ddbb11561: entering follower state: follower="Node at 22fe3eee-18f3-1bb2-2bff-f15ddbb11561 [Follower]" leader-address= leader-id= -2025-10-27T04:02:26.036Z [INFO] server-edeb4b77-ed1d-d2b5-e747-2902099c697e: entering follower state: follower="Node at edeb4b77-ed1d-d2b5-e747-2902099c697e [Follower]" leader-address= leader-id= -2025-10-27T04:02:26.036Z [INFO] server-ca7093f0-3407-a770-2c0d-015092d4f9ef: entering follower state: follower="Node at ca7093f0-3407-a770-2c0d-015092d4f9ef [Follower]" leader-address= leader-id= -2025-10-27T04:02:26.538Z [WARN] server-22fe3eee-18f3-1bb2-2bff-f15ddbb11561: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:26.538Z [INFO] server-22fe3eee-18f3-1bb2-2bff-f15ddbb11561: entering candidate state: node="Node at 22fe3eee-18f3-1bb2-2bff-f15ddbb11561 [Candidate]" term=2 -2025-10-27T04:02:26.539Z [INFO] server-22fe3eee-18f3-1bb2-2bff-f15ddbb11561: election won: term=2 tally=2 -2025-10-27T04:02:26.539Z [INFO] server-22fe3eee-18f3-1bb2-2bff-f15ddbb11561: entering leader state: leader="Node at 22fe3eee-18f3-1bb2-2bff-f15ddbb11561 [Leader]" -2025-10-27T04:02:26.539Z [INFO] server-22fe3eee-18f3-1bb2-2bff-f15ddbb11561: added peer, starting replication: peer=server-edeb4b77-ed1d-d2b5-e747-2902099c697e -2025-10-27T04:02:26.539Z [INFO] server-22fe3eee-18f3-1bb2-2bff-f15ddbb11561: added peer, starting replication: peer=server-ca7093f0-3407-a770-2c0d-015092d4f9ef -2025-10-27T04:02:26.539Z [INFO] server-22fe3eee-18f3-1bb2-2bff-f15ddbb11561: pipelining replication: peer="{Voter server-edeb4b77-ed1d-d2b5-e747-2902099c697e edeb4b77-ed1d-d2b5-e747-2902099c697e}" -2025-10-27T04:02:26.539Z [INFO] server-22fe3eee-18f3-1bb2-2bff-f15ddbb11561: pipelining replication: peer="{Voter server-ca7093f0-3407-a770-2c0d-015092d4f9ef ca7093f0-3407-a770-2c0d-015092d4f9ef}" -2025-10-27T04:02:27.544Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:26.539227467 +0000 UTC m=+21.534099985 (1 nodes), 502.560115ms from start of poll, 502.847197ms from cluster start. Timeout at 2025-10-27 04:02:27.544231307 +0000 UTC m=+22.539103785, 1.0050038s after stability -2025-10-27T04:02:27.544Z [INFO] creating new snapshot: path=/tmp/raft503646495/snapshots/2-13-1761537747544.tmp -2025-10-27T04:02:27.545Z [INFO] server-22fe3eee-18f3-1bb2-2bff-f15ddbb11561: copied to local snapshot: bytes=71 -2025-10-27T04:02:27.545Z [INFO] server-22fe3eee-18f3-1bb2-2bff-f15ddbb11561: snapshot restore progress: id=2-13-1761537747544 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:27.545Z [INFO] server-22fe3eee-18f3-1bb2-2bff-f15ddbb11561: restored user snapshot: index=13 -2025-10-27T04:02:27.545Z [ERROR] server-22fe3eee-18f3-1bb2-2bff-f15ddbb11561: failed to get log: index=3 error="log not found" -2025-10-27T04:02:27.545Z [INFO] server-22fe3eee-18f3-1bb2-2bff-f15ddbb11561: aborting pipeline replication: peer="{Voter server-ca7093f0-3407-a770-2c0d-015092d4f9ef ca7093f0-3407-a770-2c0d-015092d4f9ef}" -2025-10-27T04:02:27.545Z [ERROR] server-22fe3eee-18f3-1bb2-2bff-f15ddbb11561: failed to get log: index=3 error="log not found" -2025-10-27T04:02:27.545Z [INFO] server-22fe3eee-18f3-1bb2-2bff-f15ddbb11561: aborting pipeline replication: peer="{Voter server-edeb4b77-ed1d-d2b5-e747-2902099c697e edeb4b77-ed1d-d2b5-e747-2902099c697e}" -2025-10-27T04:02:27.550Z [ERROR] server-22fe3eee-18f3-1bb2-2bff-f15ddbb11561: failed to get log: index=3 error="log not found" -2025-10-27T04:02:27.551Z [INFO] creating new snapshot: path=/tmp/raft3626763421/snapshots/2-13-1761537747551.tmp -2025-10-27T04:02:27.551Z [INFO] server-ca7093f0-3407-a770-2c0d-015092d4f9ef: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:27.551Z [INFO] server-ca7093f0-3407-a770-2c0d-015092d4f9ef: copied to local snapshot: bytes=71 -2025-10-27T04:02:27.552Z [INFO] server-ca7093f0-3407-a770-2c0d-015092d4f9ef: snapshot restore progress: id=2-13-1761537747551 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:27.552Z [INFO] server-ca7093f0-3407-a770-2c0d-015092d4f9ef: Installed remote snapshot -2025-10-27T04:02:27.552Z [INFO] server-22fe3eee-18f3-1bb2-2bff-f15ddbb11561: pipelining replication: peer="{Voter server-ca7093f0-3407-a770-2c0d-015092d4f9ef ca7093f0-3407-a770-2c0d-015092d4f9ef}" -2025-10-27T04:02:27.552Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:02:27.555Z [ERROR] server-22fe3eee-18f3-1bb2-2bff-f15ddbb11561: failed to get log: index=3 error="log not found" -2025-10-27T04:02:27.555Z [INFO] creating new snapshot: path=/tmp/raft3111292940/snapshots/2-13-1761537747555.tmp -2025-10-27T04:02:27.556Z [INFO] server-edeb4b77-ed1d-d2b5-e747-2902099c697e: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:27.556Z [INFO] server-edeb4b77-ed1d-d2b5-e747-2902099c697e: copied to local snapshot: bytes=71 -2025-10-27T04:02:27.556Z [INFO] server-edeb4b77-ed1d-d2b5-e747-2902099c697e: snapshot restore progress: id=2-13-1761537747555 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:27.556Z [INFO] server-edeb4b77-ed1d-d2b5-e747-2902099c697e: Installed remote snapshot -2025-10-27T04:02:27.556Z [INFO] server-22fe3eee-18f3-1bb2-2bff-f15ddbb11561: pipelining replication: peer="{Voter server-edeb4b77-ed1d-d2b5-e747-2902099c697e edeb4b77-ed1d-d2b5-e747-2902099c697e}" -2025-10-27T04:02:28.558Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:02:27.552432352 +0000 UTC m=+22.547304830 (2 nodes), 920ns from start of poll, 1.516052042s from cluster start. Timeout at 2025-10-27 04:02:28.557436312 +0000 UTC m=+23.552308790, 1.00500396s after stability -2025-10-27T04:02:28.570Z [INFO] server-7ce29438-e738-c1ce-c543-f42c93713715: aborting pipeline replication: peer="{Voter server-412fcf48-d80a-7d91-e5c0-de8ba4af6053 412fcf48-d80a-7d91-e5c0-de8ba4af6053}" -2025-10-27T04:02:28.570Z [INFO] server-7ce29438-e738-c1ce-c543-f42c93713715: aborting pipeline replication: peer="{Voter server-ca69c3df-ea39-3913-06ec-86b598e94c27 ca69c3df-ea39-3913-06ec-86b598e94c27}" +2026-11-29T10:28:27.541Z [INFO] server-04acb096-463d-ef3d-b3a5-e247a7c42059: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-04acb096-463d-ef3d-b3a5-e247a7c42059 Address:04acb096-463d-ef3d-b3a5-e247a7c42059} {Suffrage:Voter ID:server-5efc46ad-7f5d-e7ab-6348-4f38ffac9c30 Address:5efc46ad-7f5d-e7ab-6348-4f38ffac9c30} {Suffrage:Voter ID:server-af9447c1-ff88-5424-0120-d7732a612efc Address:af9447c1-ff88-5424-0120-d7732a612efc}]" +2026-11-29T10:28:27.541Z [INFO] server-5efc46ad-7f5d-e7ab-6348-4f38ffac9c30: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-04acb096-463d-ef3d-b3a5-e247a7c42059 Address:04acb096-463d-ef3d-b3a5-e247a7c42059} {Suffrage:Voter ID:server-5efc46ad-7f5d-e7ab-6348-4f38ffac9c30 Address:5efc46ad-7f5d-e7ab-6348-4f38ffac9c30} {Suffrage:Voter ID:server-af9447c1-ff88-5424-0120-d7732a612efc Address:af9447c1-ff88-5424-0120-d7732a612efc}]" +2026-11-29T10:28:27.541Z [INFO] server-04acb096-463d-ef3d-b3a5-e247a7c42059: entering follower state: follower="Node at 04acb096-463d-ef3d-b3a5-e247a7c42059 [Follower]" leader-address= leader-id= +2026-11-29T10:28:27.541Z [INFO] server-af9447c1-ff88-5424-0120-d7732a612efc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-04acb096-463d-ef3d-b3a5-e247a7c42059 Address:04acb096-463d-ef3d-b3a5-e247a7c42059} {Suffrage:Voter ID:server-5efc46ad-7f5d-e7ab-6348-4f38ffac9c30 Address:5efc46ad-7f5d-e7ab-6348-4f38ffac9c30} {Suffrage:Voter ID:server-af9447c1-ff88-5424-0120-d7732a612efc Address:af9447c1-ff88-5424-0120-d7732a612efc}]" +2026-11-29T10:28:27.541Z [INFO] server-5efc46ad-7f5d-e7ab-6348-4f38ffac9c30: entering follower state: follower="Node at 5efc46ad-7f5d-e7ab-6348-4f38ffac9c30 [Follower]" leader-address= leader-id= +2026-11-29T10:28:27.542Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:27.542Z [INFO] server-af9447c1-ff88-5424-0120-d7732a612efc: entering follower state: follower="Node at af9447c1-ff88-5424-0120-d7732a612efc [Follower]" leader-address= leader-id= +2026-11-29T10:28:28.104Z [WARN] server-5efc46ad-7f5d-e7ab-6348-4f38ffac9c30: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:28.104Z [INFO] server-5efc46ad-7f5d-e7ab-6348-4f38ffac9c30: entering candidate state: node="Node at 5efc46ad-7f5d-e7ab-6348-4f38ffac9c30 [Candidate]" term=2 +2026-11-29T10:28:28.105Z [INFO] server-5efc46ad-7f5d-e7ab-6348-4f38ffac9c30: election won: term=2 tally=2 +2026-11-29T10:28:28.105Z [INFO] server-5efc46ad-7f5d-e7ab-6348-4f38ffac9c30: entering leader state: leader="Node at 5efc46ad-7f5d-e7ab-6348-4f38ffac9c30 [Leader]" +2026-11-29T10:28:28.105Z [INFO] server-5efc46ad-7f5d-e7ab-6348-4f38ffac9c30: added peer, starting replication: peer=server-04acb096-463d-ef3d-b3a5-e247a7c42059 +2026-11-29T10:28:28.105Z [INFO] server-5efc46ad-7f5d-e7ab-6348-4f38ffac9c30: added peer, starting replication: peer=server-af9447c1-ff88-5424-0120-d7732a612efc +2026-11-29T10:28:28.105Z [INFO] server-5efc46ad-7f5d-e7ab-6348-4f38ffac9c30: pipelining replication: peer="{Voter server-af9447c1-ff88-5424-0120-d7732a612efc af9447c1-ff88-5424-0120-d7732a612efc}" +2026-11-29T10:28:28.105Z [INFO] server-5efc46ad-7f5d-e7ab-6348-4f38ffac9c30: pipelining replication: peer="{Voter server-04acb096-463d-ef3d-b3a5-e247a7c42059 04acb096-463d-ef3d-b3a5-e247a7c42059}" +2026-11-29T10:28:29.110Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:28.105235299 +0000 UTC m=+18.840289441 (1 nodes), 563.16664ms from start of poll, 563.5824ms from cluster start. Timeout at 2026-11-29 10:28:29.110238939 +0000 UTC m=+19.845293081, 1.00500364s after stability +2026-11-29T10:28:29.111Z [INFO] server-5efc46ad-7f5d-e7ab-6348-4f38ffac9c30: starting snapshot up to: index=12 +2026-11-29T10:28:29.111Z [INFO] creating new snapshot: path=/tmp/raft96592445/snapshots/2-12-1795948109111.tmp +2026-11-29T10:28:29.112Z [INFO] server-5efc46ad-7f5d-e7ab-6348-4f38ffac9c30: snapshot complete up to: index=12 +2026-11-29T10:28:29.113Z [INFO] server-82f905cd-5130-5fb5-a320-776feb9a7586: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-82f905cd-5130-5fb5-a320-776feb9a7586 Address:82f905cd-5130-5fb5-a320-776feb9a7586} {Suffrage:Voter ID:server-013cbf0d-bc9b-0aea-cd3d-90bffe362de4 Address:013cbf0d-bc9b-0aea-cd3d-90bffe362de4} {Suffrage:Voter ID:server-0f0f9b01-5eaf-1f03-12d4-c96ab15bedd9 Address:0f0f9b01-5eaf-1f03-12d4-c96ab15bedd9}]" +2026-11-29T10:28:29.113Z [INFO] server-013cbf0d-bc9b-0aea-cd3d-90bffe362de4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-82f905cd-5130-5fb5-a320-776feb9a7586 Address:82f905cd-5130-5fb5-a320-776feb9a7586} {Suffrage:Voter ID:server-013cbf0d-bc9b-0aea-cd3d-90bffe362de4 Address:013cbf0d-bc9b-0aea-cd3d-90bffe362de4} {Suffrage:Voter ID:server-0f0f9b01-5eaf-1f03-12d4-c96ab15bedd9 Address:0f0f9b01-5eaf-1f03-12d4-c96ab15bedd9}]" +2026-11-29T10:28:29.113Z [INFO] server-013cbf0d-bc9b-0aea-cd3d-90bffe362de4: entering follower state: follower="Node at 013cbf0d-bc9b-0aea-cd3d-90bffe362de4 [Follower]" leader-address= leader-id= +2026-11-29T10:28:29.113Z [INFO] server-82f905cd-5130-5fb5-a320-776feb9a7586: entering follower state: follower="Node at 82f905cd-5130-5fb5-a320-776feb9a7586 [Follower]" leader-address= leader-id= +2026-11-29T10:28:29.113Z [INFO] server-0f0f9b01-5eaf-1f03-12d4-c96ab15bedd9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-82f905cd-5130-5fb5-a320-776feb9a7586 Address:82f905cd-5130-5fb5-a320-776feb9a7586} {Suffrage:Voter ID:server-013cbf0d-bc9b-0aea-cd3d-90bffe362de4 Address:013cbf0d-bc9b-0aea-cd3d-90bffe362de4} {Suffrage:Voter ID:server-0f0f9b01-5eaf-1f03-12d4-c96ab15bedd9 Address:0f0f9b01-5eaf-1f03-12d4-c96ab15bedd9}]" +2026-11-29T10:28:29.113Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:29.113Z [INFO] server-0f0f9b01-5eaf-1f03-12d4-c96ab15bedd9: entering follower state: follower="Node at 0f0f9b01-5eaf-1f03-12d4-c96ab15bedd9 [Follower]" leader-address= leader-id= +2026-11-29T10:28:29.740Z [WARN] server-82f905cd-5130-5fb5-a320-776feb9a7586: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:29.740Z [INFO] server-82f905cd-5130-5fb5-a320-776feb9a7586: entering candidate state: node="Node at 82f905cd-5130-5fb5-a320-776feb9a7586 [Candidate]" term=2 +2026-11-29T10:28:29.740Z [INFO] server-82f905cd-5130-5fb5-a320-776feb9a7586: election won: term=2 tally=2 +2026-11-29T10:28:29.740Z [INFO] server-82f905cd-5130-5fb5-a320-776feb9a7586: entering leader state: leader="Node at 82f905cd-5130-5fb5-a320-776feb9a7586 [Leader]" +2026-11-29T10:28:29.740Z [INFO] server-82f905cd-5130-5fb5-a320-776feb9a7586: added peer, starting replication: peer=server-013cbf0d-bc9b-0aea-cd3d-90bffe362de4 +2026-11-29T10:28:29.740Z [INFO] server-82f905cd-5130-5fb5-a320-776feb9a7586: added peer, starting replication: peer=server-0f0f9b01-5eaf-1f03-12d4-c96ab15bedd9 +2026-11-29T10:28:29.741Z [INFO] server-82f905cd-5130-5fb5-a320-776feb9a7586: pipelining replication: peer="{Voter server-0f0f9b01-5eaf-1f03-12d4-c96ab15bedd9 0f0f9b01-5eaf-1f03-12d4-c96ab15bedd9}" +2026-11-29T10:28:29.741Z [INFO] server-82f905cd-5130-5fb5-a320-776feb9a7586: pipelining replication: peer="{Voter server-013cbf0d-bc9b-0aea-cd3d-90bffe362de4 013cbf0d-bc9b-0aea-cd3d-90bffe362de4}" +2026-11-29T10:28:30.746Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:29.740973979 +0000 UTC m=+20.476028161 (1 nodes), 627.21456ms from start of poll, 627.65316ms from cluster start. Timeout at 2026-11-29 10:28:30.745978419 +0000 UTC m=+21.481032561, 1.0050044s after stability +2026-11-29T10:28:30.746Z [INFO] creating new snapshot: path=/tmp/raft4290121781/snapshots/2-13-1795948110746.tmp +2026-11-29T10:28:30.746Z [INFO] server-82f905cd-5130-5fb5-a320-776feb9a7586: copied to local snapshot: bytes=71 +2026-11-29T10:28:30.747Z [INFO] server-82f905cd-5130-5fb5-a320-776feb9a7586: snapshot restore progress: id=2-13-1795948110746 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:30.747Z [INFO] server-82f905cd-5130-5fb5-a320-776feb9a7586: restored user snapshot: index=13 +2026-11-29T10:28:30.747Z [ERROR] server-82f905cd-5130-5fb5-a320-776feb9a7586: failed to get log: index=3 error="log not found" +2026-11-29T10:28:30.747Z [INFO] server-82f905cd-5130-5fb5-a320-776feb9a7586: aborting pipeline replication: peer="{Voter server-0f0f9b01-5eaf-1f03-12d4-c96ab15bedd9 0f0f9b01-5eaf-1f03-12d4-c96ab15bedd9}" +2026-11-29T10:28:30.747Z [ERROR] server-82f905cd-5130-5fb5-a320-776feb9a7586: failed to get log: index=3 error="log not found" +2026-11-29T10:28:30.747Z [INFO] server-82f905cd-5130-5fb5-a320-776feb9a7586: aborting pipeline replication: peer="{Voter server-013cbf0d-bc9b-0aea-cd3d-90bffe362de4 013cbf0d-bc9b-0aea-cd3d-90bffe362de4}" +2026-11-29T10:28:30.755Z [ERROR] server-82f905cd-5130-5fb5-a320-776feb9a7586: failed to get log: index=3 error="log not found" +2026-11-29T10:28:30.756Z [INFO] creating new snapshot: path=/tmp/raft828697874/snapshots/2-13-1795948110756.tmp +2026-11-29T10:28:30.756Z [INFO] server-013cbf0d-bc9b-0aea-cd3d-90bffe362de4: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:30.756Z [ERROR] server-82f905cd-5130-5fb5-a320-776feb9a7586: failed to get log: index=3 error="log not found" +2026-11-29T10:28:30.756Z [INFO] server-013cbf0d-bc9b-0aea-cd3d-90bffe362de4: copied to local snapshot: bytes=71 +2026-11-29T10:28:30.756Z [INFO] server-013cbf0d-bc9b-0aea-cd3d-90bffe362de4: snapshot restore progress: id=2-13-1795948110756 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:30.756Z [INFO] server-013cbf0d-bc9b-0aea-cd3d-90bffe362de4: Installed remote snapshot +2026-11-29T10:28:30.756Z [INFO] server-82f905cd-5130-5fb5-a320-776feb9a7586: pipelining replication: peer="{Voter server-013cbf0d-bc9b-0aea-cd3d-90bffe362de4 013cbf0d-bc9b-0aea-cd3d-90bffe362de4}" +2026-11-29T10:28:30.756Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:28:30.756Z [INFO] creating new snapshot: path=/tmp/raft2273024949/snapshots/2-13-1795948110756.tmp +2026-11-29T10:28:30.756Z [INFO] server-0f0f9b01-5eaf-1f03-12d4-c96ab15bedd9: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:30.757Z [INFO] server-0f0f9b01-5eaf-1f03-12d4-c96ab15bedd9: copied to local snapshot: bytes=71 +2026-11-29T10:28:30.757Z [INFO] server-0f0f9b01-5eaf-1f03-12d4-c96ab15bedd9: snapshot restore progress: id=2-13-1795948110756 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:30.757Z [INFO] server-0f0f9b01-5eaf-1f03-12d4-c96ab15bedd9: Installed remote snapshot +2026-11-29T10:28:30.757Z [INFO] server-82f905cd-5130-5fb5-a320-776feb9a7586: pipelining replication: peer="{Voter server-0f0f9b01-5eaf-1f03-12d4-c96ab15bedd9 0f0f9b01-5eaf-1f03-12d4-c96ab15bedd9}" +2026-11-29T10:28:31.762Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:28:30.756658299 +0000 UTC m=+21.491712401 (2 nodes), 640ns from start of poll, 1.6433374s from cluster start. Timeout at 2026-11-29 10:28:31.761662779 +0000 UTC m=+22.496716841, 1.00500444s after stability +2026-11-29T10:28:31.774Z [INFO] server-5efc46ad-7f5d-e7ab-6348-4f38ffac9c30: aborting pipeline replication: peer="{Voter server-af9447c1-ff88-5424-0120-d7732a612efc af9447c1-ff88-5424-0120-d7732a612efc}" +2026-11-29T10:28:31.774Z [INFO] server-5efc46ad-7f5d-e7ab-6348-4f38ffac9c30: aborting pipeline replication: peer="{Voter server-04acb096-463d-ef3d-b3a5-e247a7c42059 04acb096-463d-ef3d-b3a5-e247a7c42059}" === RUN TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_true -2025-10-27T04:02:28.571Z [INFO] server-56fcbe5f-4478-1d60-8c01-ee17faa3100b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-56fcbe5f-4478-1d60-8c01-ee17faa3100b Address:56fcbe5f-4478-1d60-8c01-ee17faa3100b} {Suffrage:Voter ID:server-e066bbe1-0612-6c03-34ea-b5c4998edca6 Address:e066bbe1-0612-6c03-34ea-b5c4998edca6} {Suffrage:Voter ID:server-59893ade-f75e-923b-b9b0-76788f84a5d3 Address:59893ade-f75e-923b-b9b0-76788f84a5d3}]" -2025-10-27T04:02:28.571Z [INFO] server-e066bbe1-0612-6c03-34ea-b5c4998edca6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-56fcbe5f-4478-1d60-8c01-ee17faa3100b Address:56fcbe5f-4478-1d60-8c01-ee17faa3100b} {Suffrage:Voter ID:server-e066bbe1-0612-6c03-34ea-b5c4998edca6 Address:e066bbe1-0612-6c03-34ea-b5c4998edca6} {Suffrage:Voter ID:server-59893ade-f75e-923b-b9b0-76788f84a5d3 Address:59893ade-f75e-923b-b9b0-76788f84a5d3}]" -2025-10-27T04:02:28.571Z [INFO] server-56fcbe5f-4478-1d60-8c01-ee17faa3100b: entering follower state: follower="Node at 56fcbe5f-4478-1d60-8c01-ee17faa3100b [Follower]" leader-address= leader-id= -2025-10-27T04:02:28.571Z [INFO] server-59893ade-f75e-923b-b9b0-76788f84a5d3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-56fcbe5f-4478-1d60-8c01-ee17faa3100b Address:56fcbe5f-4478-1d60-8c01-ee17faa3100b} {Suffrage:Voter ID:server-e066bbe1-0612-6c03-34ea-b5c4998edca6 Address:e066bbe1-0612-6c03-34ea-b5c4998edca6} {Suffrage:Voter ID:server-59893ade-f75e-923b-b9b0-76788f84a5d3 Address:59893ade-f75e-923b-b9b0-76788f84a5d3}]" -2025-10-27T04:02:28.571Z [INFO] server-e066bbe1-0612-6c03-34ea-b5c4998edca6: entering follower state: follower="Node at e066bbe1-0612-6c03-34ea-b5c4998edca6 [Follower]" leader-address= leader-id= -2025-10-27T04:02:28.571Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:28.571Z [INFO] server-59893ade-f75e-923b-b9b0-76788f84a5d3: entering follower state: follower="Node at 59893ade-f75e-923b-b9b0-76788f84a5d3 [Follower]" leader-address= leader-id= -2025-10-27T04:02:29.154Z [WARN] server-56fcbe5f-4478-1d60-8c01-ee17faa3100b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:29.154Z [INFO] server-56fcbe5f-4478-1d60-8c01-ee17faa3100b: entering candidate state: node="Node at 56fcbe5f-4478-1d60-8c01-ee17faa3100b [Candidate]" term=2 -2025-10-27T04:02:29.154Z [INFO] server-56fcbe5f-4478-1d60-8c01-ee17faa3100b: election won: term=2 tally=2 -2025-10-27T04:02:29.154Z [INFO] server-56fcbe5f-4478-1d60-8c01-ee17faa3100b: entering leader state: leader="Node at 56fcbe5f-4478-1d60-8c01-ee17faa3100b [Leader]" -2025-10-27T04:02:29.154Z [INFO] server-56fcbe5f-4478-1d60-8c01-ee17faa3100b: added peer, starting replication: peer=server-e066bbe1-0612-6c03-34ea-b5c4998edca6 -2025-10-27T04:02:29.154Z [INFO] server-56fcbe5f-4478-1d60-8c01-ee17faa3100b: added peer, starting replication: peer=server-59893ade-f75e-923b-b9b0-76788f84a5d3 -2025-10-27T04:02:29.154Z [INFO] server-56fcbe5f-4478-1d60-8c01-ee17faa3100b: pipelining replication: peer="{Voter server-e066bbe1-0612-6c03-34ea-b5c4998edca6 e066bbe1-0612-6c03-34ea-b5c4998edca6}" -2025-10-27T04:02:29.154Z [INFO] server-56fcbe5f-4478-1d60-8c01-ee17faa3100b: pipelining replication: peer="{Voter server-59893ade-f75e-923b-b9b0-76788f84a5d3 59893ade-f75e-923b-b9b0-76788f84a5d3}" -2025-10-27T04:02:30.160Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:29.15439078 +0000 UTC m=+24.149263218 (1 nodes), 582.908463ms from start of poll, 583.287226ms from cluster start. Timeout at 2025-10-27 04:02:30.15939186 +0000 UTC m=+25.154264378, 1.00500116s after stability -2025-10-27T04:02:30.161Z [INFO] server-56fcbe5f-4478-1d60-8c01-ee17faa3100b: starting snapshot up to: index=12 -2025-10-27T04:02:30.161Z [INFO] creating new snapshot: path=/tmp/raft2505484428/snapshots/2-12-1761537750161.tmp -2025-10-27T04:02:30.161Z [INFO] server-56fcbe5f-4478-1d60-8c01-ee17faa3100b: snapshot complete up to: index=12 -2025-10-27T04:02:30.162Z [INFO] server-4905624d-6f18-586f-88a6-645d91bf1ec3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4905624d-6f18-586f-88a6-645d91bf1ec3 Address:4905624d-6f18-586f-88a6-645d91bf1ec3} {Suffrage:Voter ID:server-19a0b61d-946c-8d6f-0974-322e03717445 Address:19a0b61d-946c-8d6f-0974-322e03717445} {Suffrage:Voter ID:server-6e369eb0-1d6d-3ef3-7307-fe2794695fc6 Address:6e369eb0-1d6d-3ef3-7307-fe2794695fc6}]" -2025-10-27T04:02:30.162Z [INFO] server-19a0b61d-946c-8d6f-0974-322e03717445: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4905624d-6f18-586f-88a6-645d91bf1ec3 Address:4905624d-6f18-586f-88a6-645d91bf1ec3} {Suffrage:Voter ID:server-19a0b61d-946c-8d6f-0974-322e03717445 Address:19a0b61d-946c-8d6f-0974-322e03717445} {Suffrage:Voter ID:server-6e369eb0-1d6d-3ef3-7307-fe2794695fc6 Address:6e369eb0-1d6d-3ef3-7307-fe2794695fc6}]" -2025-10-27T04:02:30.162Z [INFO] server-4905624d-6f18-586f-88a6-645d91bf1ec3: entering follower state: follower="Node at 4905624d-6f18-586f-88a6-645d91bf1ec3 [Follower]" leader-address= leader-id= -2025-10-27T04:02:30.162Z [INFO] server-6e369eb0-1d6d-3ef3-7307-fe2794695fc6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4905624d-6f18-586f-88a6-645d91bf1ec3 Address:4905624d-6f18-586f-88a6-645d91bf1ec3} {Suffrage:Voter ID:server-19a0b61d-946c-8d6f-0974-322e03717445 Address:19a0b61d-946c-8d6f-0974-322e03717445} {Suffrage:Voter ID:server-6e369eb0-1d6d-3ef3-7307-fe2794695fc6 Address:6e369eb0-1d6d-3ef3-7307-fe2794695fc6}]" -2025-10-27T04:02:30.162Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:30.162Z [INFO] server-19a0b61d-946c-8d6f-0974-322e03717445: entering follower state: follower="Node at 19a0b61d-946c-8d6f-0974-322e03717445 [Follower]" leader-address= leader-id= -2025-10-27T04:02:30.162Z [INFO] server-6e369eb0-1d6d-3ef3-7307-fe2794695fc6: entering follower state: follower="Node at 6e369eb0-1d6d-3ef3-7307-fe2794695fc6 [Follower]" leader-address= leader-id= -2025-10-27T04:02:30.715Z [WARN] server-4905624d-6f18-586f-88a6-645d91bf1ec3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:30.715Z [INFO] server-4905624d-6f18-586f-88a6-645d91bf1ec3: entering candidate state: node="Node at 4905624d-6f18-586f-88a6-645d91bf1ec3 [Candidate]" term=2 -2025-10-27T04:02:30.715Z [INFO] server-4905624d-6f18-586f-88a6-645d91bf1ec3: election won: term=2 tally=2 -2025-10-27T04:02:30.715Z [INFO] server-4905624d-6f18-586f-88a6-645d91bf1ec3: entering leader state: leader="Node at 4905624d-6f18-586f-88a6-645d91bf1ec3 [Leader]" -2025-10-27T04:02:30.715Z [INFO] server-4905624d-6f18-586f-88a6-645d91bf1ec3: added peer, starting replication: peer=server-19a0b61d-946c-8d6f-0974-322e03717445 -2025-10-27T04:02:30.715Z [INFO] server-4905624d-6f18-586f-88a6-645d91bf1ec3: added peer, starting replication: peer=server-6e369eb0-1d6d-3ef3-7307-fe2794695fc6 -2025-10-27T04:02:30.715Z [INFO] server-4905624d-6f18-586f-88a6-645d91bf1ec3: pipelining replication: peer="{Voter server-19a0b61d-946c-8d6f-0974-322e03717445 19a0b61d-946c-8d6f-0974-322e03717445}" -2025-10-27T04:02:30.715Z [INFO] server-4905624d-6f18-586f-88a6-645d91bf1ec3: pipelining replication: peer="{Voter server-6e369eb0-1d6d-3ef3-7307-fe2794695fc6 6e369eb0-1d6d-3ef3-7307-fe2794695fc6}" -2025-10-27T04:02:31.721Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:30.715678691 +0000 UTC m=+25.710551169 (1 nodes), 552.958859ms from start of poll, 553.240941ms from cluster start. Timeout at 2025-10-27 04:02:31.720682611 +0000 UTC m=+26.715555049, 1.00500388s after stability -2025-10-27T04:02:31.721Z [INFO] creating new snapshot: path=/tmp/raft1396713612/snapshots/2-13-1761537751721.tmp -2025-10-27T04:02:31.721Z [INFO] server-4905624d-6f18-586f-88a6-645d91bf1ec3: copied to local snapshot: bytes=71 -2025-10-27T04:02:31.722Z [INFO] server-4905624d-6f18-586f-88a6-645d91bf1ec3: snapshot restore progress: id=2-13-1761537751721 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:31.722Z [INFO] server-4905624d-6f18-586f-88a6-645d91bf1ec3: removing all old logs from log store -2025-10-27T04:02:31.722Z [INFO] server-4905624d-6f18-586f-88a6-645d91bf1ec3: compacting logs: from=1 to=2 -2025-10-27T04:02:31.722Z [INFO] server-4905624d-6f18-586f-88a6-645d91bf1ec3: restored user snapshot: index=13 -2025-10-27T04:02:31.722Z [ERROR] server-4905624d-6f18-586f-88a6-645d91bf1ec3: failed to get log: index=2 error="log not found" -2025-10-27T04:02:31.722Z [ERROR] server-4905624d-6f18-586f-88a6-645d91bf1ec3: failed to get log: index=2 error="log not found" -2025-10-27T04:02:31.722Z [INFO] server-4905624d-6f18-586f-88a6-645d91bf1ec3: aborting pipeline replication: peer="{Voter server-19a0b61d-946c-8d6f-0974-322e03717445 19a0b61d-946c-8d6f-0974-322e03717445}" -2025-10-27T04:02:31.722Z [INFO] server-4905624d-6f18-586f-88a6-645d91bf1ec3: aborting pipeline replication: peer="{Voter server-6e369eb0-1d6d-3ef3-7307-fe2794695fc6 6e369eb0-1d6d-3ef3-7307-fe2794695fc6}" -2025-10-27T04:02:31.727Z [ERROR] server-4905624d-6f18-586f-88a6-645d91bf1ec3: failed to get log: index=2 error="log not found" -2025-10-27T04:02:31.728Z [INFO] creating new snapshot: path=/tmp/raft27675725/snapshots/2-13-1761537751728.tmp -2025-10-27T04:02:31.728Z [INFO] server-6e369eb0-1d6d-3ef3-7307-fe2794695fc6: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:31.728Z [INFO] server-6e369eb0-1d6d-3ef3-7307-fe2794695fc6: copied to local snapshot: bytes=71 -2025-10-27T04:02:31.728Z [INFO] server-6e369eb0-1d6d-3ef3-7307-fe2794695fc6: snapshot restore progress: id=2-13-1761537751728 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:31.728Z [INFO] server-6e369eb0-1d6d-3ef3-7307-fe2794695fc6: removing all old logs from log store -2025-10-27T04:02:31.728Z [INFO] server-6e369eb0-1d6d-3ef3-7307-fe2794695fc6: compacting logs: from=1 to=2 -2025-10-27T04:02:31.728Z [INFO] server-6e369eb0-1d6d-3ef3-7307-fe2794695fc6: Installed remote snapshot -2025-10-27T04:02:31.728Z [INFO] server-4905624d-6f18-586f-88a6-645d91bf1ec3: pipelining replication: peer="{Voter server-6e369eb0-1d6d-3ef3-7307-fe2794695fc6 6e369eb0-1d6d-3ef3-7307-fe2794695fc6}" -2025-10-27T04:02:31.728Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:02:31.730Z [ERROR] server-4905624d-6f18-586f-88a6-645d91bf1ec3: failed to get log: index=2 error="log not found" -2025-10-27T04:02:31.730Z [INFO] creating new snapshot: path=/tmp/raft2098273628/snapshots/2-13-1761537751730.tmp -2025-10-27T04:02:31.730Z [INFO] server-19a0b61d-946c-8d6f-0974-322e03717445: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:31.731Z [INFO] server-19a0b61d-946c-8d6f-0974-322e03717445: copied to local snapshot: bytes=71 -2025-10-27T04:02:31.731Z [INFO] server-19a0b61d-946c-8d6f-0974-322e03717445: snapshot restore progress: id=2-13-1761537751730 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:31.731Z [INFO] server-19a0b61d-946c-8d6f-0974-322e03717445: removing all old logs from log store -2025-10-27T04:02:31.731Z [INFO] server-19a0b61d-946c-8d6f-0974-322e03717445: compacting logs: from=1 to=2 -2025-10-27T04:02:31.731Z [INFO] server-19a0b61d-946c-8d6f-0974-322e03717445: Installed remote snapshot -2025-10-27T04:02:31.731Z [INFO] server-4905624d-6f18-586f-88a6-645d91bf1ec3: pipelining replication: peer="{Voter server-19a0b61d-946c-8d6f-0974-322e03717445 19a0b61d-946c-8d6f-0974-322e03717445}" -2025-10-27T04:02:32.734Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:02:31.728984496 +0000 UTC m=+26.723856974 (2 nodes), 760ns from start of poll, 1.566546746s from cluster start. Timeout at 2025-10-27 04:02:32.733998136 +0000 UTC m=+27.728870534, 1.00501356s after stability -2025-10-27T04:02:32.751Z [INFO] server-56fcbe5f-4478-1d60-8c01-ee17faa3100b: aborting pipeline replication: peer="{Voter server-59893ade-f75e-923b-b9b0-76788f84a5d3 59893ade-f75e-923b-b9b0-76788f84a5d3}" -2025-10-27T04:02:32.751Z [INFO] server-56fcbe5f-4478-1d60-8c01-ee17faa3100b: aborting pipeline replication: peer="{Voter server-e066bbe1-0612-6c03-34ea-b5c4998edca6 e066bbe1-0612-6c03-34ea-b5c4998edca6}" +2026-11-29T10:28:31.775Z [INFO] server-9db5f652-60a7-5d87-ef9c-3b2bd0215cf0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9db5f652-60a7-5d87-ef9c-3b2bd0215cf0 Address:9db5f652-60a7-5d87-ef9c-3b2bd0215cf0} {Suffrage:Voter ID:server-007ab776-3a01-2e01-54db-401013d081b6 Address:007ab776-3a01-2e01-54db-401013d081b6} {Suffrage:Voter ID:server-b099c69d-5e52-2fc7-ef88-6d445e4a8545 Address:b099c69d-5e52-2fc7-ef88-6d445e4a8545}]" +2026-11-29T10:28:31.775Z [INFO] server-007ab776-3a01-2e01-54db-401013d081b6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9db5f652-60a7-5d87-ef9c-3b2bd0215cf0 Address:9db5f652-60a7-5d87-ef9c-3b2bd0215cf0} {Suffrage:Voter ID:server-007ab776-3a01-2e01-54db-401013d081b6 Address:007ab776-3a01-2e01-54db-401013d081b6} {Suffrage:Voter ID:server-b099c69d-5e52-2fc7-ef88-6d445e4a8545 Address:b099c69d-5e52-2fc7-ef88-6d445e4a8545}]" +2026-11-29T10:28:31.775Z [INFO] server-b099c69d-5e52-2fc7-ef88-6d445e4a8545: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9db5f652-60a7-5d87-ef9c-3b2bd0215cf0 Address:9db5f652-60a7-5d87-ef9c-3b2bd0215cf0} {Suffrage:Voter ID:server-007ab776-3a01-2e01-54db-401013d081b6 Address:007ab776-3a01-2e01-54db-401013d081b6} {Suffrage:Voter ID:server-b099c69d-5e52-2fc7-ef88-6d445e4a8545 Address:b099c69d-5e52-2fc7-ef88-6d445e4a8545}]" +2026-11-29T10:28:31.775Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:31.775Z [INFO] server-9db5f652-60a7-5d87-ef9c-3b2bd0215cf0: entering follower state: follower="Node at 9db5f652-60a7-5d87-ef9c-3b2bd0215cf0 [Follower]" leader-address= leader-id= +2026-11-29T10:28:31.775Z [INFO] server-007ab776-3a01-2e01-54db-401013d081b6: entering follower state: follower="Node at 007ab776-3a01-2e01-54db-401013d081b6 [Follower]" leader-address= leader-id= +2026-11-29T10:28:31.775Z [INFO] server-b099c69d-5e52-2fc7-ef88-6d445e4a8545: entering follower state: follower="Node at b099c69d-5e52-2fc7-ef88-6d445e4a8545 [Follower]" leader-address= leader-id= +2026-11-29T10:28:32.285Z [WARN] server-007ab776-3a01-2e01-54db-401013d081b6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:32.285Z [INFO] server-007ab776-3a01-2e01-54db-401013d081b6: entering candidate state: node="Node at 007ab776-3a01-2e01-54db-401013d081b6 [Candidate]" term=2 +2026-11-29T10:28:32.285Z [INFO] server-007ab776-3a01-2e01-54db-401013d081b6: election won: term=2 tally=2 +2026-11-29T10:28:32.285Z [INFO] server-007ab776-3a01-2e01-54db-401013d081b6: entering leader state: leader="Node at 007ab776-3a01-2e01-54db-401013d081b6 [Leader]" +2026-11-29T10:28:32.285Z [INFO] server-007ab776-3a01-2e01-54db-401013d081b6: added peer, starting replication: peer=server-9db5f652-60a7-5d87-ef9c-3b2bd0215cf0 +2026-11-29T10:28:32.285Z [INFO] server-007ab776-3a01-2e01-54db-401013d081b6: added peer, starting replication: peer=server-b099c69d-5e52-2fc7-ef88-6d445e4a8545 +2026-11-29T10:28:32.285Z [INFO] server-007ab776-3a01-2e01-54db-401013d081b6: pipelining replication: peer="{Voter server-9db5f652-60a7-5d87-ef9c-3b2bd0215cf0 9db5f652-60a7-5d87-ef9c-3b2bd0215cf0}" +2026-11-29T10:28:32.285Z [INFO] server-007ab776-3a01-2e01-54db-401013d081b6: pipelining replication: peer="{Voter server-b099c69d-5e52-2fc7-ef88-6d445e4a8545 b099c69d-5e52-2fc7-ef88-6d445e4a8545}" +2026-11-29T10:28:33.291Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:32.285571619 +0000 UTC m=+23.020625721 (1 nodes), 510.35136ms from start of poll, 510.54624ms from cluster start. Timeout at 2026-11-29 10:28:33.290582259 +0000 UTC m=+24.025636401, 1.00501068s after stability +2026-11-29T10:28:33.291Z [INFO] server-007ab776-3a01-2e01-54db-401013d081b6: starting snapshot up to: index=12 +2026-11-29T10:28:33.292Z [INFO] creating new snapshot: path=/tmp/raft2709353295/snapshots/2-12-1795948113291.tmp +2026-11-29T10:28:33.292Z [INFO] server-007ab776-3a01-2e01-54db-401013d081b6: snapshot complete up to: index=12 +2026-11-29T10:28:33.293Z [INFO] server-f8652c62-b656-9736-97c2-a530a333b602: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f8652c62-b656-9736-97c2-a530a333b602 Address:f8652c62-b656-9736-97c2-a530a333b602} {Suffrage:Voter ID:server-843d96a2-eed3-a811-5dc9-59f5da4be81c Address:843d96a2-eed3-a811-5dc9-59f5da4be81c} {Suffrage:Voter ID:server-e4f5409a-c97a-bba0-ebf4-8aed237aa651 Address:e4f5409a-c97a-bba0-ebf4-8aed237aa651}]" +2026-11-29T10:28:33.293Z [INFO] server-843d96a2-eed3-a811-5dc9-59f5da4be81c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f8652c62-b656-9736-97c2-a530a333b602 Address:f8652c62-b656-9736-97c2-a530a333b602} {Suffrage:Voter ID:server-843d96a2-eed3-a811-5dc9-59f5da4be81c Address:843d96a2-eed3-a811-5dc9-59f5da4be81c} {Suffrage:Voter ID:server-e4f5409a-c97a-bba0-ebf4-8aed237aa651 Address:e4f5409a-c97a-bba0-ebf4-8aed237aa651}]" +2026-11-29T10:28:33.293Z [INFO] server-f8652c62-b656-9736-97c2-a530a333b602: entering follower state: follower="Node at f8652c62-b656-9736-97c2-a530a333b602 [Follower]" leader-address= leader-id= +2026-11-29T10:28:33.293Z [INFO] server-843d96a2-eed3-a811-5dc9-59f5da4be81c: entering follower state: follower="Node at 843d96a2-eed3-a811-5dc9-59f5da4be81c [Follower]" leader-address= leader-id= +2026-11-29T10:28:33.293Z [INFO] server-e4f5409a-c97a-bba0-ebf4-8aed237aa651: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f8652c62-b656-9736-97c2-a530a333b602 Address:f8652c62-b656-9736-97c2-a530a333b602} {Suffrage:Voter ID:server-843d96a2-eed3-a811-5dc9-59f5da4be81c Address:843d96a2-eed3-a811-5dc9-59f5da4be81c} {Suffrage:Voter ID:server-e4f5409a-c97a-bba0-ebf4-8aed237aa651 Address:e4f5409a-c97a-bba0-ebf4-8aed237aa651}]" +2026-11-29T10:28:33.293Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:33.293Z [INFO] server-e4f5409a-c97a-bba0-ebf4-8aed237aa651: entering follower state: follower="Node at e4f5409a-c97a-bba0-ebf4-8aed237aa651 [Follower]" leader-address= leader-id= +2026-11-29T10:28:34.038Z [WARN] server-843d96a2-eed3-a811-5dc9-59f5da4be81c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:34.038Z [INFO] server-843d96a2-eed3-a811-5dc9-59f5da4be81c: entering candidate state: node="Node at 843d96a2-eed3-a811-5dc9-59f5da4be81c [Candidate]" term=2 +2026-11-29T10:28:34.038Z [INFO] server-843d96a2-eed3-a811-5dc9-59f5da4be81c: election won: term=2 tally=2 +2026-11-29T10:28:34.038Z [INFO] server-843d96a2-eed3-a811-5dc9-59f5da4be81c: entering leader state: leader="Node at 843d96a2-eed3-a811-5dc9-59f5da4be81c [Leader]" +2026-11-29T10:28:34.038Z [INFO] server-843d96a2-eed3-a811-5dc9-59f5da4be81c: added peer, starting replication: peer=server-f8652c62-b656-9736-97c2-a530a333b602 +2026-11-29T10:28:34.038Z [INFO] server-843d96a2-eed3-a811-5dc9-59f5da4be81c: added peer, starting replication: peer=server-e4f5409a-c97a-bba0-ebf4-8aed237aa651 +2026-11-29T10:28:34.038Z [INFO] server-843d96a2-eed3-a811-5dc9-59f5da4be81c: pipelining replication: peer="{Voter server-f8652c62-b656-9736-97c2-a530a333b602 f8652c62-b656-9736-97c2-a530a333b602}" +2026-11-29T10:28:34.038Z [INFO] server-843d96a2-eed3-a811-5dc9-59f5da4be81c: pipelining replication: peer="{Voter server-e4f5409a-c97a-bba0-ebf4-8aed237aa651 e4f5409a-c97a-bba0-ebf4-8aed237aa651}" +2026-11-29T10:28:35.044Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:34.038841419 +0000 UTC m=+24.773895561 (1 nodes), 744.947ms from start of poll, 745.42548ms from cluster start. Timeout at 2026-11-29 10:28:35.043848819 +0000 UTC m=+25.778903001, 1.00500744s after stability +2026-11-29T10:28:35.044Z [INFO] creating new snapshot: path=/tmp/raft1196022507/snapshots/2-13-1795948115044.tmp +2026-11-29T10:28:35.044Z [INFO] server-843d96a2-eed3-a811-5dc9-59f5da4be81c: copied to local snapshot: bytes=71 +2026-11-29T10:28:35.045Z [INFO] server-843d96a2-eed3-a811-5dc9-59f5da4be81c: snapshot restore progress: id=2-13-1795948115044 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:35.045Z [INFO] server-843d96a2-eed3-a811-5dc9-59f5da4be81c: removing all old logs from log store +2026-11-29T10:28:35.045Z [INFO] server-843d96a2-eed3-a811-5dc9-59f5da4be81c: compacting logs: from=1 to=2 +2026-11-29T10:28:35.045Z [INFO] server-843d96a2-eed3-a811-5dc9-59f5da4be81c: restored user snapshot: index=13 +2026-11-29T10:28:35.045Z [ERROR] server-843d96a2-eed3-a811-5dc9-59f5da4be81c: failed to get log: index=2 error="log not found" +2026-11-29T10:28:35.045Z [ERROR] server-843d96a2-eed3-a811-5dc9-59f5da4be81c: failed to get log: index=2 error="log not found" +2026-11-29T10:28:35.045Z [INFO] server-843d96a2-eed3-a811-5dc9-59f5da4be81c: aborting pipeline replication: peer="{Voter server-f8652c62-b656-9736-97c2-a530a333b602 f8652c62-b656-9736-97c2-a530a333b602}" +2026-11-29T10:28:35.045Z [INFO] server-843d96a2-eed3-a811-5dc9-59f5da4be81c: aborting pipeline replication: peer="{Voter server-e4f5409a-c97a-bba0-ebf4-8aed237aa651 e4f5409a-c97a-bba0-ebf4-8aed237aa651}" +2026-11-29T10:28:35.054Z [ERROR] server-843d96a2-eed3-a811-5dc9-59f5da4be81c: failed to get log: index=2 error="log not found" +2026-11-29T10:28:35.054Z [INFO] creating new snapshot: path=/tmp/raft3490131203/snapshots/2-13-1795948115054.tmp +2026-11-29T10:28:35.055Z [INFO] server-e4f5409a-c97a-bba0-ebf4-8aed237aa651: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:35.055Z [INFO] server-e4f5409a-c97a-bba0-ebf4-8aed237aa651: copied to local snapshot: bytes=71 +2026-11-29T10:28:35.055Z [ERROR] server-843d96a2-eed3-a811-5dc9-59f5da4be81c: failed to get log: index=2 error="log not found" +2026-11-29T10:28:35.055Z [INFO] server-e4f5409a-c97a-bba0-ebf4-8aed237aa651: snapshot restore progress: id=2-13-1795948115054 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:35.055Z [INFO] server-e4f5409a-c97a-bba0-ebf4-8aed237aa651: removing all old logs from log store +2026-11-29T10:28:35.055Z [INFO] server-e4f5409a-c97a-bba0-ebf4-8aed237aa651: compacting logs: from=1 to=2 +2026-11-29T10:28:35.055Z [INFO] server-e4f5409a-c97a-bba0-ebf4-8aed237aa651: Installed remote snapshot +2026-11-29T10:28:35.055Z [INFO] creating new snapshot: path=/tmp/raft737592225/snapshots/2-13-1795948115055.tmp +2026-11-29T10:28:35.055Z [INFO] server-843d96a2-eed3-a811-5dc9-59f5da4be81c: pipelining replication: peer="{Voter server-e4f5409a-c97a-bba0-ebf4-8aed237aa651 e4f5409a-c97a-bba0-ebf4-8aed237aa651}" +2026-11-29T10:28:35.055Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:28:35.056Z [INFO] server-f8652c62-b656-9736-97c2-a530a333b602: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:35.056Z [INFO] server-f8652c62-b656-9736-97c2-a530a333b602: copied to local snapshot: bytes=71 +2026-11-29T10:28:35.056Z [INFO] server-f8652c62-b656-9736-97c2-a530a333b602: snapshot restore progress: id=2-13-1795948115055 last-index=13 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:35.056Z [INFO] server-f8652c62-b656-9736-97c2-a530a333b602: removing all old logs from log store +2026-11-29T10:28:35.056Z [INFO] server-f8652c62-b656-9736-97c2-a530a333b602: compacting logs: from=1 to=2 +2026-11-29T10:28:35.056Z [INFO] server-f8652c62-b656-9736-97c2-a530a333b602: Installed remote snapshot +2026-11-29T10:28:35.056Z [INFO] server-843d96a2-eed3-a811-5dc9-59f5da4be81c: pipelining replication: peer="{Voter server-f8652c62-b656-9736-97c2-a530a333b602 f8652c62-b656-9736-97c2-a530a333b602}" +2026-11-29T10:28:36.061Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:28:35.055930219 +0000 UTC m=+25.790984361 (2 nodes), 800ns from start of poll, 1.76251428s from cluster start. Timeout at 2026-11-29 10:28:36.060940419 +0000 UTC m=+26.795994561, 1.0050102s after stability +2026-11-29T10:28:36.072Z [INFO] server-007ab776-3a01-2e01-54db-401013d081b6: aborting pipeline replication: peer="{Voter server-b099c69d-5e52-2fc7-ef88-6d445e4a8545 b099c69d-5e52-2fc7-ef88-6d445e4a8545}" +2026-11-29T10:28:36.072Z [INFO] server-007ab776-3a01-2e01-54db-401013d081b6: aborting pipeline replication: peer="{Voter server-9db5f652-60a7-5d87-ef9c-3b2bd0215cf0 9db5f652-60a7-5d87-ef9c-3b2bd0215cf0}" === RUN TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_false -2025-10-27T04:02:32.751Z [INFO] server-8712b5ca-dcf2-7936-4f44-0c0e22b7ef74: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8712b5ca-dcf2-7936-4f44-0c0e22b7ef74 Address:8712b5ca-dcf2-7936-4f44-0c0e22b7ef74} {Suffrage:Voter ID:server-062ad266-6848-402b-2808-8a76fddb295a Address:062ad266-6848-402b-2808-8a76fddb295a} {Suffrage:Voter ID:server-afca0ee6-39e2-b2ac-c3b1-208bf8269cfa Address:afca0ee6-39e2-b2ac-c3b1-208bf8269cfa}]" -2025-10-27T04:02:32.752Z [INFO] server-8712b5ca-dcf2-7936-4f44-0c0e22b7ef74: entering follower state: follower="Node at 8712b5ca-dcf2-7936-4f44-0c0e22b7ef74 [Follower]" leader-address= leader-id= -2025-10-27T04:02:32.752Z [INFO] server-062ad266-6848-402b-2808-8a76fddb295a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8712b5ca-dcf2-7936-4f44-0c0e22b7ef74 Address:8712b5ca-dcf2-7936-4f44-0c0e22b7ef74} {Suffrage:Voter ID:server-062ad266-6848-402b-2808-8a76fddb295a Address:062ad266-6848-402b-2808-8a76fddb295a} {Suffrage:Voter ID:server-afca0ee6-39e2-b2ac-c3b1-208bf8269cfa Address:afca0ee6-39e2-b2ac-c3b1-208bf8269cfa}]" -2025-10-27T04:02:32.752Z [INFO] server-062ad266-6848-402b-2808-8a76fddb295a: entering follower state: follower="Node at 062ad266-6848-402b-2808-8a76fddb295a [Follower]" leader-address= leader-id= -2025-10-27T04:02:32.752Z [INFO] server-afca0ee6-39e2-b2ac-c3b1-208bf8269cfa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8712b5ca-dcf2-7936-4f44-0c0e22b7ef74 Address:8712b5ca-dcf2-7936-4f44-0c0e22b7ef74} {Suffrage:Voter ID:server-062ad266-6848-402b-2808-8a76fddb295a Address:062ad266-6848-402b-2808-8a76fddb295a} {Suffrage:Voter ID:server-afca0ee6-39e2-b2ac-c3b1-208bf8269cfa Address:afca0ee6-39e2-b2ac-c3b1-208bf8269cfa}]" -2025-10-27T04:02:32.752Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:32.752Z [INFO] server-afca0ee6-39e2-b2ac-c3b1-208bf8269cfa: entering follower state: follower="Node at afca0ee6-39e2-b2ac-c3b1-208bf8269cfa [Follower]" leader-address= leader-id= -2025-10-27T04:02:33.378Z [WARN] server-afca0ee6-39e2-b2ac-c3b1-208bf8269cfa: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:33.378Z [INFO] server-afca0ee6-39e2-b2ac-c3b1-208bf8269cfa: entering candidate state: node="Node at afca0ee6-39e2-b2ac-c3b1-208bf8269cfa [Candidate]" term=2 -2025-10-27T04:02:33.378Z [INFO] server-afca0ee6-39e2-b2ac-c3b1-208bf8269cfa: election won: term=2 tally=2 -2025-10-27T04:02:33.378Z [INFO] server-afca0ee6-39e2-b2ac-c3b1-208bf8269cfa: entering leader state: leader="Node at afca0ee6-39e2-b2ac-c3b1-208bf8269cfa [Leader]" -2025-10-27T04:02:33.378Z [INFO] server-afca0ee6-39e2-b2ac-c3b1-208bf8269cfa: added peer, starting replication: peer=server-8712b5ca-dcf2-7936-4f44-0c0e22b7ef74 -2025-10-27T04:02:33.378Z [INFO] server-afca0ee6-39e2-b2ac-c3b1-208bf8269cfa: added peer, starting replication: peer=server-062ad266-6848-402b-2808-8a76fddb295a -2025-10-27T04:02:33.378Z [INFO] server-afca0ee6-39e2-b2ac-c3b1-208bf8269cfa: pipelining replication: peer="{Voter server-8712b5ca-dcf2-7936-4f44-0c0e22b7ef74 8712b5ca-dcf2-7936-4f44-0c0e22b7ef74}" -2025-10-27T04:02:33.379Z [INFO] server-afca0ee6-39e2-b2ac-c3b1-208bf8269cfa: pipelining replication: peer="{Voter server-062ad266-6848-402b-2808-8a76fddb295a 062ad266-6848-402b-2808-8a76fddb295a}" -2025-10-27T04:02:34.384Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:33.378865012 +0000 UTC m=+28.373737450 (1 nodes), 626.590362ms from start of poll, 627.068206ms from cluster start. Timeout at 2025-10-27 04:02:34.383867772 +0000 UTC m=+29.378740210, 1.00500276s after stability -2025-10-27T04:02:34.386Z [INFO] server-afca0ee6-39e2-b2ac-c3b1-208bf8269cfa: starting snapshot up to: index=12 -2025-10-27T04:02:34.386Z [INFO] creating new snapshot: path=/tmp/raft644206994/snapshots/2-12-1761537754386.tmp -2025-10-27T04:02:34.387Z [INFO] server-afca0ee6-39e2-b2ac-c3b1-208bf8269cfa: snapshot complete up to: index=12 -2025-10-27T04:02:34.387Z [INFO] creating new snapshot: path=/tmp/raft644206994/snapshots/2-23-1761537754387.tmp -2025-10-27T04:02:34.387Z [INFO] server-afca0ee6-39e2-b2ac-c3b1-208bf8269cfa: copied to local snapshot: bytes=71 -2025-10-27T04:02:34.387Z [INFO] server-afca0ee6-39e2-b2ac-c3b1-208bf8269cfa: snapshot restore progress: id=2-23-1761537754387 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:34.387Z [INFO] server-afca0ee6-39e2-b2ac-c3b1-208bf8269cfa: restored user snapshot: index=23 -2025-10-27T04:02:34.387Z [ERROR] server-afca0ee6-39e2-b2ac-c3b1-208bf8269cfa: failed to get log: index=23 error="log not found" -2025-10-27T04:02:34.387Z [INFO] server-afca0ee6-39e2-b2ac-c3b1-208bf8269cfa: aborting pipeline replication: peer="{Voter server-062ad266-6848-402b-2808-8a76fddb295a 062ad266-6848-402b-2808-8a76fddb295a}" -2025-10-27T04:02:34.388Z [ERROR] server-afca0ee6-39e2-b2ac-c3b1-208bf8269cfa: failed to get log: index=23 error="log not found" -2025-10-27T04:02:34.388Z [INFO] server-afca0ee6-39e2-b2ac-c3b1-208bf8269cfa: aborting pipeline replication: peer="{Voter server-8712b5ca-dcf2-7936-4f44-0c0e22b7ef74 8712b5ca-dcf2-7936-4f44-0c0e22b7ef74}" -2025-10-27T04:02:34.397Z [ERROR] server-afca0ee6-39e2-b2ac-c3b1-208bf8269cfa: failed to get log: index=23 error="log not found" -2025-10-27T04:02:34.397Z [INFO] creating new snapshot: path=/tmp/raft4044502211/snapshots/2-23-1761537754397.tmp -2025-10-27T04:02:34.397Z [INFO] server-062ad266-6848-402b-2808-8a76fddb295a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:34.397Z [INFO] server-062ad266-6848-402b-2808-8a76fddb295a: copied to local snapshot: bytes=71 -2025-10-27T04:02:34.397Z [INFO] server-062ad266-6848-402b-2808-8a76fddb295a: snapshot restore progress: id=2-23-1761537754397 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:34.398Z [INFO] server-062ad266-6848-402b-2808-8a76fddb295a: Installed remote snapshot -2025-10-27T04:02:34.398Z [WARN] server-062ad266-6848-402b-2808-8a76fddb295a: skipping application of old log: index=22 -2025-10-27T04:02:34.398Z [INFO] server-afca0ee6-39e2-b2ac-c3b1-208bf8269cfa: pipelining replication: peer="{Voter server-062ad266-6848-402b-2808-8a76fddb295a 062ad266-6848-402b-2808-8a76fddb295a}" -2025-10-27T04:02:34.398Z [ERROR] server-afca0ee6-39e2-b2ac-c3b1-208bf8269cfa: failed to get log: index=23 error="log not found" -2025-10-27T04:02:34.398Z [INFO] creating new snapshot: path=/tmp/raft531099269/snapshots/2-23-1761537754398.tmp -2025-10-27T04:02:34.398Z [INFO] server-8712b5ca-dcf2-7936-4f44-0c0e22b7ef74: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:34.398Z [INFO] server-8712b5ca-dcf2-7936-4f44-0c0e22b7ef74: copied to local snapshot: bytes=71 -2025-10-27T04:02:34.398Z [INFO] server-8712b5ca-dcf2-7936-4f44-0c0e22b7ef74: snapshot restore progress: id=2-23-1761537754398 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:34.398Z [INFO] server-8712b5ca-dcf2-7936-4f44-0c0e22b7ef74: Installed remote snapshot -2025-10-27T04:02:34.398Z [WARN] server-8712b5ca-dcf2-7936-4f44-0c0e22b7ef74: skipping application of old log: index=22 -2025-10-27T04:02:34.398Z [INFO] server-afca0ee6-39e2-b2ac-c3b1-208bf8269cfa: pipelining replication: peer="{Voter server-8712b5ca-dcf2-7936-4f44-0c0e22b7ef74 8712b5ca-dcf2-7936-4f44-0c0e22b7ef74}" -2025-10-27T04:02:34.398Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:02:35.404Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:02:34.398627021 +0000 UTC m=+29.393499419 (2 nodes), 440ns from start of poll, 1.646830175s from cluster start. Timeout at 2025-10-27 04:02:35.403637221 +0000 UTC m=+30.398509659, 1.00501024s after stability -2025-10-27T04:02:35.415Z [INFO] server-afca0ee6-39e2-b2ac-c3b1-208bf8269cfa: aborting pipeline replication: peer="{Voter server-062ad266-6848-402b-2808-8a76fddb295a 062ad266-6848-402b-2808-8a76fddb295a}" -2025-10-27T04:02:35.415Z [INFO] server-afca0ee6-39e2-b2ac-c3b1-208bf8269cfa: aborting pipeline replication: peer="{Voter server-8712b5ca-dcf2-7936-4f44-0c0e22b7ef74 8712b5ca-dcf2-7936-4f44-0c0e22b7ef74}" +2026-11-29T10:28:36.073Z [INFO] server-4d547e5a-7dc1-a431-0b62-ab342fd3cdfe: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d547e5a-7dc1-a431-0b62-ab342fd3cdfe Address:4d547e5a-7dc1-a431-0b62-ab342fd3cdfe} {Suffrage:Voter ID:server-39afa95c-ddf7-e4df-0644-d86e2ba373a3 Address:39afa95c-ddf7-e4df-0644-d86e2ba373a3} {Suffrage:Voter ID:server-ece1311e-0f52-6780-67d6-e385b77bf20a Address:ece1311e-0f52-6780-67d6-e385b77bf20a}]" +2026-11-29T10:28:36.074Z [INFO] server-39afa95c-ddf7-e4df-0644-d86e2ba373a3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d547e5a-7dc1-a431-0b62-ab342fd3cdfe Address:4d547e5a-7dc1-a431-0b62-ab342fd3cdfe} {Suffrage:Voter ID:server-39afa95c-ddf7-e4df-0644-d86e2ba373a3 Address:39afa95c-ddf7-e4df-0644-d86e2ba373a3} {Suffrage:Voter ID:server-ece1311e-0f52-6780-67d6-e385b77bf20a Address:ece1311e-0f52-6780-67d6-e385b77bf20a}]" +2026-11-29T10:28:36.073Z [INFO] server-4d547e5a-7dc1-a431-0b62-ab342fd3cdfe: entering follower state: follower="Node at 4d547e5a-7dc1-a431-0b62-ab342fd3cdfe [Follower]" leader-address= leader-id= +2026-11-29T10:28:36.074Z [INFO] server-39afa95c-ddf7-e4df-0644-d86e2ba373a3: entering follower state: follower="Node at 39afa95c-ddf7-e4df-0644-d86e2ba373a3 [Follower]" leader-address= leader-id= +2026-11-29T10:28:36.074Z [INFO] server-ece1311e-0f52-6780-67d6-e385b77bf20a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d547e5a-7dc1-a431-0b62-ab342fd3cdfe Address:4d547e5a-7dc1-a431-0b62-ab342fd3cdfe} {Suffrage:Voter ID:server-39afa95c-ddf7-e4df-0644-d86e2ba373a3 Address:39afa95c-ddf7-e4df-0644-d86e2ba373a3} {Suffrage:Voter ID:server-ece1311e-0f52-6780-67d6-e385b77bf20a Address:ece1311e-0f52-6780-67d6-e385b77bf20a}]" +2026-11-29T10:28:36.074Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:36.074Z [INFO] server-ece1311e-0f52-6780-67d6-e385b77bf20a: entering follower state: follower="Node at ece1311e-0f52-6780-67d6-e385b77bf20a [Follower]" leader-address= leader-id= +2026-11-29T10:28:36.663Z [WARN] server-4d547e5a-7dc1-a431-0b62-ab342fd3cdfe: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:36.663Z [INFO] server-4d547e5a-7dc1-a431-0b62-ab342fd3cdfe: entering candidate state: node="Node at 4d547e5a-7dc1-a431-0b62-ab342fd3cdfe [Candidate]" term=2 +2026-11-29T10:28:36.663Z [INFO] server-4d547e5a-7dc1-a431-0b62-ab342fd3cdfe: election won: term=2 tally=2 +2026-11-29T10:28:36.663Z [INFO] server-4d547e5a-7dc1-a431-0b62-ab342fd3cdfe: entering leader state: leader="Node at 4d547e5a-7dc1-a431-0b62-ab342fd3cdfe [Leader]" +2026-11-29T10:28:36.663Z [INFO] server-4d547e5a-7dc1-a431-0b62-ab342fd3cdfe: added peer, starting replication: peer=server-39afa95c-ddf7-e4df-0644-d86e2ba373a3 +2026-11-29T10:28:36.663Z [INFO] server-4d547e5a-7dc1-a431-0b62-ab342fd3cdfe: added peer, starting replication: peer=server-ece1311e-0f52-6780-67d6-e385b77bf20a +2026-11-29T10:28:36.663Z [INFO] server-4d547e5a-7dc1-a431-0b62-ab342fd3cdfe: pipelining replication: peer="{Voter server-ece1311e-0f52-6780-67d6-e385b77bf20a ece1311e-0f52-6780-67d6-e385b77bf20a}" +2026-11-29T10:28:36.663Z [INFO] server-4d547e5a-7dc1-a431-0b62-ab342fd3cdfe: pipelining replication: peer="{Voter server-39afa95c-ddf7-e4df-0644-d86e2ba373a3 39afa95c-ddf7-e4df-0644-d86e2ba373a3}" +2026-11-29T10:28:37.668Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:36.663748299 +0000 UTC m=+27.398802441 (1 nodes), 589.57936ms from start of poll, 589.99668ms from cluster start. Timeout at 2026-11-29 10:28:37.668751939 +0000 UTC m=+28.403806081, 1.00500364s after stability +2026-11-29T10:28:37.669Z [INFO] server-4d547e5a-7dc1-a431-0b62-ab342fd3cdfe: starting snapshot up to: index=12 +2026-11-29T10:28:37.669Z [INFO] creating new snapshot: path=/tmp/raft1004542285/snapshots/2-12-1795948117669.tmp +2026-11-29T10:28:37.670Z [INFO] server-4d547e5a-7dc1-a431-0b62-ab342fd3cdfe: snapshot complete up to: index=12 +2026-11-29T10:28:37.670Z [INFO] creating new snapshot: path=/tmp/raft1004542285/snapshots/2-23-1795948117670.tmp +2026-11-29T10:28:37.671Z [INFO] server-4d547e5a-7dc1-a431-0b62-ab342fd3cdfe: copied to local snapshot: bytes=71 +2026-11-29T10:28:37.672Z [INFO] server-4d547e5a-7dc1-a431-0b62-ab342fd3cdfe: snapshot restore progress: id=2-23-1795948117670 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:37.672Z [INFO] server-4d547e5a-7dc1-a431-0b62-ab342fd3cdfe: restored user snapshot: index=23 +2026-11-29T10:28:37.672Z [ERROR] server-4d547e5a-7dc1-a431-0b62-ab342fd3cdfe: failed to get log: index=23 error="log not found" +2026-11-29T10:28:37.672Z [INFO] server-4d547e5a-7dc1-a431-0b62-ab342fd3cdfe: aborting pipeline replication: peer="{Voter server-ece1311e-0f52-6780-67d6-e385b77bf20a ece1311e-0f52-6780-67d6-e385b77bf20a}" +2026-11-29T10:28:37.672Z [ERROR] server-4d547e5a-7dc1-a431-0b62-ab342fd3cdfe: failed to get log: index=23 error="log not found" +2026-11-29T10:28:37.672Z [INFO] server-4d547e5a-7dc1-a431-0b62-ab342fd3cdfe: aborting pipeline replication: peer="{Voter server-39afa95c-ddf7-e4df-0644-d86e2ba373a3 39afa95c-ddf7-e4df-0644-d86e2ba373a3}" +2026-11-29T10:28:37.680Z [ERROR] server-4d547e5a-7dc1-a431-0b62-ab342fd3cdfe: failed to get log: index=23 error="log not found" +2026-11-29T10:28:37.681Z [ERROR] server-4d547e5a-7dc1-a431-0b62-ab342fd3cdfe: failed to get log: index=23 error="log not found" +2026-11-29T10:28:37.681Z [INFO] creating new snapshot: path=/tmp/raft3325421086/snapshots/2-23-1795948117681.tmp +2026-11-29T10:28:37.681Z [INFO] creating new snapshot: path=/tmp/raft3667773414/snapshots/2-23-1795948117681.tmp +2026-11-29T10:28:37.681Z [INFO] server-ece1311e-0f52-6780-67d6-e385b77bf20a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:37.681Z [INFO] server-39afa95c-ddf7-e4df-0644-d86e2ba373a3: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:37.681Z [INFO] server-ece1311e-0f52-6780-67d6-e385b77bf20a: copied to local snapshot: bytes=71 +2026-11-29T10:28:37.682Z [INFO] server-ece1311e-0f52-6780-67d6-e385b77bf20a: snapshot restore progress: id=2-23-1795948117681 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:37.682Z [INFO] server-ece1311e-0f52-6780-67d6-e385b77bf20a: Installed remote snapshot +2026-11-29T10:28:37.682Z [WARN] server-ece1311e-0f52-6780-67d6-e385b77bf20a: skipping application of old log: index=22 +2026-11-29T10:28:37.682Z [INFO] server-4d547e5a-7dc1-a431-0b62-ab342fd3cdfe: pipelining replication: peer="{Voter server-ece1311e-0f52-6780-67d6-e385b77bf20a ece1311e-0f52-6780-67d6-e385b77bf20a}" +2026-11-29T10:28:37.682Z [INFO] server-39afa95c-ddf7-e4df-0644-d86e2ba373a3: copied to local snapshot: bytes=71 +2026-11-29T10:28:37.682Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:28:37.682Z [INFO] server-39afa95c-ddf7-e4df-0644-d86e2ba373a3: snapshot restore progress: id=2-23-1795948117681 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:37.682Z [INFO] server-39afa95c-ddf7-e4df-0644-d86e2ba373a3: Installed remote snapshot +2026-11-29T10:28:37.682Z [INFO] server-4d547e5a-7dc1-a431-0b62-ab342fd3cdfe: pipelining replication: peer="{Voter server-39afa95c-ddf7-e4df-0644-d86e2ba373a3 39afa95c-ddf7-e4df-0644-d86e2ba373a3}" +2026-11-29T10:28:38.688Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:28:37.682223779 +0000 UTC m=+28.417277921 (2 nodes), 880ns from start of poll, 1.60847216s from cluster start. Timeout at 2026-11-29 10:28:38.687237299 +0000 UTC m=+29.422291441, 1.00501352s after stability +2026-11-29T10:28:38.701Z [INFO] server-4d547e5a-7dc1-a431-0b62-ab342fd3cdfe: aborting pipeline replication: peer="{Voter server-ece1311e-0f52-6780-67d6-e385b77bf20a ece1311e-0f52-6780-67d6-e385b77bf20a}" +2026-11-29T10:28:38.701Z [INFO] server-4d547e5a-7dc1-a431-0b62-ab342fd3cdfe: aborting pipeline replication: peer="{Voter server-39afa95c-ddf7-e4df-0644-d86e2ba373a3 39afa95c-ddf7-e4df-0644-d86e2ba373a3}" === RUN TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_false -2025-10-27T04:02:35.416Z [INFO] server-3a76ad45-1ec6-3adc-c4be-a4c68eab57d6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3a76ad45-1ec6-3adc-c4be-a4c68eab57d6 Address:3a76ad45-1ec6-3adc-c4be-a4c68eab57d6} {Suffrage:Voter ID:server-a5a0c255-699f-cf01-4341-92e5db4309c1 Address:a5a0c255-699f-cf01-4341-92e5db4309c1} {Suffrage:Voter ID:server-9c86b0d6-9281-03f5-6c7d-11e4f685efe9 Address:9c86b0d6-9281-03f5-6c7d-11e4f685efe9}]" -2025-10-27T04:02:35.416Z [INFO] server-a5a0c255-699f-cf01-4341-92e5db4309c1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3a76ad45-1ec6-3adc-c4be-a4c68eab57d6 Address:3a76ad45-1ec6-3adc-c4be-a4c68eab57d6} {Suffrage:Voter ID:server-a5a0c255-699f-cf01-4341-92e5db4309c1 Address:a5a0c255-699f-cf01-4341-92e5db4309c1} {Suffrage:Voter ID:server-9c86b0d6-9281-03f5-6c7d-11e4f685efe9 Address:9c86b0d6-9281-03f5-6c7d-11e4f685efe9}]" -2025-10-27T04:02:35.416Z [INFO] server-9c86b0d6-9281-03f5-6c7d-11e4f685efe9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3a76ad45-1ec6-3adc-c4be-a4c68eab57d6 Address:3a76ad45-1ec6-3adc-c4be-a4c68eab57d6} {Suffrage:Voter ID:server-a5a0c255-699f-cf01-4341-92e5db4309c1 Address:a5a0c255-699f-cf01-4341-92e5db4309c1} {Suffrage:Voter ID:server-9c86b0d6-9281-03f5-6c7d-11e4f685efe9 Address:9c86b0d6-9281-03f5-6c7d-11e4f685efe9}]" -2025-10-27T04:02:35.416Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:35.416Z [INFO] server-3a76ad45-1ec6-3adc-c4be-a4c68eab57d6: entering follower state: follower="Node at 3a76ad45-1ec6-3adc-c4be-a4c68eab57d6 [Follower]" leader-address= leader-id= -2025-10-27T04:02:35.416Z [INFO] server-a5a0c255-699f-cf01-4341-92e5db4309c1: entering follower state: follower="Node at a5a0c255-699f-cf01-4341-92e5db4309c1 [Follower]" leader-address= leader-id= -2025-10-27T04:02:35.416Z [INFO] server-9c86b0d6-9281-03f5-6c7d-11e4f685efe9: entering follower state: follower="Node at 9c86b0d6-9281-03f5-6c7d-11e4f685efe9 [Follower]" leader-address= leader-id= -2025-10-27T04:02:36.072Z [WARN] server-a5a0c255-699f-cf01-4341-92e5db4309c1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:36.072Z [INFO] server-a5a0c255-699f-cf01-4341-92e5db4309c1: entering candidate state: node="Node at a5a0c255-699f-cf01-4341-92e5db4309c1 [Candidate]" term=2 -2025-10-27T04:02:36.072Z [INFO] server-a5a0c255-699f-cf01-4341-92e5db4309c1: election won: term=2 tally=2 -2025-10-27T04:02:36.072Z [INFO] server-a5a0c255-699f-cf01-4341-92e5db4309c1: entering leader state: leader="Node at a5a0c255-699f-cf01-4341-92e5db4309c1 [Leader]" -2025-10-27T04:02:36.072Z [INFO] server-a5a0c255-699f-cf01-4341-92e5db4309c1: added peer, starting replication: peer=server-3a76ad45-1ec6-3adc-c4be-a4c68eab57d6 -2025-10-27T04:02:36.072Z [INFO] server-a5a0c255-699f-cf01-4341-92e5db4309c1: added peer, starting replication: peer=server-9c86b0d6-9281-03f5-6c7d-11e4f685efe9 -2025-10-27T04:02:36.072Z [INFO] server-a5a0c255-699f-cf01-4341-92e5db4309c1: pipelining replication: peer="{Voter server-9c86b0d6-9281-03f5-6c7d-11e4f685efe9 9c86b0d6-9281-03f5-6c7d-11e4f685efe9}" -2025-10-27T04:02:36.072Z [INFO] server-a5a0c255-699f-cf01-4341-92e5db4309c1: pipelining replication: peer="{Voter server-3a76ad45-1ec6-3adc-c4be-a4c68eab57d6 3a76ad45-1ec6-3adc-c4be-a4c68eab57d6}" -2025-10-27T04:02:37.078Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:36.07233014 +0000 UTC m=+31.067202578 (1 nodes), 656.099764ms from start of poll, 656.269965ms from cluster start. Timeout at 2025-10-27 04:02:37.07733642 +0000 UTC m=+32.072208858, 1.00500628s after stability -2025-10-27T04:02:37.078Z [INFO] server-a5a0c255-699f-cf01-4341-92e5db4309c1: starting snapshot up to: index=12 -2025-10-27T04:02:37.078Z [INFO] creating new snapshot: path=/tmp/raft177180038/snapshots/2-12-1761537757078.tmp -2025-10-27T04:02:37.079Z [INFO] server-a5a0c255-699f-cf01-4341-92e5db4309c1: snapshot complete up to: index=12 -2025-10-27T04:02:37.081Z [INFO] creating new snapshot: path=/tmp/raft177180038/snapshots/2-23-1761537757081.tmp -2025-10-27T04:02:37.082Z [INFO] server-a5a0c255-699f-cf01-4341-92e5db4309c1: copied to local snapshot: bytes=71 -2025-10-27T04:02:37.082Z [INFO] server-a5a0c255-699f-cf01-4341-92e5db4309c1: snapshot restore progress: id=2-23-1761537757081 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:37.082Z [INFO] server-a5a0c255-699f-cf01-4341-92e5db4309c1: removing all old logs from log store -2025-10-27T04:02:37.082Z [INFO] server-a5a0c255-699f-cf01-4341-92e5db4309c1: compacting logs: from=1 to=22 -2025-10-27T04:02:37.082Z [INFO] server-a5a0c255-699f-cf01-4341-92e5db4309c1: restored user snapshot: index=23 -2025-10-27T04:02:37.082Z [ERROR] server-a5a0c255-699f-cf01-4341-92e5db4309c1: failed to get log: index=22 error="log not found" -2025-10-27T04:02:37.082Z [INFO] server-a5a0c255-699f-cf01-4341-92e5db4309c1: aborting pipeline replication: peer="{Voter server-9c86b0d6-9281-03f5-6c7d-11e4f685efe9 9c86b0d6-9281-03f5-6c7d-11e4f685efe9}" -2025-10-27T04:02:37.082Z [ERROR] server-a5a0c255-699f-cf01-4341-92e5db4309c1: failed to get log: index=22 error="log not found" -2025-10-27T04:02:37.082Z [INFO] server-a5a0c255-699f-cf01-4341-92e5db4309c1: aborting pipeline replication: peer="{Voter server-3a76ad45-1ec6-3adc-c4be-a4c68eab57d6 3a76ad45-1ec6-3adc-c4be-a4c68eab57d6}" -2025-10-27T04:02:37.089Z [ERROR] server-a5a0c255-699f-cf01-4341-92e5db4309c1: failed to get log: index=22 error="log not found" -2025-10-27T04:02:37.090Z [INFO] creating new snapshot: path=/tmp/raft3333853061/snapshots/2-23-1761537757090.tmp -2025-10-27T04:02:37.090Z [INFO] server-9c86b0d6-9281-03f5-6c7d-11e4f685efe9: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:37.090Z [INFO] server-9c86b0d6-9281-03f5-6c7d-11e4f685efe9: copied to local snapshot: bytes=71 -2025-10-27T04:02:37.090Z [INFO] server-9c86b0d6-9281-03f5-6c7d-11e4f685efe9: snapshot restore progress: id=2-23-1761537757090 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:37.090Z [INFO] server-9c86b0d6-9281-03f5-6c7d-11e4f685efe9: removing all old logs from log store -2025-10-27T04:02:37.090Z [INFO] server-9c86b0d6-9281-03f5-6c7d-11e4f685efe9: compacting logs: from=1 to=22 -2025-10-27T04:02:37.090Z [INFO] server-9c86b0d6-9281-03f5-6c7d-11e4f685efe9: Installed remote snapshot -2025-10-27T04:02:37.090Z [WARN] server-9c86b0d6-9281-03f5-6c7d-11e4f685efe9: skipping application of old log: index=22 -2025-10-27T04:02:37.090Z [INFO] server-a5a0c255-699f-cf01-4341-92e5db4309c1: pipelining replication: peer="{Voter server-9c86b0d6-9281-03f5-6c7d-11e4f685efe9 9c86b0d6-9281-03f5-6c7d-11e4f685efe9}" -2025-10-27T04:02:37.090Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:02:37.091Z [ERROR] server-a5a0c255-699f-cf01-4341-92e5db4309c1: failed to get log: index=22 error="log not found" -2025-10-27T04:02:37.092Z [INFO] creating new snapshot: path=/tmp/raft641336691/snapshots/2-23-1761537757092.tmp -2025-10-27T04:02:37.092Z [INFO] server-3a76ad45-1ec6-3adc-c4be-a4c68eab57d6: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:37.092Z [INFO] server-3a76ad45-1ec6-3adc-c4be-a4c68eab57d6: copied to local snapshot: bytes=71 -2025-10-27T04:02:37.092Z [INFO] server-3a76ad45-1ec6-3adc-c4be-a4c68eab57d6: snapshot restore progress: id=2-23-1761537757092 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:37.092Z [INFO] server-3a76ad45-1ec6-3adc-c4be-a4c68eab57d6: removing all old logs from log store -2025-10-27T04:02:37.092Z [INFO] server-3a76ad45-1ec6-3adc-c4be-a4c68eab57d6: compacting logs: from=1 to=22 -2025-10-27T04:02:37.092Z [INFO] server-3a76ad45-1ec6-3adc-c4be-a4c68eab57d6: Installed remote snapshot -2025-10-27T04:02:37.092Z [INFO] server-a5a0c255-699f-cf01-4341-92e5db4309c1: pipelining replication: peer="{Voter server-3a76ad45-1ec6-3adc-c4be-a4c68eab57d6 3a76ad45-1ec6-3adc-c4be-a4c68eab57d6}" -2025-10-27T04:02:38.095Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:02:37.090587539 +0000 UTC m=+32.085459977 (2 nodes), 680ns from start of poll, 1.674527364s from cluster start. Timeout at 2025-10-27 04:02:38.095599739 +0000 UTC m=+33.090472177, 1.0050122s after stability -2025-10-27T04:02:38.107Z [INFO] server-a5a0c255-699f-cf01-4341-92e5db4309c1: aborting pipeline replication: peer="{Voter server-9c86b0d6-9281-03f5-6c7d-11e4f685efe9 9c86b0d6-9281-03f5-6c7d-11e4f685efe9}" -2025-10-27T04:02:38.107Z [INFO] server-a5a0c255-699f-cf01-4341-92e5db4309c1: aborting pipeline replication: peer="{Voter server-3a76ad45-1ec6-3adc-c4be-a4c68eab57d6 3a76ad45-1ec6-3adc-c4be-a4c68eab57d6}" +2026-11-29T10:28:38.702Z [INFO] server-73639abc-3f53-7e9d-13ec-e5f0091ab9df: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-73639abc-3f53-7e9d-13ec-e5f0091ab9df Address:73639abc-3f53-7e9d-13ec-e5f0091ab9df} {Suffrage:Voter ID:server-ab813b41-1419-b432-a97e-f2a197337bfa Address:ab813b41-1419-b432-a97e-f2a197337bfa} {Suffrage:Voter ID:server-d6ff325c-7c3b-6cfe-0962-70a5c41daab5 Address:d6ff325c-7c3b-6cfe-0962-70a5c41daab5}]" +2026-11-29T10:28:38.702Z [INFO] server-ab813b41-1419-b432-a97e-f2a197337bfa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-73639abc-3f53-7e9d-13ec-e5f0091ab9df Address:73639abc-3f53-7e9d-13ec-e5f0091ab9df} {Suffrage:Voter ID:server-ab813b41-1419-b432-a97e-f2a197337bfa Address:ab813b41-1419-b432-a97e-f2a197337bfa} {Suffrage:Voter ID:server-d6ff325c-7c3b-6cfe-0962-70a5c41daab5 Address:d6ff325c-7c3b-6cfe-0962-70a5c41daab5}]" +2026-11-29T10:28:38.702Z [INFO] server-73639abc-3f53-7e9d-13ec-e5f0091ab9df: entering follower state: follower="Node at 73639abc-3f53-7e9d-13ec-e5f0091ab9df [Follower]" leader-address= leader-id= +2026-11-29T10:28:38.702Z [INFO] server-d6ff325c-7c3b-6cfe-0962-70a5c41daab5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-73639abc-3f53-7e9d-13ec-e5f0091ab9df Address:73639abc-3f53-7e9d-13ec-e5f0091ab9df} {Suffrage:Voter ID:server-ab813b41-1419-b432-a97e-f2a197337bfa Address:ab813b41-1419-b432-a97e-f2a197337bfa} {Suffrage:Voter ID:server-d6ff325c-7c3b-6cfe-0962-70a5c41daab5 Address:d6ff325c-7c3b-6cfe-0962-70a5c41daab5}]" +2026-11-29T10:28:38.702Z [INFO] server-ab813b41-1419-b432-a97e-f2a197337bfa: entering follower state: follower="Node at ab813b41-1419-b432-a97e-f2a197337bfa [Follower]" leader-address= leader-id= +2026-11-29T10:28:38.702Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:38.702Z [INFO] server-d6ff325c-7c3b-6cfe-0962-70a5c41daab5: entering follower state: follower="Node at d6ff325c-7c3b-6cfe-0962-70a5c41daab5 [Follower]" leader-address= leader-id= +2026-11-29T10:28:39.432Z [WARN] server-d6ff325c-7c3b-6cfe-0962-70a5c41daab5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:39.432Z [INFO] server-d6ff325c-7c3b-6cfe-0962-70a5c41daab5: entering candidate state: node="Node at d6ff325c-7c3b-6cfe-0962-70a5c41daab5 [Candidate]" term=2 +2026-11-29T10:28:39.433Z [INFO] server-d6ff325c-7c3b-6cfe-0962-70a5c41daab5: election won: term=2 tally=2 +2026-11-29T10:28:39.433Z [INFO] server-d6ff325c-7c3b-6cfe-0962-70a5c41daab5: entering leader state: leader="Node at d6ff325c-7c3b-6cfe-0962-70a5c41daab5 [Leader]" +2026-11-29T10:28:39.433Z [INFO] server-d6ff325c-7c3b-6cfe-0962-70a5c41daab5: added peer, starting replication: peer=server-73639abc-3f53-7e9d-13ec-e5f0091ab9df +2026-11-29T10:28:39.433Z [INFO] server-d6ff325c-7c3b-6cfe-0962-70a5c41daab5: added peer, starting replication: peer=server-ab813b41-1419-b432-a97e-f2a197337bfa +2026-11-29T10:28:39.433Z [INFO] server-d6ff325c-7c3b-6cfe-0962-70a5c41daab5: pipelining replication: peer="{Voter server-73639abc-3f53-7e9d-13ec-e5f0091ab9df 73639abc-3f53-7e9d-13ec-e5f0091ab9df}" +2026-11-29T10:28:39.433Z [INFO] server-d6ff325c-7c3b-6cfe-0962-70a5c41daab5: pipelining replication: peer="{Voter server-ab813b41-1419-b432-a97e-f2a197337bfa ab813b41-1419-b432-a97e-f2a197337bfa}" +2026-11-29T10:28:40.439Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:39.433154179 +0000 UTC m=+30.168208281 (1 nodes), 730.59132ms from start of poll, 730.91616ms from cluster start. Timeout at 2026-11-29 10:28:40.438157619 +0000 UTC m=+31.173211761, 1.00500348s after stability +2026-11-29T10:28:40.441Z [INFO] server-d6ff325c-7c3b-6cfe-0962-70a5c41daab5: starting snapshot up to: index=12 +2026-11-29T10:28:40.441Z [INFO] creating new snapshot: path=/tmp/raft4122625933/snapshots/2-12-1795948120441.tmp +2026-11-29T10:28:40.441Z [INFO] server-d6ff325c-7c3b-6cfe-0962-70a5c41daab5: snapshot complete up to: index=12 +2026-11-29T10:28:40.442Z [INFO] creating new snapshot: path=/tmp/raft4122625933/snapshots/2-23-1795948120442.tmp +2026-11-29T10:28:40.442Z [INFO] server-d6ff325c-7c3b-6cfe-0962-70a5c41daab5: copied to local snapshot: bytes=71 +2026-11-29T10:28:40.442Z [INFO] server-d6ff325c-7c3b-6cfe-0962-70a5c41daab5: snapshot restore progress: id=2-23-1795948120442 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:40.442Z [INFO] server-d6ff325c-7c3b-6cfe-0962-70a5c41daab5: removing all old logs from log store +2026-11-29T10:28:40.442Z [INFO] server-d6ff325c-7c3b-6cfe-0962-70a5c41daab5: compacting logs: from=1 to=22 +2026-11-29T10:28:40.442Z [INFO] server-d6ff325c-7c3b-6cfe-0962-70a5c41daab5: restored user snapshot: index=23 +2026-11-29T10:28:40.442Z [ERROR] server-d6ff325c-7c3b-6cfe-0962-70a5c41daab5: failed to get log: index=22 error="log not found" +2026-11-29T10:28:40.442Z [ERROR] server-d6ff325c-7c3b-6cfe-0962-70a5c41daab5: failed to get log: index=22 error="log not found" +2026-11-29T10:28:40.442Z [INFO] server-d6ff325c-7c3b-6cfe-0962-70a5c41daab5: aborting pipeline replication: peer="{Voter server-73639abc-3f53-7e9d-13ec-e5f0091ab9df 73639abc-3f53-7e9d-13ec-e5f0091ab9df}" +2026-11-29T10:28:40.442Z [INFO] server-d6ff325c-7c3b-6cfe-0962-70a5c41daab5: aborting pipeline replication: peer="{Voter server-ab813b41-1419-b432-a97e-f2a197337bfa ab813b41-1419-b432-a97e-f2a197337bfa}" +2026-11-29T10:28:40.450Z [ERROR] server-d6ff325c-7c3b-6cfe-0962-70a5c41daab5: failed to get log: index=22 error="log not found" +2026-11-29T10:28:40.450Z [INFO] creating new snapshot: path=/tmp/raft2741569840/snapshots/2-23-1795948120450.tmp +2026-11-29T10:28:40.451Z [INFO] server-ab813b41-1419-b432-a97e-f2a197337bfa: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:40.451Z [INFO] server-ab813b41-1419-b432-a97e-f2a197337bfa: copied to local snapshot: bytes=71 +2026-11-29T10:28:40.451Z [INFO] server-ab813b41-1419-b432-a97e-f2a197337bfa: snapshot restore progress: id=2-23-1795948120450 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:40.451Z [INFO] server-ab813b41-1419-b432-a97e-f2a197337bfa: removing all old logs from log store +2026-11-29T10:28:40.451Z [INFO] server-ab813b41-1419-b432-a97e-f2a197337bfa: compacting logs: from=1 to=22 +2026-11-29T10:28:40.451Z [INFO] server-ab813b41-1419-b432-a97e-f2a197337bfa: Installed remote snapshot +2026-11-29T10:28:40.451Z [WARN] server-ab813b41-1419-b432-a97e-f2a197337bfa: skipping application of old log: index=22 +2026-11-29T10:28:40.451Z [INFO] server-d6ff325c-7c3b-6cfe-0962-70a5c41daab5: pipelining replication: peer="{Voter server-ab813b41-1419-b432-a97e-f2a197337bfa ab813b41-1419-b432-a97e-f2a197337bfa}" +2026-11-29T10:28:40.451Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:28:40.452Z [ERROR] server-d6ff325c-7c3b-6cfe-0962-70a5c41daab5: failed to get log: index=22 error="log not found" +2026-11-29T10:28:40.453Z [INFO] creating new snapshot: path=/tmp/raft2433978452/snapshots/2-23-1795948120453.tmp +2026-11-29T10:28:40.453Z [INFO] server-73639abc-3f53-7e9d-13ec-e5f0091ab9df: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:40.453Z [INFO] server-73639abc-3f53-7e9d-13ec-e5f0091ab9df: copied to local snapshot: bytes=71 +2026-11-29T10:28:40.454Z [INFO] server-73639abc-3f53-7e9d-13ec-e5f0091ab9df: snapshot restore progress: id=2-23-1795948120453 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:40.454Z [INFO] server-73639abc-3f53-7e9d-13ec-e5f0091ab9df: removing all old logs from log store +2026-11-29T10:28:40.454Z [INFO] server-73639abc-3f53-7e9d-13ec-e5f0091ab9df: compacting logs: from=1 to=22 +2026-11-29T10:28:40.454Z [INFO] server-73639abc-3f53-7e9d-13ec-e5f0091ab9df: Installed remote snapshot +2026-11-29T10:28:40.454Z [INFO] server-d6ff325c-7c3b-6cfe-0962-70a5c41daab5: pipelining replication: peer="{Voter server-73639abc-3f53-7e9d-13ec-e5f0091ab9df 73639abc-3f53-7e9d-13ec-e5f0091ab9df}" +2026-11-29T10:28:41.456Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:28:40.451491459 +0000 UTC m=+31.186545521 (2 nodes), 880ns from start of poll, 1.7492534s from cluster start. Timeout at 2026-11-29 10:28:41.456506019 +0000 UTC m=+32.191560161, 1.00501464s after stability +2026-11-29T10:28:41.469Z [INFO] server-d6ff325c-7c3b-6cfe-0962-70a5c41daab5: aborting pipeline replication: peer="{Voter server-ab813b41-1419-b432-a97e-f2a197337bfa ab813b41-1419-b432-a97e-f2a197337bfa}" +2026-11-29T10:28:41.469Z [INFO] server-d6ff325c-7c3b-6cfe-0962-70a5c41daab5: aborting pipeline replication: peer="{Voter server-73639abc-3f53-7e9d-13ec-e5f0091ab9df 73639abc-3f53-7e9d-13ec-e5f0091ab9df}" === RUN TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_true -2025-10-27T04:02:38.108Z [INFO] server-09464ff2-b1f6-bd37-40b3-3c6df5d5b8c2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-09464ff2-b1f6-bd37-40b3-3c6df5d5b8c2 Address:09464ff2-b1f6-bd37-40b3-3c6df5d5b8c2} {Suffrage:Voter ID:server-c6fc7443-24b3-6e6e-6bca-c5374a14ea9b Address:c6fc7443-24b3-6e6e-6bca-c5374a14ea9b} {Suffrage:Voter ID:server-57acc957-fcea-9d09-1f88-9aed3fa4c11c Address:57acc957-fcea-9d09-1f88-9aed3fa4c11c}]" -2025-10-27T04:02:38.108Z [INFO] server-c6fc7443-24b3-6e6e-6bca-c5374a14ea9b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-09464ff2-b1f6-bd37-40b3-3c6df5d5b8c2 Address:09464ff2-b1f6-bd37-40b3-3c6df5d5b8c2} {Suffrage:Voter ID:server-c6fc7443-24b3-6e6e-6bca-c5374a14ea9b Address:c6fc7443-24b3-6e6e-6bca-c5374a14ea9b} {Suffrage:Voter ID:server-57acc957-fcea-9d09-1f88-9aed3fa4c11c Address:57acc957-fcea-9d09-1f88-9aed3fa4c11c}]" -2025-10-27T04:02:38.108Z [INFO] server-09464ff2-b1f6-bd37-40b3-3c6df5d5b8c2: entering follower state: follower="Node at 09464ff2-b1f6-bd37-40b3-3c6df5d5b8c2 [Follower]" leader-address= leader-id= -2025-10-27T04:02:38.108Z [INFO] server-57acc957-fcea-9d09-1f88-9aed3fa4c11c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-09464ff2-b1f6-bd37-40b3-3c6df5d5b8c2 Address:09464ff2-b1f6-bd37-40b3-3c6df5d5b8c2} {Suffrage:Voter ID:server-c6fc7443-24b3-6e6e-6bca-c5374a14ea9b Address:c6fc7443-24b3-6e6e-6bca-c5374a14ea9b} {Suffrage:Voter ID:server-57acc957-fcea-9d09-1f88-9aed3fa4c11c Address:57acc957-fcea-9d09-1f88-9aed3fa4c11c}]" -2025-10-27T04:02:38.108Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:38.108Z [INFO] server-c6fc7443-24b3-6e6e-6bca-c5374a14ea9b: entering follower state: follower="Node at c6fc7443-24b3-6e6e-6bca-c5374a14ea9b [Follower]" leader-address= leader-id= -2025-10-27T04:02:38.108Z [INFO] server-57acc957-fcea-9d09-1f88-9aed3fa4c11c: entering follower state: follower="Node at 57acc957-fcea-9d09-1f88-9aed3fa4c11c [Follower]" leader-address= leader-id= -2025-10-27T04:02:38.652Z [WARN] server-c6fc7443-24b3-6e6e-6bca-c5374a14ea9b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:38.653Z [INFO] server-c6fc7443-24b3-6e6e-6bca-c5374a14ea9b: entering candidate state: node="Node at c6fc7443-24b3-6e6e-6bca-c5374a14ea9b [Candidate]" term=2 -2025-10-27T04:02:38.653Z [INFO] server-c6fc7443-24b3-6e6e-6bca-c5374a14ea9b: election won: term=2 tally=2 -2025-10-27T04:02:38.653Z [INFO] server-c6fc7443-24b3-6e6e-6bca-c5374a14ea9b: entering leader state: leader="Node at c6fc7443-24b3-6e6e-6bca-c5374a14ea9b [Leader]" -2025-10-27T04:02:38.653Z [INFO] server-c6fc7443-24b3-6e6e-6bca-c5374a14ea9b: added peer, starting replication: peer=server-09464ff2-b1f6-bd37-40b3-3c6df5d5b8c2 -2025-10-27T04:02:38.653Z [INFO] server-c6fc7443-24b3-6e6e-6bca-c5374a14ea9b: added peer, starting replication: peer=server-57acc957-fcea-9d09-1f88-9aed3fa4c11c -2025-10-27T04:02:38.653Z [INFO] server-c6fc7443-24b3-6e6e-6bca-c5374a14ea9b: pipelining replication: peer="{Voter server-57acc957-fcea-9d09-1f88-9aed3fa4c11c 57acc957-fcea-9d09-1f88-9aed3fa4c11c}" -2025-10-27T04:02:38.653Z [INFO] server-c6fc7443-24b3-6e6e-6bca-c5374a14ea9b: pipelining replication: peer="{Voter server-09464ff2-b1f6-bd37-40b3-3c6df5d5b8c2 09464ff2-b1f6-bd37-40b3-3c6df5d5b8c2}" -2025-10-27T04:02:39.658Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:38.653232098 +0000 UTC m=+33.648104536 (1 nodes), 545.046565ms from start of poll, 545.227686ms from cluster start. Timeout at 2025-10-27 04:02:39.658236618 +0000 UTC m=+34.653109096, 1.00500456s after stability -2025-10-27T04:02:39.659Z [INFO] server-c6fc7443-24b3-6e6e-6bca-c5374a14ea9b: starting snapshot up to: index=12 -2025-10-27T04:02:39.659Z [INFO] creating new snapshot: path=/tmp/raft287475780/snapshots/2-12-1761537759659.tmp -2025-10-27T04:02:39.659Z [INFO] server-c6fc7443-24b3-6e6e-6bca-c5374a14ea9b: snapshot complete up to: index=12 -2025-10-27T04:02:39.660Z [INFO] server-08153ebc-1fa3-f020-10d5-f45c5b811d2f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-08153ebc-1fa3-f020-10d5-f45c5b811d2f Address:08153ebc-1fa3-f020-10d5-f45c5b811d2f} {Suffrage:Voter ID:server-728fe845-0745-e242-ae8f-3ccbdfb5ebe4 Address:728fe845-0745-e242-ae8f-3ccbdfb5ebe4} {Suffrage:Voter ID:server-7e88a31d-d446-6dd1-9777-ed0ddd3b638b Address:7e88a31d-d446-6dd1-9777-ed0ddd3b638b}]" -2025-10-27T04:02:39.660Z [INFO] server-08153ebc-1fa3-f020-10d5-f45c5b811d2f: entering follower state: follower="Node at 08153ebc-1fa3-f020-10d5-f45c5b811d2f [Follower]" leader-address= leader-id= -2025-10-27T04:02:39.660Z [INFO] server-728fe845-0745-e242-ae8f-3ccbdfb5ebe4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-08153ebc-1fa3-f020-10d5-f45c5b811d2f Address:08153ebc-1fa3-f020-10d5-f45c5b811d2f} {Suffrage:Voter ID:server-728fe845-0745-e242-ae8f-3ccbdfb5ebe4 Address:728fe845-0745-e242-ae8f-3ccbdfb5ebe4} {Suffrage:Voter ID:server-7e88a31d-d446-6dd1-9777-ed0ddd3b638b Address:7e88a31d-d446-6dd1-9777-ed0ddd3b638b}]" -2025-10-27T04:02:39.661Z [INFO] server-728fe845-0745-e242-ae8f-3ccbdfb5ebe4: entering follower state: follower="Node at 728fe845-0745-e242-ae8f-3ccbdfb5ebe4 [Follower]" leader-address= leader-id= -2025-10-27T04:02:39.661Z [INFO] server-7e88a31d-d446-6dd1-9777-ed0ddd3b638b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-08153ebc-1fa3-f020-10d5-f45c5b811d2f Address:08153ebc-1fa3-f020-10d5-f45c5b811d2f} {Suffrage:Voter ID:server-728fe845-0745-e242-ae8f-3ccbdfb5ebe4 Address:728fe845-0745-e242-ae8f-3ccbdfb5ebe4} {Suffrage:Voter ID:server-7e88a31d-d446-6dd1-9777-ed0ddd3b638b Address:7e88a31d-d446-6dd1-9777-ed0ddd3b638b}]" -2025-10-27T04:02:39.661Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:39.661Z [INFO] server-7e88a31d-d446-6dd1-9777-ed0ddd3b638b: entering follower state: follower="Node at 7e88a31d-d446-6dd1-9777-ed0ddd3b638b [Follower]" leader-address= leader-id= -2025-10-27T04:02:40.198Z [WARN] server-7e88a31d-d446-6dd1-9777-ed0ddd3b638b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:40.198Z [INFO] server-7e88a31d-d446-6dd1-9777-ed0ddd3b638b: entering candidate state: node="Node at 7e88a31d-d446-6dd1-9777-ed0ddd3b638b [Candidate]" term=2 -2025-10-27T04:02:40.199Z [INFO] server-7e88a31d-d446-6dd1-9777-ed0ddd3b638b: election won: term=2 tally=2 -2025-10-27T04:02:40.199Z [INFO] server-7e88a31d-d446-6dd1-9777-ed0ddd3b638b: entering leader state: leader="Node at 7e88a31d-d446-6dd1-9777-ed0ddd3b638b [Leader]" -2025-10-27T04:02:40.199Z [INFO] server-7e88a31d-d446-6dd1-9777-ed0ddd3b638b: added peer, starting replication: peer=server-08153ebc-1fa3-f020-10d5-f45c5b811d2f -2025-10-27T04:02:40.199Z [INFO] server-7e88a31d-d446-6dd1-9777-ed0ddd3b638b: added peer, starting replication: peer=server-728fe845-0745-e242-ae8f-3ccbdfb5ebe4 -2025-10-27T04:02:40.200Z [INFO] server-7e88a31d-d446-6dd1-9777-ed0ddd3b638b: pipelining replication: peer="{Voter server-728fe845-0745-e242-ae8f-3ccbdfb5ebe4 728fe845-0745-e242-ae8f-3ccbdfb5ebe4}" -2025-10-27T04:02:40.200Z [INFO] server-7e88a31d-d446-6dd1-9777-ed0ddd3b638b: pipelining replication: peer="{Voter server-08153ebc-1fa3-f020-10d5-f45c5b811d2f 08153ebc-1fa3-f020-10d5-f45c5b811d2f}" -2025-10-27T04:02:41.205Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:40.199499225 +0000 UTC m=+35.194371703 (1 nodes), 538.171197ms from start of poll, 538.774562ms from cluster start. Timeout at 2025-10-27 04:02:41.204501145 +0000 UTC m=+36.199373623, 1.00500192s after stability -2025-10-27T04:02:41.205Z [INFO] creating new snapshot: path=/tmp/raft1001490981/snapshots/2-14-1761537761205.tmp -2025-10-27T04:02:41.206Z [INFO] server-7e88a31d-d446-6dd1-9777-ed0ddd3b638b: copied to local snapshot: bytes=71 -2025-10-27T04:02:41.207Z [INFO] server-7e88a31d-d446-6dd1-9777-ed0ddd3b638b: snapshot restore progress: id=2-14-1761537761205 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:41.207Z [INFO] server-7e88a31d-d446-6dd1-9777-ed0ddd3b638b: restored user snapshot: index=14 -2025-10-27T04:02:41.207Z [ERROR] server-7e88a31d-d446-6dd1-9777-ed0ddd3b638b: failed to get log: index=3 error="log not found" -2025-10-27T04:02:41.207Z [INFO] server-7e88a31d-d446-6dd1-9777-ed0ddd3b638b: aborting pipeline replication: peer="{Voter server-728fe845-0745-e242-ae8f-3ccbdfb5ebe4 728fe845-0745-e242-ae8f-3ccbdfb5ebe4}" -2025-10-27T04:02:41.207Z [ERROR] server-7e88a31d-d446-6dd1-9777-ed0ddd3b638b: failed to get log: index=3 error="log not found" -2025-10-27T04:02:41.207Z [INFO] server-7e88a31d-d446-6dd1-9777-ed0ddd3b638b: aborting pipeline replication: peer="{Voter server-08153ebc-1fa3-f020-10d5-f45c5b811d2f 08153ebc-1fa3-f020-10d5-f45c5b811d2f}" -2025-10-27T04:02:41.214Z [ERROR] server-7e88a31d-d446-6dd1-9777-ed0ddd3b638b: failed to get log: index=3 error="log not found" -2025-10-27T04:02:41.214Z [INFO] creating new snapshot: path=/tmp/raft1750307047/snapshots/2-14-1761537761214.tmp -2025-10-27T04:02:41.214Z [ERROR] server-7e88a31d-d446-6dd1-9777-ed0ddd3b638b: failed to get log: index=3 error="log not found" -2025-10-27T04:02:41.215Z [INFO] server-08153ebc-1fa3-f020-10d5-f45c5b811d2f: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:41.215Z [INFO] creating new snapshot: path=/tmp/raft1783835590/snapshots/2-14-1761537761215.tmp -2025-10-27T04:02:41.215Z [INFO] server-08153ebc-1fa3-f020-10d5-f45c5b811d2f: copied to local snapshot: bytes=71 -2025-10-27T04:02:41.215Z [INFO] server-728fe845-0745-e242-ae8f-3ccbdfb5ebe4: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:41.215Z [INFO] server-08153ebc-1fa3-f020-10d5-f45c5b811d2f: snapshot restore progress: id=2-14-1761537761214 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:41.215Z [INFO] server-728fe845-0745-e242-ae8f-3ccbdfb5ebe4: copied to local snapshot: bytes=71 -2025-10-27T04:02:41.215Z [INFO] server-08153ebc-1fa3-f020-10d5-f45c5b811d2f: Installed remote snapshot -2025-10-27T04:02:41.216Z [INFO] server-7e88a31d-d446-6dd1-9777-ed0ddd3b638b: pipelining replication: peer="{Voter server-08153ebc-1fa3-f020-10d5-f45c5b811d2f 08153ebc-1fa3-f020-10d5-f45c5b811d2f}" -2025-10-27T04:02:41.216Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:02:41.216Z [INFO] server-728fe845-0745-e242-ae8f-3ccbdfb5ebe4: snapshot restore progress: id=2-14-1761537761215 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:41.216Z [INFO] server-728fe845-0745-e242-ae8f-3ccbdfb5ebe4: Installed remote snapshot -2025-10-27T04:02:41.216Z [INFO] server-7e88a31d-d446-6dd1-9777-ed0ddd3b638b: pipelining replication: peer="{Voter server-728fe845-0745-e242-ae8f-3ccbdfb5ebe4 728fe845-0745-e242-ae8f-3ccbdfb5ebe4}" -2025-10-27T04:02:42.221Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:02:41.216081813 +0000 UTC m=+36.210954291 (2 nodes), 800ns from start of poll, 1.55535715s from cluster start. Timeout at 2025-10-27 04:02:42.221092453 +0000 UTC m=+37.215964891, 1.0050106s after stability -2025-10-27T04:02:42.233Z [INFO] server-c6fc7443-24b3-6e6e-6bca-c5374a14ea9b: aborting pipeline replication: peer="{Voter server-57acc957-fcea-9d09-1f88-9aed3fa4c11c 57acc957-fcea-9d09-1f88-9aed3fa4c11c}" -2025-10-27T04:02:42.233Z [INFO] server-c6fc7443-24b3-6e6e-6bca-c5374a14ea9b: aborting pipeline replication: peer="{Voter server-09464ff2-b1f6-bd37-40b3-3c6df5d5b8c2 09464ff2-b1f6-bd37-40b3-3c6df5d5b8c2}" +2026-11-29T10:28:41.470Z [INFO] server-ee4253ba-1caa-e00e-af49-3c169c8888db: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ee4253ba-1caa-e00e-af49-3c169c8888db Address:ee4253ba-1caa-e00e-af49-3c169c8888db} {Suffrage:Voter ID:server-0068226d-cc17-a13e-2be2-d379dbcc4dba Address:0068226d-cc17-a13e-2be2-d379dbcc4dba} {Suffrage:Voter ID:server-57d60a43-7a64-8562-0a0b-5e3b38c8b28c Address:57d60a43-7a64-8562-0a0b-5e3b38c8b28c}]" +2026-11-29T10:28:41.470Z [INFO] server-ee4253ba-1caa-e00e-af49-3c169c8888db: entering follower state: follower="Node at ee4253ba-1caa-e00e-af49-3c169c8888db [Follower]" leader-address= leader-id= +2026-11-29T10:28:41.470Z [INFO] server-0068226d-cc17-a13e-2be2-d379dbcc4dba: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ee4253ba-1caa-e00e-af49-3c169c8888db Address:ee4253ba-1caa-e00e-af49-3c169c8888db} {Suffrage:Voter ID:server-0068226d-cc17-a13e-2be2-d379dbcc4dba Address:0068226d-cc17-a13e-2be2-d379dbcc4dba} {Suffrage:Voter ID:server-57d60a43-7a64-8562-0a0b-5e3b38c8b28c Address:57d60a43-7a64-8562-0a0b-5e3b38c8b28c}]" +2026-11-29T10:28:41.471Z [INFO] server-57d60a43-7a64-8562-0a0b-5e3b38c8b28c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ee4253ba-1caa-e00e-af49-3c169c8888db Address:ee4253ba-1caa-e00e-af49-3c169c8888db} {Suffrage:Voter ID:server-0068226d-cc17-a13e-2be2-d379dbcc4dba Address:0068226d-cc17-a13e-2be2-d379dbcc4dba} {Suffrage:Voter ID:server-57d60a43-7a64-8562-0a0b-5e3b38c8b28c Address:57d60a43-7a64-8562-0a0b-5e3b38c8b28c}]" +2026-11-29T10:28:41.471Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:41.471Z [INFO] server-0068226d-cc17-a13e-2be2-d379dbcc4dba: entering follower state: follower="Node at 0068226d-cc17-a13e-2be2-d379dbcc4dba [Follower]" leader-address= leader-id= +2026-11-29T10:28:41.471Z [INFO] server-57d60a43-7a64-8562-0a0b-5e3b38c8b28c: entering follower state: follower="Node at 57d60a43-7a64-8562-0a0b-5e3b38c8b28c [Follower]" leader-address= leader-id= +2026-11-29T10:28:42.068Z [WARN] server-ee4253ba-1caa-e00e-af49-3c169c8888db: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:42.068Z [INFO] server-ee4253ba-1caa-e00e-af49-3c169c8888db: entering candidate state: node="Node at ee4253ba-1caa-e00e-af49-3c169c8888db [Candidate]" term=2 +2026-11-29T10:28:42.068Z [INFO] server-ee4253ba-1caa-e00e-af49-3c169c8888db: election won: term=2 tally=2 +2026-11-29T10:28:42.068Z [INFO] server-ee4253ba-1caa-e00e-af49-3c169c8888db: entering leader state: leader="Node at ee4253ba-1caa-e00e-af49-3c169c8888db [Leader]" +2026-11-29T10:28:42.068Z [INFO] server-ee4253ba-1caa-e00e-af49-3c169c8888db: added peer, starting replication: peer=server-0068226d-cc17-a13e-2be2-d379dbcc4dba +2026-11-29T10:28:42.068Z [INFO] server-ee4253ba-1caa-e00e-af49-3c169c8888db: added peer, starting replication: peer=server-57d60a43-7a64-8562-0a0b-5e3b38c8b28c +2026-11-29T10:28:42.068Z [INFO] server-ee4253ba-1caa-e00e-af49-3c169c8888db: pipelining replication: peer="{Voter server-57d60a43-7a64-8562-0a0b-5e3b38c8b28c 57d60a43-7a64-8562-0a0b-5e3b38c8b28c}" +2026-11-29T10:28:42.068Z [INFO] server-ee4253ba-1caa-e00e-af49-3c169c8888db: pipelining replication: peer="{Voter server-0068226d-cc17-a13e-2be2-d379dbcc4dba 0068226d-cc17-a13e-2be2-d379dbcc4dba}" +2026-11-29T10:28:43.074Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:42.068733099 +0000 UTC m=+32.803787241 (1 nodes), 597.66288ms from start of poll, 598.07528ms from cluster start. Timeout at 2026-11-29 10:28:43.073736019 +0000 UTC m=+33.808790121, 1.00500288s after stability +2026-11-29T10:28:43.074Z [INFO] server-ee4253ba-1caa-e00e-af49-3c169c8888db: starting snapshot up to: index=12 +2026-11-29T10:28:43.074Z [INFO] creating new snapshot: path=/tmp/raft2388932693/snapshots/2-12-1795948123074.tmp +2026-11-29T10:28:43.074Z [INFO] server-ee4253ba-1caa-e00e-af49-3c169c8888db: snapshot complete up to: index=12 +2026-11-29T10:28:43.075Z [INFO] server-ea67aeca-5693-52e1-9b6d-7478a40967f0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ea67aeca-5693-52e1-9b6d-7478a40967f0 Address:ea67aeca-5693-52e1-9b6d-7478a40967f0} {Suffrage:Voter ID:server-93b02683-0fea-6f91-1a8d-43090949f199 Address:93b02683-0fea-6f91-1a8d-43090949f199} {Suffrage:Voter ID:server-718478a0-2ec7-8603-13a5-56906166bcb5 Address:718478a0-2ec7-8603-13a5-56906166bcb5}]" +2026-11-29T10:28:43.075Z [INFO] server-93b02683-0fea-6f91-1a8d-43090949f199: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ea67aeca-5693-52e1-9b6d-7478a40967f0 Address:ea67aeca-5693-52e1-9b6d-7478a40967f0} {Suffrage:Voter ID:server-93b02683-0fea-6f91-1a8d-43090949f199 Address:93b02683-0fea-6f91-1a8d-43090949f199} {Suffrage:Voter ID:server-718478a0-2ec7-8603-13a5-56906166bcb5 Address:718478a0-2ec7-8603-13a5-56906166bcb5}]" +2026-11-29T10:28:43.075Z [INFO] server-718478a0-2ec7-8603-13a5-56906166bcb5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ea67aeca-5693-52e1-9b6d-7478a40967f0 Address:ea67aeca-5693-52e1-9b6d-7478a40967f0} {Suffrage:Voter ID:server-93b02683-0fea-6f91-1a8d-43090949f199 Address:93b02683-0fea-6f91-1a8d-43090949f199} {Suffrage:Voter ID:server-718478a0-2ec7-8603-13a5-56906166bcb5 Address:718478a0-2ec7-8603-13a5-56906166bcb5}]" +2026-11-29T10:28:43.075Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:43.075Z [INFO] server-ea67aeca-5693-52e1-9b6d-7478a40967f0: entering follower state: follower="Node at ea67aeca-5693-52e1-9b6d-7478a40967f0 [Follower]" leader-address= leader-id= +2026-11-29T10:28:43.075Z [INFO] server-718478a0-2ec7-8603-13a5-56906166bcb5: entering follower state: follower="Node at 718478a0-2ec7-8603-13a5-56906166bcb5 [Follower]" leader-address= leader-id= +2026-11-29T10:28:43.075Z [INFO] server-93b02683-0fea-6f91-1a8d-43090949f199: entering follower state: follower="Node at 93b02683-0fea-6f91-1a8d-43090949f199 [Follower]" leader-address= leader-id= +2026-11-29T10:28:43.641Z [WARN] server-ea67aeca-5693-52e1-9b6d-7478a40967f0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:43.641Z [INFO] server-ea67aeca-5693-52e1-9b6d-7478a40967f0: entering candidate state: node="Node at ea67aeca-5693-52e1-9b6d-7478a40967f0 [Candidate]" term=2 +2026-11-29T10:28:43.641Z [INFO] server-ea67aeca-5693-52e1-9b6d-7478a40967f0: election won: term=2 tally=2 +2026-11-29T10:28:43.641Z [INFO] server-ea67aeca-5693-52e1-9b6d-7478a40967f0: entering leader state: leader="Node at ea67aeca-5693-52e1-9b6d-7478a40967f0 [Leader]" +2026-11-29T10:28:43.641Z [INFO] server-ea67aeca-5693-52e1-9b6d-7478a40967f0: added peer, starting replication: peer=server-93b02683-0fea-6f91-1a8d-43090949f199 +2026-11-29T10:28:43.641Z [INFO] server-ea67aeca-5693-52e1-9b6d-7478a40967f0: added peer, starting replication: peer=server-718478a0-2ec7-8603-13a5-56906166bcb5 +2026-11-29T10:28:43.641Z [INFO] server-ea67aeca-5693-52e1-9b6d-7478a40967f0: pipelining replication: peer="{Voter server-93b02683-0fea-6f91-1a8d-43090949f199 93b02683-0fea-6f91-1a8d-43090949f199}" +2026-11-29T10:28:43.641Z [INFO] server-ea67aeca-5693-52e1-9b6d-7478a40967f0: pipelining replication: peer="{Voter server-718478a0-2ec7-8603-13a5-56906166bcb5 718478a0-2ec7-8603-13a5-56906166bcb5}" +2026-11-29T10:28:44.646Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:43.641345179 +0000 UTC m=+34.376399281 (1 nodes), 565.85524ms from start of poll, 566.04476ms from cluster start. Timeout at 2026-11-29 10:28:44.646351939 +0000 UTC m=+35.381406041, 1.00500676s after stability +2026-11-29T10:28:44.647Z [INFO] creating new snapshot: path=/tmp/raft4118424541/snapshots/2-14-1795948124647.tmp +2026-11-29T10:28:44.647Z [INFO] server-ea67aeca-5693-52e1-9b6d-7478a40967f0: copied to local snapshot: bytes=71 +2026-11-29T10:28:44.647Z [INFO] server-ea67aeca-5693-52e1-9b6d-7478a40967f0: snapshot restore progress: id=2-14-1795948124647 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:44.647Z [INFO] server-ea67aeca-5693-52e1-9b6d-7478a40967f0: restored user snapshot: index=14 +2026-11-29T10:28:44.647Z [ERROR] server-ea67aeca-5693-52e1-9b6d-7478a40967f0: failed to get log: index=3 error="log not found" +2026-11-29T10:28:44.647Z [INFO] server-ea67aeca-5693-52e1-9b6d-7478a40967f0: aborting pipeline replication: peer="{Voter server-718478a0-2ec7-8603-13a5-56906166bcb5 718478a0-2ec7-8603-13a5-56906166bcb5}" +2026-11-29T10:28:44.647Z [ERROR] server-ea67aeca-5693-52e1-9b6d-7478a40967f0: failed to get log: index=3 error="log not found" +2026-11-29T10:28:44.647Z [INFO] server-ea67aeca-5693-52e1-9b6d-7478a40967f0: aborting pipeline replication: peer="{Voter server-93b02683-0fea-6f91-1a8d-43090949f199 93b02683-0fea-6f91-1a8d-43090949f199}" +2026-11-29T10:28:44.654Z [ERROR] server-ea67aeca-5693-52e1-9b6d-7478a40967f0: failed to get log: index=3 error="log not found" +2026-11-29T10:28:44.654Z [INFO] creating new snapshot: path=/tmp/raft2780085675/snapshots/2-14-1795948124654.tmp +2026-11-29T10:28:44.655Z [INFO] server-718478a0-2ec7-8603-13a5-56906166bcb5: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:44.655Z [INFO] server-718478a0-2ec7-8603-13a5-56906166bcb5: copied to local snapshot: bytes=71 +2026-11-29T10:28:44.655Z [INFO] server-718478a0-2ec7-8603-13a5-56906166bcb5: snapshot restore progress: id=2-14-1795948124654 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:44.655Z [INFO] server-718478a0-2ec7-8603-13a5-56906166bcb5: Installed remote snapshot +2026-11-29T10:28:44.655Z [INFO] server-ea67aeca-5693-52e1-9b6d-7478a40967f0: pipelining replication: peer="{Voter server-718478a0-2ec7-8603-13a5-56906166bcb5 718478a0-2ec7-8603-13a5-56906166bcb5}" +2026-11-29T10:28:44.655Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:28:44.656Z [ERROR] server-ea67aeca-5693-52e1-9b6d-7478a40967f0: failed to get log: index=3 error="log not found" +2026-11-29T10:28:44.657Z [INFO] creating new snapshot: path=/tmp/raft892032171/snapshots/2-14-1795948124657.tmp +2026-11-29T10:28:44.657Z [INFO] server-93b02683-0fea-6f91-1a8d-43090949f199: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:44.657Z [INFO] server-93b02683-0fea-6f91-1a8d-43090949f199: copied to local snapshot: bytes=71 +2026-11-29T10:28:44.657Z [INFO] server-93b02683-0fea-6f91-1a8d-43090949f199: snapshot restore progress: id=2-14-1795948124657 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:44.657Z [INFO] server-93b02683-0fea-6f91-1a8d-43090949f199: Installed remote snapshot +2026-11-29T10:28:44.657Z [INFO] server-ea67aeca-5693-52e1-9b6d-7478a40967f0: pipelining replication: peer="{Voter server-93b02683-0fea-6f91-1a8d-43090949f199 93b02683-0fea-6f91-1a8d-43090949f199}" +2026-11-29T10:28:45.661Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:28:44.655457539 +0000 UTC m=+35.390511641 (2 nodes), 560ns from start of poll, 1.58015712s from cluster start. Timeout at 2026-11-29 10:28:45.660465099 +0000 UTC m=+36.395519201, 1.00500756s after stability +2026-11-29T10:28:45.672Z [INFO] server-ee4253ba-1caa-e00e-af49-3c169c8888db: aborting pipeline replication: peer="{Voter server-57d60a43-7a64-8562-0a0b-5e3b38c8b28c 57d60a43-7a64-8562-0a0b-5e3b38c8b28c}" +2026-11-29T10:28:45.673Z [INFO] server-ee4253ba-1caa-e00e-af49-3c169c8888db: aborting pipeline replication: peer="{Voter server-0068226d-cc17-a13e-2be2-d379dbcc4dba 0068226d-cc17-a13e-2be2-d379dbcc4dba}" === RUN TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_true -2025-10-27T04:02:42.235Z [INFO] server-541d484e-f435-acce-5d63-60d01e2d71d5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-541d484e-f435-acce-5d63-60d01e2d71d5 Address:541d484e-f435-acce-5d63-60d01e2d71d5} {Suffrage:Voter ID:server-dad7f2d0-a9fc-c51d-e196-4a4b560af15d Address:dad7f2d0-a9fc-c51d-e196-4a4b560af15d} {Suffrage:Voter ID:server-7d054e77-9d45-7ca6-4f7b-c90cc7b43350 Address:7d054e77-9d45-7ca6-4f7b-c90cc7b43350}]" -2025-10-27T04:02:42.235Z [INFO] server-541d484e-f435-acce-5d63-60d01e2d71d5: entering follower state: follower="Node at 541d484e-f435-acce-5d63-60d01e2d71d5 [Follower]" leader-address= leader-id= -2025-10-27T04:02:42.235Z [INFO] server-dad7f2d0-a9fc-c51d-e196-4a4b560af15d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-541d484e-f435-acce-5d63-60d01e2d71d5 Address:541d484e-f435-acce-5d63-60d01e2d71d5} {Suffrage:Voter ID:server-dad7f2d0-a9fc-c51d-e196-4a4b560af15d Address:dad7f2d0-a9fc-c51d-e196-4a4b560af15d} {Suffrage:Voter ID:server-7d054e77-9d45-7ca6-4f7b-c90cc7b43350 Address:7d054e77-9d45-7ca6-4f7b-c90cc7b43350}]" -2025-10-27T04:02:42.235Z [INFO] server-dad7f2d0-a9fc-c51d-e196-4a4b560af15d: entering follower state: follower="Node at dad7f2d0-a9fc-c51d-e196-4a4b560af15d [Follower]" leader-address= leader-id= -2025-10-27T04:02:42.235Z [INFO] server-7d054e77-9d45-7ca6-4f7b-c90cc7b43350: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-541d484e-f435-acce-5d63-60d01e2d71d5 Address:541d484e-f435-acce-5d63-60d01e2d71d5} {Suffrage:Voter ID:server-dad7f2d0-a9fc-c51d-e196-4a4b560af15d Address:dad7f2d0-a9fc-c51d-e196-4a4b560af15d} {Suffrage:Voter ID:server-7d054e77-9d45-7ca6-4f7b-c90cc7b43350 Address:7d054e77-9d45-7ca6-4f7b-c90cc7b43350}]" -2025-10-27T04:02:42.235Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:42.235Z [INFO] server-7d054e77-9d45-7ca6-4f7b-c90cc7b43350: entering follower state: follower="Node at 7d054e77-9d45-7ca6-4f7b-c90cc7b43350 [Follower]" leader-address= leader-id= -2025-10-27T04:02:42.852Z [WARN] server-541d484e-f435-acce-5d63-60d01e2d71d5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:42.852Z [INFO] server-541d484e-f435-acce-5d63-60d01e2d71d5: entering candidate state: node="Node at 541d484e-f435-acce-5d63-60d01e2d71d5 [Candidate]" term=2 -2025-10-27T04:02:42.853Z [INFO] server-541d484e-f435-acce-5d63-60d01e2d71d5: election won: term=2 tally=2 -2025-10-27T04:02:42.853Z [INFO] server-541d484e-f435-acce-5d63-60d01e2d71d5: entering leader state: leader="Node at 541d484e-f435-acce-5d63-60d01e2d71d5 [Leader]" -2025-10-27T04:02:42.853Z [INFO] server-541d484e-f435-acce-5d63-60d01e2d71d5: added peer, starting replication: peer=server-dad7f2d0-a9fc-c51d-e196-4a4b560af15d -2025-10-27T04:02:42.853Z [INFO] server-541d484e-f435-acce-5d63-60d01e2d71d5: added peer, starting replication: peer=server-7d054e77-9d45-7ca6-4f7b-c90cc7b43350 -2025-10-27T04:02:42.853Z [INFO] server-541d484e-f435-acce-5d63-60d01e2d71d5: pipelining replication: peer="{Voter server-7d054e77-9d45-7ca6-4f7b-c90cc7b43350 7d054e77-9d45-7ca6-4f7b-c90cc7b43350}" -2025-10-27T04:02:42.853Z [INFO] server-541d484e-f435-acce-5d63-60d01e2d71d5: pipelining replication: peer="{Voter server-dad7f2d0-a9fc-c51d-e196-4a4b560af15d dad7f2d0-a9fc-c51d-e196-4a4b560af15d}" -2025-10-27T04:02:43.858Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:42.85312096 +0000 UTC m=+37.847993438 (1 nodes), 617.457739ms from start of poll, 618.404106ms from cluster start. Timeout at 2025-10-27 04:02:43.85812536 +0000 UTC m=+38.852997798, 1.00500436s after stability -2025-10-27T04:02:43.858Z [INFO] server-541d484e-f435-acce-5d63-60d01e2d71d5: starting snapshot up to: index=12 -2025-10-27T04:02:43.858Z [INFO] creating new snapshot: path=/tmp/raft501183767/snapshots/2-12-1761537763858.tmp -2025-10-27T04:02:43.859Z [INFO] server-541d484e-f435-acce-5d63-60d01e2d71d5: snapshot complete up to: index=12 -2025-10-27T04:02:43.859Z [INFO] server-91a80850-0c05-c7ae-4011-657d7185ae98: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-91a80850-0c05-c7ae-4011-657d7185ae98 Address:91a80850-0c05-c7ae-4011-657d7185ae98} {Suffrage:Voter ID:server-d5f9d4dc-681e-03d5-e704-ca558df89241 Address:d5f9d4dc-681e-03d5-e704-ca558df89241} {Suffrage:Voter ID:server-af299ba7-8261-44ba-adc7-570402193c1f Address:af299ba7-8261-44ba-adc7-570402193c1f}]" -2025-10-27T04:02:43.859Z [INFO] server-d5f9d4dc-681e-03d5-e704-ca558df89241: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-91a80850-0c05-c7ae-4011-657d7185ae98 Address:91a80850-0c05-c7ae-4011-657d7185ae98} {Suffrage:Voter ID:server-d5f9d4dc-681e-03d5-e704-ca558df89241 Address:d5f9d4dc-681e-03d5-e704-ca558df89241} {Suffrage:Voter ID:server-af299ba7-8261-44ba-adc7-570402193c1f Address:af299ba7-8261-44ba-adc7-570402193c1f}]" -2025-10-27T04:02:43.859Z [INFO] server-91a80850-0c05-c7ae-4011-657d7185ae98: entering follower state: follower="Node at 91a80850-0c05-c7ae-4011-657d7185ae98 [Follower]" leader-address= leader-id= -2025-10-27T04:02:43.859Z [INFO] server-af299ba7-8261-44ba-adc7-570402193c1f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-91a80850-0c05-c7ae-4011-657d7185ae98 Address:91a80850-0c05-c7ae-4011-657d7185ae98} {Suffrage:Voter ID:server-d5f9d4dc-681e-03d5-e704-ca558df89241 Address:d5f9d4dc-681e-03d5-e704-ca558df89241} {Suffrage:Voter ID:server-af299ba7-8261-44ba-adc7-570402193c1f Address:af299ba7-8261-44ba-adc7-570402193c1f}]" -2025-10-27T04:02:43.859Z [INFO] server-d5f9d4dc-681e-03d5-e704-ca558df89241: entering follower state: follower="Node at d5f9d4dc-681e-03d5-e704-ca558df89241 [Follower]" leader-address= leader-id= -2025-10-27T04:02:43.859Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:43.859Z [INFO] server-af299ba7-8261-44ba-adc7-570402193c1f: entering follower state: follower="Node at af299ba7-8261-44ba-adc7-570402193c1f [Follower]" leader-address= leader-id= -2025-10-27T04:02:44.379Z [WARN] server-af299ba7-8261-44ba-adc7-570402193c1f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:44.379Z [INFO] server-af299ba7-8261-44ba-adc7-570402193c1f: entering candidate state: node="Node at af299ba7-8261-44ba-adc7-570402193c1f [Candidate]" term=2 -2025-10-27T04:02:44.379Z [INFO] server-af299ba7-8261-44ba-adc7-570402193c1f: election won: term=2 tally=2 -2025-10-27T04:02:44.379Z [INFO] server-af299ba7-8261-44ba-adc7-570402193c1f: entering leader state: leader="Node at af299ba7-8261-44ba-adc7-570402193c1f [Leader]" -2025-10-27T04:02:44.379Z [INFO] server-af299ba7-8261-44ba-adc7-570402193c1f: added peer, starting replication: peer=server-91a80850-0c05-c7ae-4011-657d7185ae98 -2025-10-27T04:02:44.379Z [INFO] server-af299ba7-8261-44ba-adc7-570402193c1f: added peer, starting replication: peer=server-d5f9d4dc-681e-03d5-e704-ca558df89241 -2025-10-27T04:02:44.379Z [INFO] server-af299ba7-8261-44ba-adc7-570402193c1f: pipelining replication: peer="{Voter server-d5f9d4dc-681e-03d5-e704-ca558df89241 d5f9d4dc-681e-03d5-e704-ca558df89241}" -2025-10-27T04:02:44.379Z [INFO] server-af299ba7-8261-44ba-adc7-570402193c1f: pipelining replication: peer="{Voter server-91a80850-0c05-c7ae-4011-657d7185ae98 91a80850-0c05-c7ae-4011-657d7185ae98}" -2025-10-27T04:02:45.384Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:44.379449631 +0000 UTC m=+39.374322109 (1 nodes), 519.605271ms from start of poll, 519.796872ms from cluster start. Timeout at 2025-10-27 04:02:45.384452951 +0000 UTC m=+40.379325429, 1.00500332s after stability -2025-10-27T04:02:45.384Z [INFO] creating new snapshot: path=/tmp/raft2862714389/snapshots/2-14-1761537765384.tmp -2025-10-27T04:02:45.385Z [INFO] server-af299ba7-8261-44ba-adc7-570402193c1f: copied to local snapshot: bytes=71 -2025-10-27T04:02:45.386Z [INFO] server-af299ba7-8261-44ba-adc7-570402193c1f: snapshot restore progress: id=2-14-1761537765384 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:45.386Z [INFO] server-af299ba7-8261-44ba-adc7-570402193c1f: removing all old logs from log store -2025-10-27T04:02:45.386Z [INFO] server-af299ba7-8261-44ba-adc7-570402193c1f: compacting logs: from=1 to=2 -2025-10-27T04:02:45.386Z [INFO] server-af299ba7-8261-44ba-adc7-570402193c1f: restored user snapshot: index=14 -2025-10-27T04:02:45.386Z [ERROR] server-af299ba7-8261-44ba-adc7-570402193c1f: failed to get log: index=2 error="log not found" -2025-10-27T04:02:45.386Z [ERROR] server-af299ba7-8261-44ba-adc7-570402193c1f: failed to get log: index=2 error="log not found" -2025-10-27T04:02:45.386Z [INFO] server-af299ba7-8261-44ba-adc7-570402193c1f: aborting pipeline replication: peer="{Voter server-91a80850-0c05-c7ae-4011-657d7185ae98 91a80850-0c05-c7ae-4011-657d7185ae98}" -2025-10-27T04:02:45.386Z [INFO] server-af299ba7-8261-44ba-adc7-570402193c1f: aborting pipeline replication: peer="{Voter server-d5f9d4dc-681e-03d5-e704-ca558df89241 d5f9d4dc-681e-03d5-e704-ca558df89241}" -2025-10-27T04:02:45.393Z [ERROR] server-af299ba7-8261-44ba-adc7-570402193c1f: failed to get log: index=2 error="log not found" -2025-10-27T04:02:45.394Z [INFO] creating new snapshot: path=/tmp/raft342321313/snapshots/2-14-1761537765394.tmp -2025-10-27T04:02:45.394Z [INFO] server-91a80850-0c05-c7ae-4011-657d7185ae98: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:45.394Z [INFO] server-91a80850-0c05-c7ae-4011-657d7185ae98: copied to local snapshot: bytes=71 -2025-10-27T04:02:45.394Z [INFO] server-91a80850-0c05-c7ae-4011-657d7185ae98: snapshot restore progress: id=2-14-1761537765394 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:45.394Z [INFO] server-91a80850-0c05-c7ae-4011-657d7185ae98: removing all old logs from log store -2025-10-27T04:02:45.394Z [INFO] server-91a80850-0c05-c7ae-4011-657d7185ae98: compacting logs: from=1 to=2 -2025-10-27T04:02:45.394Z [INFO] server-91a80850-0c05-c7ae-4011-657d7185ae98: Installed remote snapshot -2025-10-27T04:02:45.395Z [INFO] server-af299ba7-8261-44ba-adc7-570402193c1f: pipelining replication: peer="{Voter server-91a80850-0c05-c7ae-4011-657d7185ae98 91a80850-0c05-c7ae-4011-657d7185ae98}" -2025-10-27T04:02:45.395Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:02:45.395Z [ERROR] server-af299ba7-8261-44ba-adc7-570402193c1f: failed to get log: index=2 error="log not found" -2025-10-27T04:02:45.396Z [INFO] creating new snapshot: path=/tmp/raft3792134540/snapshots/2-14-1761537765396.tmp -2025-10-27T04:02:45.396Z [INFO] server-d5f9d4dc-681e-03d5-e704-ca558df89241: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:45.396Z [INFO] server-d5f9d4dc-681e-03d5-e704-ca558df89241: copied to local snapshot: bytes=71 -2025-10-27T04:02:45.396Z [INFO] server-d5f9d4dc-681e-03d5-e704-ca558df89241: snapshot restore progress: id=2-14-1761537765396 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:45.396Z [INFO] server-d5f9d4dc-681e-03d5-e704-ca558df89241: removing all old logs from log store -2025-10-27T04:02:45.396Z [INFO] server-d5f9d4dc-681e-03d5-e704-ca558df89241: compacting logs: from=1 to=2 -2025-10-27T04:02:45.397Z [INFO] server-d5f9d4dc-681e-03d5-e704-ca558df89241: Installed remote snapshot -2025-10-27T04:02:45.397Z [INFO] server-af299ba7-8261-44ba-adc7-570402193c1f: pipelining replication: peer="{Voter server-d5f9d4dc-681e-03d5-e704-ca558df89241 d5f9d4dc-681e-03d5-e704-ca558df89241}" -2025-10-27T04:02:46.400Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:02:45.395146292 +0000 UTC m=+40.390018770 (2 nodes), 1.2µs from start of poll, 1.535493533s from cluster start. Timeout at 2025-10-27 04:02:46.400167453 +0000 UTC m=+41.395039931, 1.005021161s after stability -2025-10-27T04:02:46.412Z [INFO] server-541d484e-f435-acce-5d63-60d01e2d71d5: aborting pipeline replication: peer="{Voter server-dad7f2d0-a9fc-c51d-e196-4a4b560af15d dad7f2d0-a9fc-c51d-e196-4a4b560af15d}" -2025-10-27T04:02:46.412Z [INFO] server-541d484e-f435-acce-5d63-60d01e2d71d5: aborting pipeline replication: peer="{Voter server-7d054e77-9d45-7ca6-4f7b-c90cc7b43350 7d054e77-9d45-7ca6-4f7b-c90cc7b43350}" +2026-11-29T10:28:45.673Z [INFO] server-702f16bc-8b42-be9f-0ff5-d34727735a55: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-702f16bc-8b42-be9f-0ff5-d34727735a55 Address:702f16bc-8b42-be9f-0ff5-d34727735a55} {Suffrage:Voter ID:server-ea614023-4e76-4181-16f1-e9748dce9384 Address:ea614023-4e76-4181-16f1-e9748dce9384} {Suffrage:Voter ID:server-6ab90e96-5cef-8ea3-2c4b-00ceb3696c72 Address:6ab90e96-5cef-8ea3-2c4b-00ceb3696c72}]" +2026-11-29T10:28:45.673Z [INFO] server-ea614023-4e76-4181-16f1-e9748dce9384: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-702f16bc-8b42-be9f-0ff5-d34727735a55 Address:702f16bc-8b42-be9f-0ff5-d34727735a55} {Suffrage:Voter ID:server-ea614023-4e76-4181-16f1-e9748dce9384 Address:ea614023-4e76-4181-16f1-e9748dce9384} {Suffrage:Voter ID:server-6ab90e96-5cef-8ea3-2c4b-00ceb3696c72 Address:6ab90e96-5cef-8ea3-2c4b-00ceb3696c72}]" +2026-11-29T10:28:45.673Z [INFO] server-6ab90e96-5cef-8ea3-2c4b-00ceb3696c72: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-702f16bc-8b42-be9f-0ff5-d34727735a55 Address:702f16bc-8b42-be9f-0ff5-d34727735a55} {Suffrage:Voter ID:server-ea614023-4e76-4181-16f1-e9748dce9384 Address:ea614023-4e76-4181-16f1-e9748dce9384} {Suffrage:Voter ID:server-6ab90e96-5cef-8ea3-2c4b-00ceb3696c72 Address:6ab90e96-5cef-8ea3-2c4b-00ceb3696c72}]" +2026-11-29T10:28:45.673Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:45.673Z [INFO] server-702f16bc-8b42-be9f-0ff5-d34727735a55: entering follower state: follower="Node at 702f16bc-8b42-be9f-0ff5-d34727735a55 [Follower]" leader-address= leader-id= +2026-11-29T10:28:45.673Z [INFO] server-ea614023-4e76-4181-16f1-e9748dce9384: entering follower state: follower="Node at ea614023-4e76-4181-16f1-e9748dce9384 [Follower]" leader-address= leader-id= +2026-11-29T10:28:45.673Z [INFO] server-6ab90e96-5cef-8ea3-2c4b-00ceb3696c72: entering follower state: follower="Node at 6ab90e96-5cef-8ea3-2c4b-00ceb3696c72 [Follower]" leader-address= leader-id= +2026-11-29T10:28:46.524Z [WARN] server-702f16bc-8b42-be9f-0ff5-d34727735a55: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:46.524Z [INFO] server-702f16bc-8b42-be9f-0ff5-d34727735a55: entering candidate state: node="Node at 702f16bc-8b42-be9f-0ff5-d34727735a55 [Candidate]" term=2 +2026-11-29T10:28:46.524Z [INFO] server-702f16bc-8b42-be9f-0ff5-d34727735a55: election won: term=2 tally=2 +2026-11-29T10:28:46.524Z [INFO] server-702f16bc-8b42-be9f-0ff5-d34727735a55: entering leader state: leader="Node at 702f16bc-8b42-be9f-0ff5-d34727735a55 [Leader]" +2026-11-29T10:28:46.524Z [INFO] server-702f16bc-8b42-be9f-0ff5-d34727735a55: added peer, starting replication: peer=server-ea614023-4e76-4181-16f1-e9748dce9384 +2026-11-29T10:28:46.524Z [INFO] server-702f16bc-8b42-be9f-0ff5-d34727735a55: added peer, starting replication: peer=server-6ab90e96-5cef-8ea3-2c4b-00ceb3696c72 +2026-11-29T10:28:46.524Z [INFO] server-702f16bc-8b42-be9f-0ff5-d34727735a55: pipelining replication: peer="{Voter server-6ab90e96-5cef-8ea3-2c4b-00ceb3696c72 6ab90e96-5cef-8ea3-2c4b-00ceb3696c72}" +2026-11-29T10:28:46.524Z [INFO] server-702f16bc-8b42-be9f-0ff5-d34727735a55: pipelining replication: peer="{Voter server-ea614023-4e76-4181-16f1-e9748dce9384 ea614023-4e76-4181-16f1-e9748dce9384}" +2026-11-29T10:28:47.529Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:46.524592539 +0000 UTC m=+37.259646601 (1 nodes), 851.03512ms from start of poll, 851.19508ms from cluster start. Timeout at 2026-11-29 10:28:47.529597819 +0000 UTC m=+38.264651921, 1.00500532s after stability +2026-11-29T10:28:47.530Z [INFO] server-702f16bc-8b42-be9f-0ff5-d34727735a55: starting snapshot up to: index=12 +2026-11-29T10:28:47.530Z [INFO] creating new snapshot: path=/tmp/raft1548844661/snapshots/2-12-1795948127530.tmp +2026-11-29T10:28:47.530Z [INFO] server-702f16bc-8b42-be9f-0ff5-d34727735a55: snapshot complete up to: index=12 +2026-11-29T10:28:47.530Z [INFO] server-6d5bb4d1-c30f-4655-6633-86d3f4d93a81: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6d5bb4d1-c30f-4655-6633-86d3f4d93a81 Address:6d5bb4d1-c30f-4655-6633-86d3f4d93a81} {Suffrage:Voter ID:server-fbcb15e1-7ea4-a649-94d2-3cced1f70415 Address:fbcb15e1-7ea4-a649-94d2-3cced1f70415} {Suffrage:Voter ID:server-d56c3fd1-2942-403d-448e-f023200d84dd Address:d56c3fd1-2942-403d-448e-f023200d84dd}]" +2026-11-29T10:28:47.530Z [INFO] server-fbcb15e1-7ea4-a649-94d2-3cced1f70415: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6d5bb4d1-c30f-4655-6633-86d3f4d93a81 Address:6d5bb4d1-c30f-4655-6633-86d3f4d93a81} {Suffrage:Voter ID:server-fbcb15e1-7ea4-a649-94d2-3cced1f70415 Address:fbcb15e1-7ea4-a649-94d2-3cced1f70415} {Suffrage:Voter ID:server-d56c3fd1-2942-403d-448e-f023200d84dd Address:d56c3fd1-2942-403d-448e-f023200d84dd}]" +2026-11-29T10:28:47.531Z [INFO] server-d56c3fd1-2942-403d-448e-f023200d84dd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6d5bb4d1-c30f-4655-6633-86d3f4d93a81 Address:6d5bb4d1-c30f-4655-6633-86d3f4d93a81} {Suffrage:Voter ID:server-fbcb15e1-7ea4-a649-94d2-3cced1f70415 Address:fbcb15e1-7ea4-a649-94d2-3cced1f70415} {Suffrage:Voter ID:server-d56c3fd1-2942-403d-448e-f023200d84dd Address:d56c3fd1-2942-403d-448e-f023200d84dd}]" +2026-11-29T10:28:47.531Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:47.531Z [INFO] server-6d5bb4d1-c30f-4655-6633-86d3f4d93a81: entering follower state: follower="Node at 6d5bb4d1-c30f-4655-6633-86d3f4d93a81 [Follower]" leader-address= leader-id= +2026-11-29T10:28:47.531Z [INFO] server-fbcb15e1-7ea4-a649-94d2-3cced1f70415: entering follower state: follower="Node at fbcb15e1-7ea4-a649-94d2-3cced1f70415 [Follower]" leader-address= leader-id= +2026-11-29T10:28:47.531Z [INFO] server-d56c3fd1-2942-403d-448e-f023200d84dd: entering follower state: follower="Node at d56c3fd1-2942-403d-448e-f023200d84dd [Follower]" leader-address= leader-id= +2026-11-29T10:28:48.167Z [WARN] server-6d5bb4d1-c30f-4655-6633-86d3f4d93a81: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:48.167Z [INFO] server-6d5bb4d1-c30f-4655-6633-86d3f4d93a81: entering candidate state: node="Node at 6d5bb4d1-c30f-4655-6633-86d3f4d93a81 [Candidate]" term=2 +2026-11-29T10:28:48.167Z [INFO] server-6d5bb4d1-c30f-4655-6633-86d3f4d93a81: election won: term=2 tally=2 +2026-11-29T10:28:48.168Z [INFO] server-6d5bb4d1-c30f-4655-6633-86d3f4d93a81: entering leader state: leader="Node at 6d5bb4d1-c30f-4655-6633-86d3f4d93a81 [Leader]" +2026-11-29T10:28:48.168Z [INFO] server-6d5bb4d1-c30f-4655-6633-86d3f4d93a81: added peer, starting replication: peer=server-fbcb15e1-7ea4-a649-94d2-3cced1f70415 +2026-11-29T10:28:48.168Z [INFO] server-6d5bb4d1-c30f-4655-6633-86d3f4d93a81: added peer, starting replication: peer=server-d56c3fd1-2942-403d-448e-f023200d84dd +2026-11-29T10:28:48.168Z [INFO] server-6d5bb4d1-c30f-4655-6633-86d3f4d93a81: pipelining replication: peer="{Voter server-fbcb15e1-7ea4-a649-94d2-3cced1f70415 fbcb15e1-7ea4-a649-94d2-3cced1f70415}" +2026-11-29T10:28:48.168Z [INFO] server-6d5bb4d1-c30f-4655-6633-86d3f4d93a81: pipelining replication: peer="{Voter server-d56c3fd1-2942-403d-448e-f023200d84dd d56c3fd1-2942-403d-448e-f023200d84dd}" +2026-11-29T10:28:49.173Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:48.168057539 +0000 UTC m=+38.903111721 (1 nodes), 637.04012ms from start of poll, 637.19204ms from cluster start. Timeout at 2026-11-29 10:28:49.173061419 +0000 UTC m=+39.908115521, 1.0050038s after stability +2026-11-29T10:28:49.173Z [INFO] creating new snapshot: path=/tmp/raft3090814924/snapshots/2-14-1795948129173.tmp +2026-11-29T10:28:49.173Z [INFO] server-6d5bb4d1-c30f-4655-6633-86d3f4d93a81: copied to local snapshot: bytes=71 +2026-11-29T10:28:49.174Z [INFO] server-6d5bb4d1-c30f-4655-6633-86d3f4d93a81: snapshot restore progress: id=2-14-1795948129173 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:49.174Z [INFO] server-6d5bb4d1-c30f-4655-6633-86d3f4d93a81: removing all old logs from log store +2026-11-29T10:28:49.174Z [INFO] server-6d5bb4d1-c30f-4655-6633-86d3f4d93a81: compacting logs: from=1 to=2 +2026-11-29T10:28:49.174Z [INFO] server-6d5bb4d1-c30f-4655-6633-86d3f4d93a81: restored user snapshot: index=14 +2026-11-29T10:28:49.174Z [ERROR] server-6d5bb4d1-c30f-4655-6633-86d3f4d93a81: failed to get log: index=2 error="log not found" +2026-11-29T10:28:49.174Z [INFO] server-6d5bb4d1-c30f-4655-6633-86d3f4d93a81: aborting pipeline replication: peer="{Voter server-d56c3fd1-2942-403d-448e-f023200d84dd d56c3fd1-2942-403d-448e-f023200d84dd}" +2026-11-29T10:28:49.174Z [ERROR] server-6d5bb4d1-c30f-4655-6633-86d3f4d93a81: failed to get log: index=2 error="log not found" +2026-11-29T10:28:49.174Z [INFO] server-6d5bb4d1-c30f-4655-6633-86d3f4d93a81: aborting pipeline replication: peer="{Voter server-fbcb15e1-7ea4-a649-94d2-3cced1f70415 fbcb15e1-7ea4-a649-94d2-3cced1f70415}" +2026-11-29T10:28:49.182Z [ERROR] server-6d5bb4d1-c30f-4655-6633-86d3f4d93a81: failed to get log: index=2 error="log not found" +2026-11-29T10:28:49.182Z [INFO] creating new snapshot: path=/tmp/raft4095018570/snapshots/2-14-1795948129182.tmp +2026-11-29T10:28:49.182Z [INFO] server-d56c3fd1-2942-403d-448e-f023200d84dd: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:49.182Z [INFO] server-d56c3fd1-2942-403d-448e-f023200d84dd: copied to local snapshot: bytes=71 +2026-11-29T10:28:49.183Z [INFO] server-d56c3fd1-2942-403d-448e-f023200d84dd: snapshot restore progress: id=2-14-1795948129182 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:49.183Z [INFO] server-d56c3fd1-2942-403d-448e-f023200d84dd: removing all old logs from log store +2026-11-29T10:28:49.183Z [INFO] server-d56c3fd1-2942-403d-448e-f023200d84dd: compacting logs: from=1 to=2 +2026-11-29T10:28:49.183Z [INFO] server-d56c3fd1-2942-403d-448e-f023200d84dd: Installed remote snapshot +2026-11-29T10:28:49.183Z [INFO] server-6d5bb4d1-c30f-4655-6633-86d3f4d93a81: pipelining replication: peer="{Voter server-d56c3fd1-2942-403d-448e-f023200d84dd d56c3fd1-2942-403d-448e-f023200d84dd}" +2026-11-29T10:28:49.183Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:28:49.184Z [ERROR] server-6d5bb4d1-c30f-4655-6633-86d3f4d93a81: failed to get log: index=2 error="log not found" +2026-11-29T10:28:49.184Z [INFO] creating new snapshot: path=/tmp/raft2022386929/snapshots/2-14-1795948129184.tmp +2026-11-29T10:28:49.184Z [INFO] server-fbcb15e1-7ea4-a649-94d2-3cced1f70415: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:49.185Z [INFO] server-fbcb15e1-7ea4-a649-94d2-3cced1f70415: copied to local snapshot: bytes=71 +2026-11-29T10:28:49.185Z [INFO] server-fbcb15e1-7ea4-a649-94d2-3cced1f70415: snapshot restore progress: id=2-14-1795948129184 last-index=14 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:49.185Z [INFO] server-fbcb15e1-7ea4-a649-94d2-3cced1f70415: removing all old logs from log store +2026-11-29T10:28:49.185Z [INFO] server-fbcb15e1-7ea4-a649-94d2-3cced1f70415: compacting logs: from=1 to=2 +2026-11-29T10:28:49.185Z [INFO] server-fbcb15e1-7ea4-a649-94d2-3cced1f70415: Installed remote snapshot +2026-11-29T10:28:49.185Z [INFO] server-6d5bb4d1-c30f-4655-6633-86d3f4d93a81: pipelining replication: peer="{Voter server-fbcb15e1-7ea4-a649-94d2-3cced1f70415 fbcb15e1-7ea4-a649-94d2-3cced1f70415}" +2026-11-29T10:28:50.188Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:28:49.183127299 +0000 UTC m=+39.918181361 (2 nodes), 480ns from start of poll, 1.65226168s from cluster start. Timeout at 2026-11-29 10:28:50.188135019 +0000 UTC m=+40.923189121, 1.00500776s after stability +2026-11-29T10:28:50.199Z [INFO] server-702f16bc-8b42-be9f-0ff5-d34727735a55: aborting pipeline replication: peer="{Voter server-6ab90e96-5cef-8ea3-2c4b-00ceb3696c72 6ab90e96-5cef-8ea3-2c4b-00ceb3696c72}" +2026-11-29T10:28:50.199Z [INFO] server-702f16bc-8b42-be9f-0ff5-d34727735a55: aborting pipeline replication: peer="{Voter server-ea614023-4e76-4181-16f1-e9748dce9384 ea614023-4e76-4181-16f1-e9748dce9384}" === RUN TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_false -2025-10-27T04:02:46.413Z [INFO] server-3d2f1027-526e-d9cb-816a-2c55f8815963: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3d2f1027-526e-d9cb-816a-2c55f8815963 Address:3d2f1027-526e-d9cb-816a-2c55f8815963} {Suffrage:Voter ID:server-5bd9596b-f42d-5d25-9644-9fe6dc633d0d Address:5bd9596b-f42d-5d25-9644-9fe6dc633d0d} {Suffrage:Voter ID:server-0fedbb35-4b38-db0a-507d-e064b6b104ac Address:0fedbb35-4b38-db0a-507d-e064b6b104ac}]" -2025-10-27T04:02:46.413Z [INFO] server-3d2f1027-526e-d9cb-816a-2c55f8815963: entering follower state: follower="Node at 3d2f1027-526e-d9cb-816a-2c55f8815963 [Follower]" leader-address= leader-id= -2025-10-27T04:02:46.413Z [INFO] server-5bd9596b-f42d-5d25-9644-9fe6dc633d0d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3d2f1027-526e-d9cb-816a-2c55f8815963 Address:3d2f1027-526e-d9cb-816a-2c55f8815963} {Suffrage:Voter ID:server-5bd9596b-f42d-5d25-9644-9fe6dc633d0d Address:5bd9596b-f42d-5d25-9644-9fe6dc633d0d} {Suffrage:Voter ID:server-0fedbb35-4b38-db0a-507d-e064b6b104ac Address:0fedbb35-4b38-db0a-507d-e064b6b104ac}]" -2025-10-27T04:02:46.413Z [INFO] server-5bd9596b-f42d-5d25-9644-9fe6dc633d0d: entering follower state: follower="Node at 5bd9596b-f42d-5d25-9644-9fe6dc633d0d [Follower]" leader-address= leader-id= -2025-10-27T04:02:46.413Z [INFO] server-0fedbb35-4b38-db0a-507d-e064b6b104ac: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3d2f1027-526e-d9cb-816a-2c55f8815963 Address:3d2f1027-526e-d9cb-816a-2c55f8815963} {Suffrage:Voter ID:server-5bd9596b-f42d-5d25-9644-9fe6dc633d0d Address:5bd9596b-f42d-5d25-9644-9fe6dc633d0d} {Suffrage:Voter ID:server-0fedbb35-4b38-db0a-507d-e064b6b104ac Address:0fedbb35-4b38-db0a-507d-e064b6b104ac}]" -2025-10-27T04:02:46.413Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:46.413Z [INFO] server-0fedbb35-4b38-db0a-507d-e064b6b104ac: entering follower state: follower="Node at 0fedbb35-4b38-db0a-507d-e064b6b104ac [Follower]" leader-address= leader-id= -2025-10-27T04:02:46.978Z [WARN] server-3d2f1027-526e-d9cb-816a-2c55f8815963: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:46.978Z [INFO] server-3d2f1027-526e-d9cb-816a-2c55f8815963: entering candidate state: node="Node at 3d2f1027-526e-d9cb-816a-2c55f8815963 [Candidate]" term=2 -2025-10-27T04:02:46.978Z [INFO] server-3d2f1027-526e-d9cb-816a-2c55f8815963: election won: term=2 tally=2 -2025-10-27T04:02:46.978Z [INFO] server-3d2f1027-526e-d9cb-816a-2c55f8815963: entering leader state: leader="Node at 3d2f1027-526e-d9cb-816a-2c55f8815963 [Leader]" -2025-10-27T04:02:46.978Z [INFO] server-3d2f1027-526e-d9cb-816a-2c55f8815963: added peer, starting replication: peer=server-5bd9596b-f42d-5d25-9644-9fe6dc633d0d -2025-10-27T04:02:46.978Z [INFO] server-3d2f1027-526e-d9cb-816a-2c55f8815963: added peer, starting replication: peer=server-0fedbb35-4b38-db0a-507d-e064b6b104ac -2025-10-27T04:02:46.978Z [INFO] server-3d2f1027-526e-d9cb-816a-2c55f8815963: pipelining replication: peer="{Voter server-0fedbb35-4b38-db0a-507d-e064b6b104ac 0fedbb35-4b38-db0a-507d-e064b6b104ac}" -2025-10-27T04:02:46.979Z [INFO] server-3d2f1027-526e-d9cb-816a-2c55f8815963: pipelining replication: peer="{Voter server-5bd9596b-f42d-5d25-9644-9fe6dc633d0d 5bd9596b-f42d-5d25-9644-9fe6dc633d0d}" -2025-10-27T04:02:47.984Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:46.978866995 +0000 UTC m=+41.973739513 (1 nodes), 565.135382ms from start of poll, 565.438784ms from cluster start. Timeout at 2025-10-27 04:02:47.983869915 +0000 UTC m=+42.978742393, 1.00500288s after stability -2025-10-27T04:02:47.984Z [INFO] server-3d2f1027-526e-d9cb-816a-2c55f8815963: starting snapshot up to: index=12 -2025-10-27T04:02:47.984Z [INFO] creating new snapshot: path=/tmp/raft213945212/snapshots/2-12-1761537767984.tmp -2025-10-27T04:02:47.985Z [INFO] server-3d2f1027-526e-d9cb-816a-2c55f8815963: snapshot complete up to: index=12 -2025-10-27T04:02:47.985Z [INFO] creating new snapshot: path=/tmp/raft213945212/snapshots/2-23-1761537767985.tmp -2025-10-27T04:02:47.986Z [INFO] server-3d2f1027-526e-d9cb-816a-2c55f8815963: copied to local snapshot: bytes=71 -2025-10-27T04:02:47.986Z [INFO] server-3d2f1027-526e-d9cb-816a-2c55f8815963: snapshot restore progress: id=2-23-1761537767985 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:47.986Z [INFO] server-3d2f1027-526e-d9cb-816a-2c55f8815963: restored user snapshot: index=23 -2025-10-27T04:02:47.986Z [ERROR] server-3d2f1027-526e-d9cb-816a-2c55f8815963: failed to get log: index=23 error="log not found" -2025-10-27T04:02:47.986Z [INFO] server-3d2f1027-526e-d9cb-816a-2c55f8815963: aborting pipeline replication: peer="{Voter server-0fedbb35-4b38-db0a-507d-e064b6b104ac 0fedbb35-4b38-db0a-507d-e064b6b104ac}" -2025-10-27T04:02:47.986Z [ERROR] server-3d2f1027-526e-d9cb-816a-2c55f8815963: failed to get log: index=23 error="log not found" -2025-10-27T04:02:47.986Z [INFO] server-3d2f1027-526e-d9cb-816a-2c55f8815963: aborting pipeline replication: peer="{Voter server-5bd9596b-f42d-5d25-9644-9fe6dc633d0d 5bd9596b-f42d-5d25-9644-9fe6dc633d0d}" -2025-10-27T04:02:47.994Z [ERROR] server-3d2f1027-526e-d9cb-816a-2c55f8815963: failed to get log: index=23 error="log not found" -2025-10-27T04:02:47.994Z [INFO] creating new snapshot: path=/tmp/raft3469584225/snapshots/2-23-1761537767994.tmp -2025-10-27T04:02:47.994Z [INFO] server-5bd9596b-f42d-5d25-9644-9fe6dc633d0d: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:47.995Z [INFO] server-5bd9596b-f42d-5d25-9644-9fe6dc633d0d: copied to local snapshot: bytes=71 -2025-10-27T04:02:47.995Z [ERROR] server-3d2f1027-526e-d9cb-816a-2c55f8815963: failed to get log: index=23 error="log not found" -2025-10-27T04:02:47.995Z [INFO] creating new snapshot: path=/tmp/raft3490540376/snapshots/2-23-1761537767995.tmp -2025-10-27T04:02:47.995Z [INFO] server-0fedbb35-4b38-db0a-507d-e064b6b104ac: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:47.996Z [INFO] server-0fedbb35-4b38-db0a-507d-e064b6b104ac: copied to local snapshot: bytes=71 -2025-10-27T04:02:47.995Z [INFO] server-5bd9596b-f42d-5d25-9644-9fe6dc633d0d: snapshot restore progress: id=2-23-1761537767994 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:47.996Z [INFO] server-5bd9596b-f42d-5d25-9644-9fe6dc633d0d: Installed remote snapshot -2025-10-27T04:02:47.996Z [INFO] server-0fedbb35-4b38-db0a-507d-e064b6b104ac: snapshot restore progress: id=2-23-1761537767995 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:47.996Z [INFO] server-0fedbb35-4b38-db0a-507d-e064b6b104ac: Installed remote snapshot -2025-10-27T04:02:47.996Z [WARN] server-0fedbb35-4b38-db0a-507d-e064b6b104ac: skipping application of old log: index=22 -2025-10-27T04:02:47.996Z [INFO] server-3d2f1027-526e-d9cb-816a-2c55f8815963: pipelining replication: peer="{Voter server-0fedbb35-4b38-db0a-507d-e064b6b104ac 0fedbb35-4b38-db0a-507d-e064b6b104ac}" -2025-10-27T04:02:47.996Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:02:47.997Z [INFO] server-3d2f1027-526e-d9cb-816a-2c55f8815963: pipelining replication: peer="{Voter server-5bd9596b-f42d-5d25-9644-9fe6dc633d0d 5bd9596b-f42d-5d25-9644-9fe6dc633d0d}" -2025-10-27T04:02:49.001Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:02:47.996716231 +0000 UTC m=+42.991588669 (2 nodes), 1.04µs from start of poll, 1.58328794s from cluster start. Timeout at 2025-10-27 04:02:49.001723791 +0000 UTC m=+43.996596309, 1.00500764s after stability -2025-10-27T04:02:49.013Z [INFO] server-3d2f1027-526e-d9cb-816a-2c55f8815963: aborting pipeline replication: peer="{Voter server-0fedbb35-4b38-db0a-507d-e064b6b104ac 0fedbb35-4b38-db0a-507d-e064b6b104ac}" -2025-10-27T04:02:49.013Z [INFO] server-3d2f1027-526e-d9cb-816a-2c55f8815963: aborting pipeline replication: peer="{Voter server-5bd9596b-f42d-5d25-9644-9fe6dc633d0d 5bd9596b-f42d-5d25-9644-9fe6dc633d0d}" +2026-11-29T10:28:50.199Z [INFO] server-ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f Address:ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f} {Suffrage:Voter ID:server-e0b5f1ae-f4a0-714e-931c-2daf5257036e Address:e0b5f1ae-f4a0-714e-931c-2daf5257036e} {Suffrage:Voter ID:server-963100ce-19f4-306c-3e9f-dbe1ee31d1c1 Address:963100ce-19f4-306c-3e9f-dbe1ee31d1c1}]" +2026-11-29T10:28:50.199Z [INFO] server-e0b5f1ae-f4a0-714e-931c-2daf5257036e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f Address:ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f} {Suffrage:Voter ID:server-e0b5f1ae-f4a0-714e-931c-2daf5257036e Address:e0b5f1ae-f4a0-714e-931c-2daf5257036e} {Suffrage:Voter ID:server-963100ce-19f4-306c-3e9f-dbe1ee31d1c1 Address:963100ce-19f4-306c-3e9f-dbe1ee31d1c1}]" +2026-11-29T10:28:50.200Z [INFO] server-963100ce-19f4-306c-3e9f-dbe1ee31d1c1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f Address:ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f} {Suffrage:Voter ID:server-e0b5f1ae-f4a0-714e-931c-2daf5257036e Address:e0b5f1ae-f4a0-714e-931c-2daf5257036e} {Suffrage:Voter ID:server-963100ce-19f4-306c-3e9f-dbe1ee31d1c1 Address:963100ce-19f4-306c-3e9f-dbe1ee31d1c1}]" +2026-11-29T10:28:50.200Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:50.200Z [INFO] server-ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f: entering follower state: follower="Node at ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f [Follower]" leader-address= leader-id= +2026-11-29T10:28:50.200Z [INFO] server-e0b5f1ae-f4a0-714e-931c-2daf5257036e: entering follower state: follower="Node at e0b5f1ae-f4a0-714e-931c-2daf5257036e [Follower]" leader-address= leader-id= +2026-11-29T10:28:50.200Z [INFO] server-963100ce-19f4-306c-3e9f-dbe1ee31d1c1: entering follower state: follower="Node at 963100ce-19f4-306c-3e9f-dbe1ee31d1c1 [Follower]" leader-address= leader-id= +2026-11-29T10:28:50.710Z [WARN] server-ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:50.710Z [INFO] server-ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f: entering candidate state: node="Node at ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f [Candidate]" term=2 +2026-11-29T10:28:50.711Z [INFO] server-ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f: election won: term=2 tally=2 +2026-11-29T10:28:50.711Z [INFO] server-ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f: entering leader state: leader="Node at ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f [Leader]" +2026-11-29T10:28:50.711Z [INFO] server-ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f: added peer, starting replication: peer=server-e0b5f1ae-f4a0-714e-931c-2daf5257036e +2026-11-29T10:28:50.711Z [INFO] server-ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f: added peer, starting replication: peer=server-963100ce-19f4-306c-3e9f-dbe1ee31d1c1 +2026-11-29T10:28:50.711Z [INFO] server-ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f: pipelining replication: peer="{Voter server-963100ce-19f4-306c-3e9f-dbe1ee31d1c1 963100ce-19f4-306c-3e9f-dbe1ee31d1c1}" +2026-11-29T10:28:50.711Z [INFO] server-ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f: pipelining replication: peer="{Voter server-e0b5f1ae-f4a0-714e-931c-2daf5257036e e0b5f1ae-f4a0-714e-931c-2daf5257036e}" +2026-11-29T10:28:51.716Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:50.711218379 +0000 UTC m=+41.446272441 (1 nodes), 511.17604ms from start of poll, 511.33976ms from cluster start. Timeout at 2026-11-29 10:28:51.716219419 +0000 UTC m=+42.451273481, 1.00500104s after stability +2026-11-29T10:28:51.716Z [INFO] server-ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f: starting snapshot up to: index=12 +2026-11-29T10:28:51.716Z [INFO] creating new snapshot: path=/tmp/raft699502856/snapshots/2-12-1795948131716.tmp +2026-11-29T10:28:51.717Z [INFO] server-ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f: snapshot complete up to: index=12 +2026-11-29T10:28:51.717Z [INFO] creating new snapshot: path=/tmp/raft699502856/snapshots/2-23-1795948131717.tmp +2026-11-29T10:28:51.717Z [INFO] server-ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f: copied to local snapshot: bytes=71 +2026-11-29T10:28:51.717Z [INFO] server-ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f: snapshot restore progress: id=2-23-1795948131717 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:51.717Z [INFO] server-ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f: restored user snapshot: index=23 +2026-11-29T10:28:51.717Z [ERROR] server-ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f: failed to get log: index=23 error="log not found" +2026-11-29T10:28:51.717Z [INFO] server-ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f: aborting pipeline replication: peer="{Voter server-963100ce-19f4-306c-3e9f-dbe1ee31d1c1 963100ce-19f4-306c-3e9f-dbe1ee31d1c1}" +2026-11-29T10:28:51.718Z [ERROR] server-ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f: failed to get log: index=23 error="log not found" +2026-11-29T10:28:51.718Z [INFO] server-ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f: aborting pipeline replication: peer="{Voter server-e0b5f1ae-f4a0-714e-931c-2daf5257036e e0b5f1ae-f4a0-714e-931c-2daf5257036e}" +2026-11-29T10:28:51.726Z [ERROR] server-ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f: failed to get log: index=23 error="log not found" +2026-11-29T10:28:51.726Z [INFO] creating new snapshot: path=/tmp/raft2244169562/snapshots/2-23-1795948131726.tmp +2026-11-29T10:28:51.726Z [INFO] server-e0b5f1ae-f4a0-714e-931c-2daf5257036e: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:51.726Z [INFO] server-e0b5f1ae-f4a0-714e-931c-2daf5257036e: copied to local snapshot: bytes=71 +2026-11-29T10:28:51.726Z [INFO] server-e0b5f1ae-f4a0-714e-931c-2daf5257036e: snapshot restore progress: id=2-23-1795948131726 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:51.726Z [INFO] server-e0b5f1ae-f4a0-714e-931c-2daf5257036e: Installed remote snapshot +2026-11-29T10:28:51.726Z [WARN] server-e0b5f1ae-f4a0-714e-931c-2daf5257036e: skipping application of old log: index=22 +2026-11-29T10:28:51.726Z [INFO] server-ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f: pipelining replication: peer="{Voter server-e0b5f1ae-f4a0-714e-931c-2daf5257036e e0b5f1ae-f4a0-714e-931c-2daf5257036e}" +2026-11-29T10:28:51.726Z [ERROR] server-ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f: failed to get log: index=23 error="log not found" +2026-11-29T10:28:51.726Z [INFO] creating new snapshot: path=/tmp/raft801382751/snapshots/2-23-1795948131726.tmp +2026-11-29T10:28:51.727Z [INFO] server-963100ce-19f4-306c-3e9f-dbe1ee31d1c1: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:51.727Z [INFO] server-963100ce-19f4-306c-3e9f-dbe1ee31d1c1: copied to local snapshot: bytes=71 +2026-11-29T10:28:51.727Z [INFO] server-963100ce-19f4-306c-3e9f-dbe1ee31d1c1: snapshot restore progress: id=2-23-1795948131726 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:51.727Z [INFO] server-963100ce-19f4-306c-3e9f-dbe1ee31d1c1: Installed remote snapshot +2026-11-29T10:28:51.727Z [WARN] server-963100ce-19f4-306c-3e9f-dbe1ee31d1c1: skipping application of old log: index=22 +2026-11-29T10:28:51.727Z [INFO] server-ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f: pipelining replication: peer="{Voter server-963100ce-19f4-306c-3e9f-dbe1ee31d1c1 963100ce-19f4-306c-3e9f-dbe1ee31d1c1}" +2026-11-29T10:28:51.727Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:28:52.732Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:28:51.727277379 +0000 UTC m=+42.462331481 (2 nodes), 400ns from start of poll, 1.5273988s from cluster start. Timeout at 2026-11-29 10:28:52.732286659 +0000 UTC m=+43.467340761, 1.00500928s after stability +2026-11-29T10:28:52.743Z [INFO] server-ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f: aborting pipeline replication: peer="{Voter server-963100ce-19f4-306c-3e9f-dbe1ee31d1c1 963100ce-19f4-306c-3e9f-dbe1ee31d1c1}" +2026-11-29T10:28:52.743Z [INFO] server-ba01cf90-4173-0b5b-dcc4-8f9af8c4c60f: aborting pipeline replication: peer="{Voter server-e0b5f1ae-f4a0-714e-931c-2daf5257036e e0b5f1ae-f4a0-714e-931c-2daf5257036e}" === RUN TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_false -2025-10-27T04:02:49.014Z [INFO] server-879213a4-e1f0-a999-ae0b-630c0acad512: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-879213a4-e1f0-a999-ae0b-630c0acad512 Address:879213a4-e1f0-a999-ae0b-630c0acad512} {Suffrage:Voter ID:server-b3030a22-54ec-d513-c36d-13cec008deba Address:b3030a22-54ec-d513-c36d-13cec008deba} {Suffrage:Voter ID:server-42bdc3ae-430f-9a1b-d909-4b1ca6cffe06 Address:42bdc3ae-430f-9a1b-d909-4b1ca6cffe06}]" -2025-10-27T04:02:49.014Z [INFO] server-b3030a22-54ec-d513-c36d-13cec008deba: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-879213a4-e1f0-a999-ae0b-630c0acad512 Address:879213a4-e1f0-a999-ae0b-630c0acad512} {Suffrage:Voter ID:server-b3030a22-54ec-d513-c36d-13cec008deba Address:b3030a22-54ec-d513-c36d-13cec008deba} {Suffrage:Voter ID:server-42bdc3ae-430f-9a1b-d909-4b1ca6cffe06 Address:42bdc3ae-430f-9a1b-d909-4b1ca6cffe06}]" -2025-10-27T04:02:49.014Z [INFO] server-879213a4-e1f0-a999-ae0b-630c0acad512: entering follower state: follower="Node at 879213a4-e1f0-a999-ae0b-630c0acad512 [Follower]" leader-address= leader-id= -2025-10-27T04:02:49.015Z [INFO] server-42bdc3ae-430f-9a1b-d909-4b1ca6cffe06: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-879213a4-e1f0-a999-ae0b-630c0acad512 Address:879213a4-e1f0-a999-ae0b-630c0acad512} {Suffrage:Voter ID:server-b3030a22-54ec-d513-c36d-13cec008deba Address:b3030a22-54ec-d513-c36d-13cec008deba} {Suffrage:Voter ID:server-42bdc3ae-430f-9a1b-d909-4b1ca6cffe06 Address:42bdc3ae-430f-9a1b-d909-4b1ca6cffe06}]" -2025-10-27T04:02:49.015Z [INFO] server-b3030a22-54ec-d513-c36d-13cec008deba: entering follower state: follower="Node at b3030a22-54ec-d513-c36d-13cec008deba [Follower]" leader-address= leader-id= -2025-10-27T04:02:49.015Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:49.015Z [INFO] server-42bdc3ae-430f-9a1b-d909-4b1ca6cffe06: entering follower state: follower="Node at 42bdc3ae-430f-9a1b-d909-4b1ca6cffe06 [Follower]" leader-address= leader-id= -2025-10-27T04:02:49.616Z [WARN] server-42bdc3ae-430f-9a1b-d909-4b1ca6cffe06: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:49.617Z [INFO] server-42bdc3ae-430f-9a1b-d909-4b1ca6cffe06: entering candidate state: node="Node at 42bdc3ae-430f-9a1b-d909-4b1ca6cffe06 [Candidate]" term=2 -2025-10-27T04:02:49.617Z [INFO] server-42bdc3ae-430f-9a1b-d909-4b1ca6cffe06: election won: term=2 tally=2 -2025-10-27T04:02:49.617Z [INFO] server-42bdc3ae-430f-9a1b-d909-4b1ca6cffe06: entering leader state: leader="Node at 42bdc3ae-430f-9a1b-d909-4b1ca6cffe06 [Leader]" -2025-10-27T04:02:49.617Z [INFO] server-42bdc3ae-430f-9a1b-d909-4b1ca6cffe06: added peer, starting replication: peer=server-879213a4-e1f0-a999-ae0b-630c0acad512 -2025-10-27T04:02:49.617Z [INFO] server-42bdc3ae-430f-9a1b-d909-4b1ca6cffe06: added peer, starting replication: peer=server-b3030a22-54ec-d513-c36d-13cec008deba -2025-10-27T04:02:49.617Z [INFO] server-42bdc3ae-430f-9a1b-d909-4b1ca6cffe06: pipelining replication: peer="{Voter server-b3030a22-54ec-d513-c36d-13cec008deba b3030a22-54ec-d513-c36d-13cec008deba}" -2025-10-27T04:02:49.617Z [INFO] server-42bdc3ae-430f-9a1b-d909-4b1ca6cffe06: pipelining replication: peer="{Voter server-879213a4-e1f0-a999-ae0b-630c0acad512 879213a4-e1f0-a999-ae0b-630c0acad512}" -2025-10-27T04:02:50.622Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:49.617422906 +0000 UTC m=+44.612295344 (1 nodes), 602.271355ms from start of poll, 602.710718ms from cluster start. Timeout at 2025-10-27 04:02:50.622430346 +0000 UTC m=+45.617302864, 1.00500752s after stability -2025-10-27T04:02:50.624Z [INFO] server-42bdc3ae-430f-9a1b-d909-4b1ca6cffe06: starting snapshot up to: index=12 -2025-10-27T04:02:50.624Z [INFO] creating new snapshot: path=/tmp/raft4153119421/snapshots/2-12-1761537770624.tmp -2025-10-27T04:02:50.625Z [INFO] server-42bdc3ae-430f-9a1b-d909-4b1ca6cffe06: snapshot complete up to: index=12 -2025-10-27T04:02:50.625Z [INFO] creating new snapshot: path=/tmp/raft4153119421/snapshots/2-23-1761537770625.tmp -2025-10-27T04:02:50.626Z [INFO] server-42bdc3ae-430f-9a1b-d909-4b1ca6cffe06: copied to local snapshot: bytes=71 -2025-10-27T04:02:50.626Z [INFO] server-42bdc3ae-430f-9a1b-d909-4b1ca6cffe06: snapshot restore progress: id=2-23-1761537770625 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:50.626Z [INFO] server-42bdc3ae-430f-9a1b-d909-4b1ca6cffe06: removing all old logs from log store -2025-10-27T04:02:50.626Z [INFO] server-42bdc3ae-430f-9a1b-d909-4b1ca6cffe06: compacting logs: from=1 to=22 -2025-10-27T04:02:50.626Z [INFO] server-42bdc3ae-430f-9a1b-d909-4b1ca6cffe06: restored user snapshot: index=23 -2025-10-27T04:02:50.626Z [ERROR] server-42bdc3ae-430f-9a1b-d909-4b1ca6cffe06: failed to get log: index=22 error="log not found" -2025-10-27T04:02:50.626Z [INFO] server-42bdc3ae-430f-9a1b-d909-4b1ca6cffe06: aborting pipeline replication: peer="{Voter server-879213a4-e1f0-a999-ae0b-630c0acad512 879213a4-e1f0-a999-ae0b-630c0acad512}" -2025-10-27T04:02:50.626Z [ERROR] server-42bdc3ae-430f-9a1b-d909-4b1ca6cffe06: failed to get log: index=22 error="log not found" -2025-10-27T04:02:50.626Z [INFO] server-42bdc3ae-430f-9a1b-d909-4b1ca6cffe06: aborting pipeline replication: peer="{Voter server-b3030a22-54ec-d513-c36d-13cec008deba b3030a22-54ec-d513-c36d-13cec008deba}" -2025-10-27T04:02:50.637Z [ERROR] server-42bdc3ae-430f-9a1b-d909-4b1ca6cffe06: failed to get log: index=22 error="log not found" -2025-10-27T04:02:50.637Z [ERROR] server-42bdc3ae-430f-9a1b-d909-4b1ca6cffe06: failed to get log: index=22 error="log not found" -2025-10-27T04:02:50.637Z [INFO] creating new snapshot: path=/tmp/raft2971426559/snapshots/2-23-1761537770637.tmp -2025-10-27T04:02:50.637Z [INFO] creating new snapshot: path=/tmp/raft1786065281/snapshots/2-23-1761537770637.tmp -2025-10-27T04:02:50.637Z [INFO] server-b3030a22-54ec-d513-c36d-13cec008deba: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:50.637Z [INFO] server-b3030a22-54ec-d513-c36d-13cec008deba: copied to local snapshot: bytes=71 -2025-10-27T04:02:50.637Z [INFO] server-879213a4-e1f0-a999-ae0b-630c0acad512: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:50.638Z [INFO] server-b3030a22-54ec-d513-c36d-13cec008deba: snapshot restore progress: id=2-23-1761537770637 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:50.638Z [INFO] server-b3030a22-54ec-d513-c36d-13cec008deba: removing all old logs from log store -2025-10-27T04:02:50.638Z [INFO] server-b3030a22-54ec-d513-c36d-13cec008deba: compacting logs: from=1 to=22 -2025-10-27T04:02:50.638Z [INFO] server-b3030a22-54ec-d513-c36d-13cec008deba: Installed remote snapshot -2025-10-27T04:02:50.638Z [WARN] server-b3030a22-54ec-d513-c36d-13cec008deba: skipping application of old log: index=22 -2025-10-27T04:02:50.638Z [INFO] server-42bdc3ae-430f-9a1b-d909-4b1ca6cffe06: pipelining replication: peer="{Voter server-b3030a22-54ec-d513-c36d-13cec008deba b3030a22-54ec-d513-c36d-13cec008deba}" -2025-10-27T04:02:50.638Z [INFO] server-879213a4-e1f0-a999-ae0b-630c0acad512: copied to local snapshot: bytes=71 -2025-10-27T04:02:50.638Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:02:50.638Z [INFO] server-879213a4-e1f0-a999-ae0b-630c0acad512: snapshot restore progress: id=2-23-1761537770637 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:50.638Z [INFO] server-879213a4-e1f0-a999-ae0b-630c0acad512: removing all old logs from log store -2025-10-27T04:02:50.638Z [INFO] server-879213a4-e1f0-a999-ae0b-630c0acad512: compacting logs: from=1 to=22 -2025-10-27T04:02:50.638Z [INFO] server-879213a4-e1f0-a999-ae0b-630c0acad512: Installed remote snapshot -2025-10-27T04:02:50.638Z [INFO] server-42bdc3ae-430f-9a1b-d909-4b1ca6cffe06: pipelining replication: peer="{Voter server-879213a4-e1f0-a999-ae0b-630c0acad512 879213a4-e1f0-a999-ae0b-630c0acad512}" -2025-10-27T04:02:51.645Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:02:50.638226922 +0000 UTC m=+45.633099400 (2 nodes), 1.28µs from start of poll, 1.623514774s from cluster start. Timeout at 2025-10-27 04:02:51.643249842 +0000 UTC m=+46.638122320, 1.00502292s after stability -2025-10-27T04:02:51.661Z [INFO] server-42bdc3ae-430f-9a1b-d909-4b1ca6cffe06: aborting pipeline replication: peer="{Voter server-b3030a22-54ec-d513-c36d-13cec008deba b3030a22-54ec-d513-c36d-13cec008deba}" -2025-10-27T04:02:51.661Z [INFO] server-42bdc3ae-430f-9a1b-d909-4b1ca6cffe06: aborting pipeline replication: peer="{Voter server-879213a4-e1f0-a999-ae0b-630c0acad512 879213a4-e1f0-a999-ae0b-630c0acad512}" +2026-11-29T10:28:52.744Z [INFO] server-d04afa72-42dd-3982-0685-41c3b203d324: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d04afa72-42dd-3982-0685-41c3b203d324 Address:d04afa72-42dd-3982-0685-41c3b203d324} {Suffrage:Voter ID:server-ae71f7dd-fe74-6a88-4f6e-450d70358d41 Address:ae71f7dd-fe74-6a88-4f6e-450d70358d41} {Suffrage:Voter ID:server-55965e3c-9710-c606-aba4-44063921d7e5 Address:55965e3c-9710-c606-aba4-44063921d7e5}]" +2026-11-29T10:28:52.744Z [INFO] server-ae71f7dd-fe74-6a88-4f6e-450d70358d41: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d04afa72-42dd-3982-0685-41c3b203d324 Address:d04afa72-42dd-3982-0685-41c3b203d324} {Suffrage:Voter ID:server-ae71f7dd-fe74-6a88-4f6e-450d70358d41 Address:ae71f7dd-fe74-6a88-4f6e-450d70358d41} {Suffrage:Voter ID:server-55965e3c-9710-c606-aba4-44063921d7e5 Address:55965e3c-9710-c606-aba4-44063921d7e5}]" +2026-11-29T10:28:52.744Z [INFO] server-55965e3c-9710-c606-aba4-44063921d7e5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d04afa72-42dd-3982-0685-41c3b203d324 Address:d04afa72-42dd-3982-0685-41c3b203d324} {Suffrage:Voter ID:server-ae71f7dd-fe74-6a88-4f6e-450d70358d41 Address:ae71f7dd-fe74-6a88-4f6e-450d70358d41} {Suffrage:Voter ID:server-55965e3c-9710-c606-aba4-44063921d7e5 Address:55965e3c-9710-c606-aba4-44063921d7e5}]" +2026-11-29T10:28:52.744Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:52.744Z [INFO] server-d04afa72-42dd-3982-0685-41c3b203d324: entering follower state: follower="Node at d04afa72-42dd-3982-0685-41c3b203d324 [Follower]" leader-address= leader-id= +2026-11-29T10:28:52.744Z [INFO] server-ae71f7dd-fe74-6a88-4f6e-450d70358d41: entering follower state: follower="Node at ae71f7dd-fe74-6a88-4f6e-450d70358d41 [Follower]" leader-address= leader-id= +2026-11-29T10:28:52.744Z [INFO] server-55965e3c-9710-c606-aba4-44063921d7e5: entering follower state: follower="Node at 55965e3c-9710-c606-aba4-44063921d7e5 [Follower]" leader-address= leader-id= +2026-11-29T10:28:53.425Z [WARN] server-55965e3c-9710-c606-aba4-44063921d7e5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:53.425Z [INFO] server-55965e3c-9710-c606-aba4-44063921d7e5: entering candidate state: node="Node at 55965e3c-9710-c606-aba4-44063921d7e5 [Candidate]" term=2 +2026-11-29T10:28:53.425Z [INFO] server-55965e3c-9710-c606-aba4-44063921d7e5: election won: term=2 tally=2 +2026-11-29T10:28:53.425Z [INFO] server-55965e3c-9710-c606-aba4-44063921d7e5: entering leader state: leader="Node at 55965e3c-9710-c606-aba4-44063921d7e5 [Leader]" +2026-11-29T10:28:53.425Z [INFO] server-55965e3c-9710-c606-aba4-44063921d7e5: added peer, starting replication: peer=server-d04afa72-42dd-3982-0685-41c3b203d324 +2026-11-29T10:28:53.425Z [INFO] server-55965e3c-9710-c606-aba4-44063921d7e5: added peer, starting replication: peer=server-ae71f7dd-fe74-6a88-4f6e-450d70358d41 +2026-11-29T10:28:53.426Z [INFO] server-55965e3c-9710-c606-aba4-44063921d7e5: pipelining replication: peer="{Voter server-d04afa72-42dd-3982-0685-41c3b203d324 d04afa72-42dd-3982-0685-41c3b203d324}" +2026-11-29T10:28:53.426Z [INFO] server-55965e3c-9710-c606-aba4-44063921d7e5: pipelining replication: peer="{Voter server-ae71f7dd-fe74-6a88-4f6e-450d70358d41 ae71f7dd-fe74-6a88-4f6e-450d70358d41}" +2026-11-29T10:28:54.432Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:53.426011739 +0000 UTC m=+44.161065841 (1 nodes), 681.8884ms from start of poll, 682.06904ms from cluster start. Timeout at 2026-11-29 10:28:54.431012539 +0000 UTC m=+45.166066641, 1.0050008s after stability +2026-11-29T10:28:54.433Z [INFO] server-55965e3c-9710-c606-aba4-44063921d7e5: starting snapshot up to: index=12 +2026-11-29T10:28:54.433Z [INFO] creating new snapshot: path=/tmp/raft2144304338/snapshots/2-12-1795948134433.tmp +2026-11-29T10:28:54.433Z [INFO] server-55965e3c-9710-c606-aba4-44063921d7e5: snapshot complete up to: index=12 +2026-11-29T10:28:54.434Z [INFO] creating new snapshot: path=/tmp/raft2144304338/snapshots/2-23-1795948134434.tmp +2026-11-29T10:28:54.434Z [INFO] server-55965e3c-9710-c606-aba4-44063921d7e5: copied to local snapshot: bytes=71 +2026-11-29T10:28:54.436Z [INFO] server-55965e3c-9710-c606-aba4-44063921d7e5: snapshot restore progress: id=2-23-1795948134434 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:54.436Z [INFO] server-55965e3c-9710-c606-aba4-44063921d7e5: removing all old logs from log store +2026-11-29T10:28:54.436Z [INFO] server-55965e3c-9710-c606-aba4-44063921d7e5: compacting logs: from=1 to=22 +2026-11-29T10:28:54.436Z [INFO] server-55965e3c-9710-c606-aba4-44063921d7e5: restored user snapshot: index=23 +2026-11-29T10:28:54.436Z [ERROR] server-55965e3c-9710-c606-aba4-44063921d7e5: failed to get log: index=22 error="log not found" +2026-11-29T10:28:54.436Z [INFO] server-55965e3c-9710-c606-aba4-44063921d7e5: aborting pipeline replication: peer="{Voter server-ae71f7dd-fe74-6a88-4f6e-450d70358d41 ae71f7dd-fe74-6a88-4f6e-450d70358d41}" +2026-11-29T10:28:54.436Z [ERROR] server-55965e3c-9710-c606-aba4-44063921d7e5: failed to get log: index=22 error="log not found" +2026-11-29T10:28:54.436Z [INFO] server-55965e3c-9710-c606-aba4-44063921d7e5: aborting pipeline replication: peer="{Voter server-d04afa72-42dd-3982-0685-41c3b203d324 d04afa72-42dd-3982-0685-41c3b203d324}" +2026-11-29T10:28:54.443Z [ERROR] server-55965e3c-9710-c606-aba4-44063921d7e5: failed to get log: index=22 error="log not found" +2026-11-29T10:28:54.444Z [INFO] creating new snapshot: path=/tmp/raft4019728454/snapshots/2-23-1795948134444.tmp +2026-11-29T10:28:54.444Z [ERROR] server-55965e3c-9710-c606-aba4-44063921d7e5: failed to get log: index=22 error="log not found" +2026-11-29T10:28:54.444Z [INFO] server-d04afa72-42dd-3982-0685-41c3b203d324: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:54.444Z [INFO] server-d04afa72-42dd-3982-0685-41c3b203d324: copied to local snapshot: bytes=71 +2026-11-29T10:28:54.444Z [INFO] creating new snapshot: path=/tmp/raft502466075/snapshots/2-23-1795948134444.tmp +2026-11-29T10:28:54.444Z [INFO] server-d04afa72-42dd-3982-0685-41c3b203d324: snapshot restore progress: id=2-23-1795948134444 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:54.444Z [INFO] server-d04afa72-42dd-3982-0685-41c3b203d324: removing all old logs from log store +2026-11-29T10:28:54.444Z [INFO] server-d04afa72-42dd-3982-0685-41c3b203d324: compacting logs: from=1 to=22 +2026-11-29T10:28:54.444Z [INFO] server-d04afa72-42dd-3982-0685-41c3b203d324: Installed remote snapshot +2026-11-29T10:28:54.444Z [WARN] server-d04afa72-42dd-3982-0685-41c3b203d324: skipping application of old log: index=22 +2026-11-29T10:28:54.444Z [INFO] server-55965e3c-9710-c606-aba4-44063921d7e5: pipelining replication: peer="{Voter server-d04afa72-42dd-3982-0685-41c3b203d324 d04afa72-42dd-3982-0685-41c3b203d324}" +2026-11-29T10:28:54.444Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:28:54.444Z [INFO] server-ae71f7dd-fe74-6a88-4f6e-450d70358d41: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:54.444Z [INFO] server-ae71f7dd-fe74-6a88-4f6e-450d70358d41: copied to local snapshot: bytes=71 +2026-11-29T10:28:54.444Z [INFO] server-ae71f7dd-fe74-6a88-4f6e-450d70358d41: snapshot restore progress: id=2-23-1795948134444 last-index=23 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:54.444Z [INFO] server-ae71f7dd-fe74-6a88-4f6e-450d70358d41: removing all old logs from log store +2026-11-29T10:28:54.444Z [INFO] server-ae71f7dd-fe74-6a88-4f6e-450d70358d41: compacting logs: from=1 to=22 +2026-11-29T10:28:54.444Z [INFO] server-ae71f7dd-fe74-6a88-4f6e-450d70358d41: Installed remote snapshot +2026-11-29T10:28:54.444Z [INFO] server-55965e3c-9710-c606-aba4-44063921d7e5: pipelining replication: peer="{Voter server-ae71f7dd-fe74-6a88-4f6e-450d70358d41 ae71f7dd-fe74-6a88-4f6e-450d70358d41}" +2026-11-29T10:28:55.450Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:28:54.444657579 +0000 UTC m=+45.179711641 (2 nodes), 640ns from start of poll, 1.70071484s from cluster start. Timeout at 2026-11-29 10:28:55.449676259 +0000 UTC m=+46.184730401, 1.00501876s after stability +2026-11-29T10:28:55.461Z [INFO] server-55965e3c-9710-c606-aba4-44063921d7e5: aborting pipeline replication: peer="{Voter server-ae71f7dd-fe74-6a88-4f6e-450d70358d41 ae71f7dd-fe74-6a88-4f6e-450d70358d41}" +2026-11-29T10:28:55.462Z [INFO] server-55965e3c-9710-c606-aba4-44063921d7e5: aborting pipeline replication: peer="{Voter server-d04afa72-42dd-3982-0685-41c3b203d324 d04afa72-42dd-3982-0685-41c3b203d324}" === RUN TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_true -2025-10-27T04:02:51.662Z [INFO] server-41321cd0-f191-56d3-6f09-2e2c36bb0420: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-41321cd0-f191-56d3-6f09-2e2c36bb0420 Address:41321cd0-f191-56d3-6f09-2e2c36bb0420} {Suffrage:Voter ID:server-081d8429-949e-aa9e-4054-ba9905168758 Address:081d8429-949e-aa9e-4054-ba9905168758} {Suffrage:Voter ID:server-59048112-a972-e1d0-670e-dfbe0a041161 Address:59048112-a972-e1d0-670e-dfbe0a041161}]" -2025-10-27T04:02:51.662Z [INFO] server-41321cd0-f191-56d3-6f09-2e2c36bb0420: entering follower state: follower="Node at 41321cd0-f191-56d3-6f09-2e2c36bb0420 [Follower]" leader-address= leader-id= -2025-10-27T04:02:51.662Z [INFO] server-081d8429-949e-aa9e-4054-ba9905168758: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-41321cd0-f191-56d3-6f09-2e2c36bb0420 Address:41321cd0-f191-56d3-6f09-2e2c36bb0420} {Suffrage:Voter ID:server-081d8429-949e-aa9e-4054-ba9905168758 Address:081d8429-949e-aa9e-4054-ba9905168758} {Suffrage:Voter ID:server-59048112-a972-e1d0-670e-dfbe0a041161 Address:59048112-a972-e1d0-670e-dfbe0a041161}]" -2025-10-27T04:02:51.662Z [INFO] server-59048112-a972-e1d0-670e-dfbe0a041161: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-41321cd0-f191-56d3-6f09-2e2c36bb0420 Address:41321cd0-f191-56d3-6f09-2e2c36bb0420} {Suffrage:Voter ID:server-081d8429-949e-aa9e-4054-ba9905168758 Address:081d8429-949e-aa9e-4054-ba9905168758} {Suffrage:Voter ID:server-59048112-a972-e1d0-670e-dfbe0a041161 Address:59048112-a972-e1d0-670e-dfbe0a041161}]" -2025-10-27T04:02:51.662Z [INFO] server-081d8429-949e-aa9e-4054-ba9905168758: entering follower state: follower="Node at 081d8429-949e-aa9e-4054-ba9905168758 [Follower]" leader-address= leader-id= -2025-10-27T04:02:51.662Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:51.662Z [INFO] server-59048112-a972-e1d0-670e-dfbe0a041161: entering follower state: follower="Node at 59048112-a972-e1d0-670e-dfbe0a041161 [Follower]" leader-address= leader-id= -2025-10-27T04:02:52.192Z [WARN] server-081d8429-949e-aa9e-4054-ba9905168758: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:52.192Z [INFO] server-081d8429-949e-aa9e-4054-ba9905168758: entering candidate state: node="Node at 081d8429-949e-aa9e-4054-ba9905168758 [Candidate]" term=2 -2025-10-27T04:02:52.192Z [INFO] server-081d8429-949e-aa9e-4054-ba9905168758: election won: term=2 tally=2 -2025-10-27T04:02:52.192Z [INFO] server-081d8429-949e-aa9e-4054-ba9905168758: entering leader state: leader="Node at 081d8429-949e-aa9e-4054-ba9905168758 [Leader]" -2025-10-27T04:02:52.192Z [INFO] server-081d8429-949e-aa9e-4054-ba9905168758: added peer, starting replication: peer=server-41321cd0-f191-56d3-6f09-2e2c36bb0420 -2025-10-27T04:02:52.192Z [INFO] server-081d8429-949e-aa9e-4054-ba9905168758: added peer, starting replication: peer=server-59048112-a972-e1d0-670e-dfbe0a041161 -2025-10-27T04:02:52.192Z [INFO] server-081d8429-949e-aa9e-4054-ba9905168758: pipelining replication: peer="{Voter server-59048112-a972-e1d0-670e-dfbe0a041161 59048112-a972-e1d0-670e-dfbe0a041161}" -2025-10-27T04:02:52.192Z [INFO] server-081d8429-949e-aa9e-4054-ba9905168758: pipelining replication: peer="{Voter server-41321cd0-f191-56d3-6f09-2e2c36bb0420 41321cd0-f191-56d3-6f09-2e2c36bb0420}" -2025-10-27T04:02:53.197Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:52.192261822 +0000 UTC m=+47.187134340 (1 nodes), 529.580579ms from start of poll, 529.986342ms from cluster start. Timeout at 2025-10-27 04:02:53.197278222 +0000 UTC m=+48.192150740, 1.0050164s after stability -2025-10-27T04:02:53.198Z [INFO] server-081d8429-949e-aa9e-4054-ba9905168758: starting snapshot up to: index=12 -2025-10-27T04:02:53.198Z [INFO] creating new snapshot: path=/tmp/raft2667381745/snapshots/2-12-1761537773198.tmp -2025-10-27T04:02:53.198Z [INFO] server-081d8429-949e-aa9e-4054-ba9905168758: snapshot complete up to: index=12 -2025-10-27T04:02:53.199Z [INFO] server-1b88c46c-7d78-237f-4846-6b95251afe0a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1b88c46c-7d78-237f-4846-6b95251afe0a Address:1b88c46c-7d78-237f-4846-6b95251afe0a} {Suffrage:Voter ID:server-065e78c1-14b5-262a-3ae2-12c16af0e8e5 Address:065e78c1-14b5-262a-3ae2-12c16af0e8e5} {Suffrage:Voter ID:server-2db9f80f-4b2c-fa45-bae8-c2af8436c7d0 Address:2db9f80f-4b2c-fa45-bae8-c2af8436c7d0}]" -2025-10-27T04:02:53.200Z [INFO] server-065e78c1-14b5-262a-3ae2-12c16af0e8e5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1b88c46c-7d78-237f-4846-6b95251afe0a Address:1b88c46c-7d78-237f-4846-6b95251afe0a} {Suffrage:Voter ID:server-065e78c1-14b5-262a-3ae2-12c16af0e8e5 Address:065e78c1-14b5-262a-3ae2-12c16af0e8e5} {Suffrage:Voter ID:server-2db9f80f-4b2c-fa45-bae8-c2af8436c7d0 Address:2db9f80f-4b2c-fa45-bae8-c2af8436c7d0}]" -2025-10-27T04:02:53.199Z [INFO] server-1b88c46c-7d78-237f-4846-6b95251afe0a: entering follower state: follower="Node at 1b88c46c-7d78-237f-4846-6b95251afe0a [Follower]" leader-address= leader-id= -2025-10-27T04:02:53.200Z [INFO] server-065e78c1-14b5-262a-3ae2-12c16af0e8e5: entering follower state: follower="Node at 065e78c1-14b5-262a-3ae2-12c16af0e8e5 [Follower]" leader-address= leader-id= -2025-10-27T04:02:53.200Z [INFO] server-2db9f80f-4b2c-fa45-bae8-c2af8436c7d0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1b88c46c-7d78-237f-4846-6b95251afe0a Address:1b88c46c-7d78-237f-4846-6b95251afe0a} {Suffrage:Voter ID:server-065e78c1-14b5-262a-3ae2-12c16af0e8e5 Address:065e78c1-14b5-262a-3ae2-12c16af0e8e5} {Suffrage:Voter ID:server-2db9f80f-4b2c-fa45-bae8-c2af8436c7d0 Address:2db9f80f-4b2c-fa45-bae8-c2af8436c7d0}]" -2025-10-27T04:02:53.200Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:53.200Z [INFO] server-2db9f80f-4b2c-fa45-bae8-c2af8436c7d0: entering follower state: follower="Node at 2db9f80f-4b2c-fa45-bae8-c2af8436c7d0 [Follower]" leader-address= leader-id= -2025-10-27T04:02:53.964Z [WARN] server-065e78c1-14b5-262a-3ae2-12c16af0e8e5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:53.964Z [INFO] server-065e78c1-14b5-262a-3ae2-12c16af0e8e5: entering candidate state: node="Node at 065e78c1-14b5-262a-3ae2-12c16af0e8e5 [Candidate]" term=2 -2025-10-27T04:02:53.964Z [INFO] server-065e78c1-14b5-262a-3ae2-12c16af0e8e5: election won: term=2 tally=2 -2025-10-27T04:02:53.964Z [INFO] server-065e78c1-14b5-262a-3ae2-12c16af0e8e5: entering leader state: leader="Node at 065e78c1-14b5-262a-3ae2-12c16af0e8e5 [Leader]" -2025-10-27T04:02:53.964Z [INFO] server-065e78c1-14b5-262a-3ae2-12c16af0e8e5: added peer, starting replication: peer=server-1b88c46c-7d78-237f-4846-6b95251afe0a -2025-10-27T04:02:53.964Z [INFO] server-065e78c1-14b5-262a-3ae2-12c16af0e8e5: added peer, starting replication: peer=server-2db9f80f-4b2c-fa45-bae8-c2af8436c7d0 -2025-10-27T04:02:53.964Z [INFO] server-065e78c1-14b5-262a-3ae2-12c16af0e8e5: pipelining replication: peer="{Voter server-2db9f80f-4b2c-fa45-bae8-c2af8436c7d0 2db9f80f-4b2c-fa45-bae8-c2af8436c7d0}" -2025-10-27T04:02:53.964Z [INFO] server-065e78c1-14b5-262a-3ae2-12c16af0e8e5: pipelining replication: peer="{Voter server-1b88c46c-7d78-237f-4846-6b95251afe0a 1b88c46c-7d78-237f-4846-6b95251afe0a}" -2025-10-27T04:02:54.970Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:53.964955496 +0000 UTC m=+48.959828014 (1 nodes), 764.809986ms from start of poll, 765.189629ms from cluster start. Timeout at 2025-10-27 04:02:54.969959256 +0000 UTC m=+49.964831734, 1.00500372s after stability -2025-10-27T04:02:54.970Z [INFO] creating new snapshot: path=/tmp/raft2121021374/snapshots/2-15-1761537774970.tmp -2025-10-27T04:02:54.970Z [INFO] server-065e78c1-14b5-262a-3ae2-12c16af0e8e5: copied to local snapshot: bytes=71 -2025-10-27T04:02:54.971Z [INFO] server-065e78c1-14b5-262a-3ae2-12c16af0e8e5: snapshot restore progress: id=2-15-1761537774970 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:54.971Z [INFO] server-065e78c1-14b5-262a-3ae2-12c16af0e8e5: restored user snapshot: index=15 -2025-10-27T04:02:54.971Z [ERROR] server-065e78c1-14b5-262a-3ae2-12c16af0e8e5: failed to get log: index=3 error="log not found" -2025-10-27T04:02:54.971Z [INFO] server-065e78c1-14b5-262a-3ae2-12c16af0e8e5: aborting pipeline replication: peer="{Voter server-2db9f80f-4b2c-fa45-bae8-c2af8436c7d0 2db9f80f-4b2c-fa45-bae8-c2af8436c7d0}" -2025-10-27T04:02:54.971Z [ERROR] server-065e78c1-14b5-262a-3ae2-12c16af0e8e5: failed to get log: index=3 error="log not found" -2025-10-27T04:02:54.971Z [INFO] server-065e78c1-14b5-262a-3ae2-12c16af0e8e5: aborting pipeline replication: peer="{Voter server-1b88c46c-7d78-237f-4846-6b95251afe0a 1b88c46c-7d78-237f-4846-6b95251afe0a}" -2025-10-27T04:02:54.979Z [ERROR] server-065e78c1-14b5-262a-3ae2-12c16af0e8e5: failed to get log: index=3 error="log not found" -2025-10-27T04:02:54.979Z [INFO] creating new snapshot: path=/tmp/raft2473599088/snapshots/2-15-1761537774979.tmp -2025-10-27T04:02:54.980Z [INFO] server-1b88c46c-7d78-237f-4846-6b95251afe0a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:54.980Z [INFO] server-1b88c46c-7d78-237f-4846-6b95251afe0a: copied to local snapshot: bytes=71 -2025-10-27T04:02:54.980Z [INFO] server-1b88c46c-7d78-237f-4846-6b95251afe0a: snapshot restore progress: id=2-15-1761537774979 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:54.980Z [INFO] server-1b88c46c-7d78-237f-4846-6b95251afe0a: Installed remote snapshot -2025-10-27T04:02:54.980Z [ERROR] server-065e78c1-14b5-262a-3ae2-12c16af0e8e5: failed to get log: index=3 error="log not found" -2025-10-27T04:02:54.980Z [INFO] server-065e78c1-14b5-262a-3ae2-12c16af0e8e5: pipelining replication: peer="{Voter server-1b88c46c-7d78-237f-4846-6b95251afe0a 1b88c46c-7d78-237f-4846-6b95251afe0a}" -2025-10-27T04:02:54.980Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:02:54.981Z [INFO] creating new snapshot: path=/tmp/raft2248192130/snapshots/2-15-1761537774981.tmp -2025-10-27T04:02:54.981Z [INFO] server-2db9f80f-4b2c-fa45-bae8-c2af8436c7d0: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:54.981Z [INFO] server-2db9f80f-4b2c-fa45-bae8-c2af8436c7d0: copied to local snapshot: bytes=71 -2025-10-27T04:02:54.981Z [INFO] server-2db9f80f-4b2c-fa45-bae8-c2af8436c7d0: snapshot restore progress: id=2-15-1761537774981 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:54.981Z [INFO] server-2db9f80f-4b2c-fa45-bae8-c2af8436c7d0: Installed remote snapshot -2025-10-27T04:02:54.981Z [INFO] server-065e78c1-14b5-262a-3ae2-12c16af0e8e5: pipelining replication: peer="{Voter server-2db9f80f-4b2c-fa45-bae8-c2af8436c7d0 2db9f80f-4b2c-fa45-bae8-c2af8436c7d0}" -2025-10-27T04:02:55.986Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:02:54.980857558 +0000 UTC m=+49.975730036 (2 nodes), 1µs from start of poll, 1.781091651s from cluster start. Timeout at 2025-10-27 04:02:55.985878158 +0000 UTC m=+50.980750636, 1.0050206s after stability -2025-10-27T04:02:55.999Z [INFO] server-081d8429-949e-aa9e-4054-ba9905168758: aborting pipeline replication: peer="{Voter server-59048112-a972-e1d0-670e-dfbe0a041161 59048112-a972-e1d0-670e-dfbe0a041161}" -2025-10-27T04:02:55.999Z [INFO] server-081d8429-949e-aa9e-4054-ba9905168758: aborting pipeline replication: peer="{Voter server-41321cd0-f191-56d3-6f09-2e2c36bb0420 41321cd0-f191-56d3-6f09-2e2c36bb0420}" -2025-10-27T04:02:56.500Z [ERROR] server-081d8429-949e-aa9e-4054-ba9905168758: failed to heartbeat to: peer=41321cd0-f191-56d3-6f09-2e2c36bb0420 backoff time=10ms error="command timed out" +2026-11-29T10:28:55.462Z [INFO] server-a49f75b6-87d6-ae71-5aac-ea5b6f16906d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a49f75b6-87d6-ae71-5aac-ea5b6f16906d Address:a49f75b6-87d6-ae71-5aac-ea5b6f16906d} {Suffrage:Voter ID:server-7538b105-e34b-01fd-2587-2d1218bd9fdd Address:7538b105-e34b-01fd-2587-2d1218bd9fdd} {Suffrage:Voter ID:server-72915919-4415-780f-6289-7a46ae677ed3 Address:72915919-4415-780f-6289-7a46ae677ed3}]" +2026-11-29T10:28:55.462Z [INFO] server-7538b105-e34b-01fd-2587-2d1218bd9fdd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a49f75b6-87d6-ae71-5aac-ea5b6f16906d Address:a49f75b6-87d6-ae71-5aac-ea5b6f16906d} {Suffrage:Voter ID:server-7538b105-e34b-01fd-2587-2d1218bd9fdd Address:7538b105-e34b-01fd-2587-2d1218bd9fdd} {Suffrage:Voter ID:server-72915919-4415-780f-6289-7a46ae677ed3 Address:72915919-4415-780f-6289-7a46ae677ed3}]" +2026-11-29T10:28:55.462Z [INFO] server-a49f75b6-87d6-ae71-5aac-ea5b6f16906d: entering follower state: follower="Node at a49f75b6-87d6-ae71-5aac-ea5b6f16906d [Follower]" leader-address= leader-id= +2026-11-29T10:28:55.462Z [INFO] server-72915919-4415-780f-6289-7a46ae677ed3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a49f75b6-87d6-ae71-5aac-ea5b6f16906d Address:a49f75b6-87d6-ae71-5aac-ea5b6f16906d} {Suffrage:Voter ID:server-7538b105-e34b-01fd-2587-2d1218bd9fdd Address:7538b105-e34b-01fd-2587-2d1218bd9fdd} {Suffrage:Voter ID:server-72915919-4415-780f-6289-7a46ae677ed3 Address:72915919-4415-780f-6289-7a46ae677ed3}]" +2026-11-29T10:28:55.462Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:55.462Z [INFO] server-7538b105-e34b-01fd-2587-2d1218bd9fdd: entering follower state: follower="Node at 7538b105-e34b-01fd-2587-2d1218bd9fdd [Follower]" leader-address= leader-id= +2026-11-29T10:28:55.462Z [INFO] server-72915919-4415-780f-6289-7a46ae677ed3: entering follower state: follower="Node at 72915919-4415-780f-6289-7a46ae677ed3 [Follower]" leader-address= leader-id= +2026-11-29T10:28:55.985Z [WARN] server-72915919-4415-780f-6289-7a46ae677ed3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:55.985Z [INFO] server-72915919-4415-780f-6289-7a46ae677ed3: entering candidate state: node="Node at 72915919-4415-780f-6289-7a46ae677ed3 [Candidate]" term=2 +2026-11-29T10:28:55.985Z [INFO] server-72915919-4415-780f-6289-7a46ae677ed3: election won: term=2 tally=2 +2026-11-29T10:28:55.985Z [INFO] server-72915919-4415-780f-6289-7a46ae677ed3: entering leader state: leader="Node at 72915919-4415-780f-6289-7a46ae677ed3 [Leader]" +2026-11-29T10:28:55.985Z [INFO] server-72915919-4415-780f-6289-7a46ae677ed3: added peer, starting replication: peer=server-a49f75b6-87d6-ae71-5aac-ea5b6f16906d +2026-11-29T10:28:55.985Z [INFO] server-72915919-4415-780f-6289-7a46ae677ed3: added peer, starting replication: peer=server-7538b105-e34b-01fd-2587-2d1218bd9fdd +2026-11-29T10:28:55.985Z [INFO] server-72915919-4415-780f-6289-7a46ae677ed3: pipelining replication: peer="{Voter server-7538b105-e34b-01fd-2587-2d1218bd9fdd 7538b105-e34b-01fd-2587-2d1218bd9fdd}" +2026-11-29T10:28:55.985Z [INFO] server-72915919-4415-780f-6289-7a46ae677ed3: pipelining replication: peer="{Voter server-a49f75b6-87d6-ae71-5aac-ea5b6f16906d a49f75b6-87d6-ae71-5aac-ea5b6f16906d}" +2026-11-29T10:28:56.990Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:55.985353179 +0000 UTC m=+46.720407321 (1 nodes), 522.42872ms from start of poll, 522.7072ms from cluster start. Timeout at 2026-11-29 10:28:56.990357579 +0000 UTC m=+47.725411681, 1.00500436s after stability +2026-11-29T10:28:56.991Z [INFO] server-72915919-4415-780f-6289-7a46ae677ed3: starting snapshot up to: index=12 +2026-11-29T10:28:56.991Z [INFO] creating new snapshot: path=/tmp/raft4131251599/snapshots/2-12-1795948136991.tmp +2026-11-29T10:28:56.991Z [INFO] server-72915919-4415-780f-6289-7a46ae677ed3: snapshot complete up to: index=12 +2026-11-29T10:28:56.992Z [INFO] server-1e7379a9-2c0f-840b-d69a-c8fb5c9f9625: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1e7379a9-2c0f-840b-d69a-c8fb5c9f9625 Address:1e7379a9-2c0f-840b-d69a-c8fb5c9f9625} {Suffrage:Voter ID:server-6e4e52b1-0c7f-93e6-6a16-265a9e6ac546 Address:6e4e52b1-0c7f-93e6-6a16-265a9e6ac546} {Suffrage:Voter ID:server-34651360-ff70-5d5c-3008-840b5e0bf159 Address:34651360-ff70-5d5c-3008-840b5e0bf159}]" +2026-11-29T10:28:56.992Z [INFO] server-6e4e52b1-0c7f-93e6-6a16-265a9e6ac546: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1e7379a9-2c0f-840b-d69a-c8fb5c9f9625 Address:1e7379a9-2c0f-840b-d69a-c8fb5c9f9625} {Suffrage:Voter ID:server-6e4e52b1-0c7f-93e6-6a16-265a9e6ac546 Address:6e4e52b1-0c7f-93e6-6a16-265a9e6ac546} {Suffrage:Voter ID:server-34651360-ff70-5d5c-3008-840b5e0bf159 Address:34651360-ff70-5d5c-3008-840b5e0bf159}]" +2026-11-29T10:28:56.992Z [INFO] server-34651360-ff70-5d5c-3008-840b5e0bf159: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1e7379a9-2c0f-840b-d69a-c8fb5c9f9625 Address:1e7379a9-2c0f-840b-d69a-c8fb5c9f9625} {Suffrage:Voter ID:server-6e4e52b1-0c7f-93e6-6a16-265a9e6ac546 Address:6e4e52b1-0c7f-93e6-6a16-265a9e6ac546} {Suffrage:Voter ID:server-34651360-ff70-5d5c-3008-840b5e0bf159 Address:34651360-ff70-5d5c-3008-840b5e0bf159}]" +2026-11-29T10:28:56.992Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:56.992Z [INFO] server-1e7379a9-2c0f-840b-d69a-c8fb5c9f9625: entering follower state: follower="Node at 1e7379a9-2c0f-840b-d69a-c8fb5c9f9625 [Follower]" leader-address= leader-id= +2026-11-29T10:28:56.992Z [INFO] server-6e4e52b1-0c7f-93e6-6a16-265a9e6ac546: entering follower state: follower="Node at 6e4e52b1-0c7f-93e6-6a16-265a9e6ac546 [Follower]" leader-address= leader-id= +2026-11-29T10:28:56.992Z [INFO] server-34651360-ff70-5d5c-3008-840b5e0bf159: entering follower state: follower="Node at 34651360-ff70-5d5c-3008-840b5e0bf159 [Follower]" leader-address= leader-id= +2026-11-29T10:28:57.584Z [WARN] server-1e7379a9-2c0f-840b-d69a-c8fb5c9f9625: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:28:57.584Z [INFO] server-1e7379a9-2c0f-840b-d69a-c8fb5c9f9625: entering candidate state: node="Node at 1e7379a9-2c0f-840b-d69a-c8fb5c9f9625 [Candidate]" term=2 +2026-11-29T10:28:57.584Z [INFO] server-1e7379a9-2c0f-840b-d69a-c8fb5c9f9625: election won: term=2 tally=2 +2026-11-29T10:28:57.584Z [INFO] server-1e7379a9-2c0f-840b-d69a-c8fb5c9f9625: entering leader state: leader="Node at 1e7379a9-2c0f-840b-d69a-c8fb5c9f9625 [Leader]" +2026-11-29T10:28:57.584Z [INFO] server-1e7379a9-2c0f-840b-d69a-c8fb5c9f9625: added peer, starting replication: peer=server-6e4e52b1-0c7f-93e6-6a16-265a9e6ac546 +2026-11-29T10:28:57.584Z [INFO] server-1e7379a9-2c0f-840b-d69a-c8fb5c9f9625: added peer, starting replication: peer=server-34651360-ff70-5d5c-3008-840b5e0bf159 +2026-11-29T10:28:57.584Z [INFO] server-1e7379a9-2c0f-840b-d69a-c8fb5c9f9625: pipelining replication: peer="{Voter server-34651360-ff70-5d5c-3008-840b5e0bf159 34651360-ff70-5d5c-3008-840b5e0bf159}" +2026-11-29T10:28:57.584Z [INFO] server-1e7379a9-2c0f-840b-d69a-c8fb5c9f9625: pipelining replication: peer="{Voter server-6e4e52b1-0c7f-93e6-6a16-265a9e6ac546 6e4e52b1-0c7f-93e6-6a16-265a9e6ac546}" +2026-11-29T10:28:58.590Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:28:57.584523019 +0000 UTC m=+48.319577161 (1 nodes), 592.32612ms from start of poll, 592.52432ms from cluster start. Timeout at 2026-11-29 10:28:58.589525299 +0000 UTC m=+49.324579401, 1.00500224s after stability +2026-11-29T10:28:58.590Z [INFO] creating new snapshot: path=/tmp/raft973509557/snapshots/2-15-1795948138590.tmp +2026-11-29T10:28:58.591Z [INFO] server-1e7379a9-2c0f-840b-d69a-c8fb5c9f9625: copied to local snapshot: bytes=71 +2026-11-29T10:28:58.591Z [INFO] server-1e7379a9-2c0f-840b-d69a-c8fb5c9f9625: snapshot restore progress: id=2-15-1795948138590 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:58.591Z [INFO] server-1e7379a9-2c0f-840b-d69a-c8fb5c9f9625: restored user snapshot: index=15 +2026-11-29T10:28:58.591Z [ERROR] server-1e7379a9-2c0f-840b-d69a-c8fb5c9f9625: failed to get log: index=3 error="log not found" +2026-11-29T10:28:58.591Z [INFO] server-1e7379a9-2c0f-840b-d69a-c8fb5c9f9625: aborting pipeline replication: peer="{Voter server-34651360-ff70-5d5c-3008-840b5e0bf159 34651360-ff70-5d5c-3008-840b5e0bf159}" +2026-11-29T10:28:58.591Z [ERROR] server-1e7379a9-2c0f-840b-d69a-c8fb5c9f9625: failed to get log: index=3 error="log not found" +2026-11-29T10:28:58.591Z [INFO] server-1e7379a9-2c0f-840b-d69a-c8fb5c9f9625: aborting pipeline replication: peer="{Voter server-6e4e52b1-0c7f-93e6-6a16-265a9e6ac546 6e4e52b1-0c7f-93e6-6a16-265a9e6ac546}" +2026-11-29T10:28:58.597Z [ERROR] server-1e7379a9-2c0f-840b-d69a-c8fb5c9f9625: failed to get log: index=3 error="log not found" +2026-11-29T10:28:58.597Z [INFO] creating new snapshot: path=/tmp/raft2784489681/snapshots/2-15-1795948138597.tmp +2026-11-29T10:28:58.598Z [INFO] server-6e4e52b1-0c7f-93e6-6a16-265a9e6ac546: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:58.603Z [ERROR] server-1e7379a9-2c0f-840b-d69a-c8fb5c9f9625: failed to get log: index=3 error="log not found" +2026-11-29T10:28:58.604Z [INFO] creating new snapshot: path=/tmp/raft198127337/snapshots/2-15-1795948138604.tmp +2026-11-29T10:28:58.604Z [INFO] server-6e4e52b1-0c7f-93e6-6a16-265a9e6ac546: copied to local snapshot: bytes=71 +2026-11-29T10:28:58.604Z [INFO] server-34651360-ff70-5d5c-3008-840b5e0bf159: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:58.604Z [INFO] server-6e4e52b1-0c7f-93e6-6a16-265a9e6ac546: snapshot restore progress: id=2-15-1795948138597 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:58.604Z [INFO] server-6e4e52b1-0c7f-93e6-6a16-265a9e6ac546: Installed remote snapshot +2026-11-29T10:28:58.604Z [INFO] server-34651360-ff70-5d5c-3008-840b5e0bf159: copied to local snapshot: bytes=71 +2026-11-29T10:28:58.604Z [INFO] server-1e7379a9-2c0f-840b-d69a-c8fb5c9f9625: pipelining replication: peer="{Voter server-6e4e52b1-0c7f-93e6-6a16-265a9e6ac546 6e4e52b1-0c7f-93e6-6a16-265a9e6ac546}" +2026-11-29T10:28:58.604Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:28:58.604Z [INFO] server-34651360-ff70-5d5c-3008-840b5e0bf159: snapshot restore progress: id=2-15-1795948138604 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:28:58.604Z [INFO] server-34651360-ff70-5d5c-3008-840b5e0bf159: Installed remote snapshot +2026-11-29T10:28:58.604Z [INFO] server-1e7379a9-2c0f-840b-d69a-c8fb5c9f9625: pipelining replication: peer="{Voter server-34651360-ff70-5d5c-3008-840b5e0bf159 34651360-ff70-5d5c-3008-840b5e0bf159}" +2026-11-29T10:28:59.610Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:28:58.604522939 +0000 UTC m=+49.339577041 (2 nodes), 400ns from start of poll, 1.6125242s from cluster start. Timeout at 2026-11-29 10:28:59.609535539 +0000 UTC m=+50.344589721, 1.00501268s after stability +2026-11-29T10:28:59.621Z [INFO] server-72915919-4415-780f-6289-7a46ae677ed3: aborting pipeline replication: peer="{Voter server-7538b105-e34b-01fd-2587-2d1218bd9fdd 7538b105-e34b-01fd-2587-2d1218bd9fdd}" +2026-11-29T10:28:59.621Z [INFO] server-72915919-4415-780f-6289-7a46ae677ed3: aborting pipeline replication: peer="{Voter server-a49f75b6-87d6-ae71-5aac-ea5b6f16906d a49f75b6-87d6-ae71-5aac-ea5b6f16906d}" === RUN TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_true -2025-10-27T04:02:56.501Z [INFO] server-de74a030-0b7c-0969-7a64-6d61aec5f540: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-de74a030-0b7c-0969-7a64-6d61aec5f540 Address:de74a030-0b7c-0969-7a64-6d61aec5f540} {Suffrage:Voter ID:server-92eb2197-a588-7d38-8795-0d1fac785c4c Address:92eb2197-a588-7d38-8795-0d1fac785c4c} {Suffrage:Voter ID:server-76b1e6fc-5a7a-9a33-6262-50de1eaa633b Address:76b1e6fc-5a7a-9a33-6262-50de1eaa633b}]" -2025-10-27T04:02:56.501Z [INFO] server-92eb2197-a588-7d38-8795-0d1fac785c4c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-de74a030-0b7c-0969-7a64-6d61aec5f540 Address:de74a030-0b7c-0969-7a64-6d61aec5f540} {Suffrage:Voter ID:server-92eb2197-a588-7d38-8795-0d1fac785c4c Address:92eb2197-a588-7d38-8795-0d1fac785c4c} {Suffrage:Voter ID:server-76b1e6fc-5a7a-9a33-6262-50de1eaa633b Address:76b1e6fc-5a7a-9a33-6262-50de1eaa633b}]" -2025-10-27T04:02:56.501Z [INFO] server-de74a030-0b7c-0969-7a64-6d61aec5f540: entering follower state: follower="Node at de74a030-0b7c-0969-7a64-6d61aec5f540 [Follower]" leader-address= leader-id= -2025-10-27T04:02:56.501Z [INFO] server-76b1e6fc-5a7a-9a33-6262-50de1eaa633b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-de74a030-0b7c-0969-7a64-6d61aec5f540 Address:de74a030-0b7c-0969-7a64-6d61aec5f540} {Suffrage:Voter ID:server-92eb2197-a588-7d38-8795-0d1fac785c4c Address:92eb2197-a588-7d38-8795-0d1fac785c4c} {Suffrage:Voter ID:server-76b1e6fc-5a7a-9a33-6262-50de1eaa633b Address:76b1e6fc-5a7a-9a33-6262-50de1eaa633b}]" -2025-10-27T04:02:56.501Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:56.501Z [INFO] server-92eb2197-a588-7d38-8795-0d1fac785c4c: entering follower state: follower="Node at 92eb2197-a588-7d38-8795-0d1fac785c4c [Follower]" leader-address= leader-id= -2025-10-27T04:02:56.501Z [INFO] server-76b1e6fc-5a7a-9a33-6262-50de1eaa633b: entering follower state: follower="Node at 76b1e6fc-5a7a-9a33-6262-50de1eaa633b [Follower]" leader-address= leader-id= -2025-10-27T04:02:57.305Z [WARN] server-92eb2197-a588-7d38-8795-0d1fac785c4c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:57.305Z [INFO] server-92eb2197-a588-7d38-8795-0d1fac785c4c: entering candidate state: node="Node at 92eb2197-a588-7d38-8795-0d1fac785c4c [Candidate]" term=2 -2025-10-27T04:02:57.306Z [INFO] server-92eb2197-a588-7d38-8795-0d1fac785c4c: election won: term=2 tally=2 -2025-10-27T04:02:57.306Z [INFO] server-92eb2197-a588-7d38-8795-0d1fac785c4c: entering leader state: leader="Node at 92eb2197-a588-7d38-8795-0d1fac785c4c [Leader]" -2025-10-27T04:02:57.306Z [INFO] server-92eb2197-a588-7d38-8795-0d1fac785c4c: added peer, starting replication: peer=server-de74a030-0b7c-0969-7a64-6d61aec5f540 -2025-10-27T04:02:57.306Z [INFO] server-92eb2197-a588-7d38-8795-0d1fac785c4c: added peer, starting replication: peer=server-76b1e6fc-5a7a-9a33-6262-50de1eaa633b -2025-10-27T04:02:57.306Z [INFO] server-92eb2197-a588-7d38-8795-0d1fac785c4c: pipelining replication: peer="{Voter server-76b1e6fc-5a7a-9a33-6262-50de1eaa633b 76b1e6fc-5a7a-9a33-6262-50de1eaa633b}" -2025-10-27T04:02:57.306Z [INFO] server-92eb2197-a588-7d38-8795-0d1fac785c4c: pipelining replication: peer="{Voter server-de74a030-0b7c-0969-7a64-6d61aec5f540 de74a030-0b7c-0969-7a64-6d61aec5f540}" -2025-10-27T04:02:58.312Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:57.306195608 +0000 UTC m=+52.301068126 (1 nodes), 804.576338ms from start of poll, 804.8733ms from cluster start. Timeout at 2025-10-27 04:02:58.311197928 +0000 UTC m=+53.306070406, 1.00500228s after stability -2025-10-27T04:02:58.312Z [INFO] server-92eb2197-a588-7d38-8795-0d1fac785c4c: starting snapshot up to: index=12 -2025-10-27T04:02:58.312Z [INFO] creating new snapshot: path=/tmp/raft2201347714/snapshots/2-12-1761537778312.tmp -2025-10-27T04:02:58.313Z [INFO] server-92eb2197-a588-7d38-8795-0d1fac785c4c: snapshot complete up to: index=12 -2025-10-27T04:02:58.314Z [INFO] server-c5c5e7ae-5ce8-7365-ed94-f13d1d6529b5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c5c5e7ae-5ce8-7365-ed94-f13d1d6529b5 Address:c5c5e7ae-5ce8-7365-ed94-f13d1d6529b5} {Suffrage:Voter ID:server-726accb5-9e40-80c1-f32f-2b579edc5be4 Address:726accb5-9e40-80c1-f32f-2b579edc5be4} {Suffrage:Voter ID:server-c44118ed-1cdf-c4df-7f82-bb9a951c5815 Address:c44118ed-1cdf-c4df-7f82-bb9a951c5815}]" -2025-10-27T04:02:58.314Z [INFO] server-726accb5-9e40-80c1-f32f-2b579edc5be4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c5c5e7ae-5ce8-7365-ed94-f13d1d6529b5 Address:c5c5e7ae-5ce8-7365-ed94-f13d1d6529b5} {Suffrage:Voter ID:server-726accb5-9e40-80c1-f32f-2b579edc5be4 Address:726accb5-9e40-80c1-f32f-2b579edc5be4} {Suffrage:Voter ID:server-c44118ed-1cdf-c4df-7f82-bb9a951c5815 Address:c44118ed-1cdf-c4df-7f82-bb9a951c5815}]" -2025-10-27T04:02:58.314Z [INFO] server-c44118ed-1cdf-c4df-7f82-bb9a951c5815: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c5c5e7ae-5ce8-7365-ed94-f13d1d6529b5 Address:c5c5e7ae-5ce8-7365-ed94-f13d1d6529b5} {Suffrage:Voter ID:server-726accb5-9e40-80c1-f32f-2b579edc5be4 Address:726accb5-9e40-80c1-f32f-2b579edc5be4} {Suffrage:Voter ID:server-c44118ed-1cdf-c4df-7f82-bb9a951c5815 Address:c44118ed-1cdf-c4df-7f82-bb9a951c5815}]" -2025-10-27T04:02:58.314Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:02:58.314Z [INFO] server-c5c5e7ae-5ce8-7365-ed94-f13d1d6529b5: entering follower state: follower="Node at c5c5e7ae-5ce8-7365-ed94-f13d1d6529b5 [Follower]" leader-address= leader-id= -2025-10-27T04:02:58.314Z [INFO] server-726accb5-9e40-80c1-f32f-2b579edc5be4: entering follower state: follower="Node at 726accb5-9e40-80c1-f32f-2b579edc5be4 [Follower]" leader-address= leader-id= -2025-10-27T04:02:58.314Z [INFO] server-c44118ed-1cdf-c4df-7f82-bb9a951c5815: entering follower state: follower="Node at c44118ed-1cdf-c4df-7f82-bb9a951c5815 [Follower]" leader-address= leader-id= -2025-10-27T04:02:58.932Z [WARN] server-c5c5e7ae-5ce8-7365-ed94-f13d1d6529b5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:02:58.932Z [INFO] server-c5c5e7ae-5ce8-7365-ed94-f13d1d6529b5: entering candidate state: node="Node at c5c5e7ae-5ce8-7365-ed94-f13d1d6529b5 [Candidate]" term=2 -2025-10-27T04:02:58.932Z [INFO] server-c5c5e7ae-5ce8-7365-ed94-f13d1d6529b5: election won: term=2 tally=2 -2025-10-27T04:02:58.932Z [INFO] server-c5c5e7ae-5ce8-7365-ed94-f13d1d6529b5: entering leader state: leader="Node at c5c5e7ae-5ce8-7365-ed94-f13d1d6529b5 [Leader]" -2025-10-27T04:02:58.932Z [INFO] server-c5c5e7ae-5ce8-7365-ed94-f13d1d6529b5: added peer, starting replication: peer=server-726accb5-9e40-80c1-f32f-2b579edc5be4 -2025-10-27T04:02:58.932Z [INFO] server-c5c5e7ae-5ce8-7365-ed94-f13d1d6529b5: added peer, starting replication: peer=server-c44118ed-1cdf-c4df-7f82-bb9a951c5815 -2025-10-27T04:02:58.932Z [INFO] server-c5c5e7ae-5ce8-7365-ed94-f13d1d6529b5: pipelining replication: peer="{Voter server-c44118ed-1cdf-c4df-7f82-bb9a951c5815 c44118ed-1cdf-c4df-7f82-bb9a951c5815}" -2025-10-27T04:02:58.932Z [INFO] server-c5c5e7ae-5ce8-7365-ed94-f13d1d6529b5: pipelining replication: peer="{Voter server-726accb5-9e40-80c1-f32f-2b579edc5be4 726accb5-9e40-80c1-f32f-2b579edc5be4}" -2025-10-27T04:02:59.938Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:02:58.93232384 +0000 UTC m=+53.927196278 (1 nodes), 617.753341ms from start of poll, 618.101983ms from cluster start. Timeout at 2025-10-27 04:02:59.93732988 +0000 UTC m=+54.932202318, 1.00500604s after stability -2025-10-27T04:02:59.938Z [INFO] creating new snapshot: path=/tmp/raft4088146274/snapshots/2-15-1761537779938.tmp -2025-10-27T04:02:59.938Z [INFO] server-c5c5e7ae-5ce8-7365-ed94-f13d1d6529b5: copied to local snapshot: bytes=71 -2025-10-27T04:02:59.938Z [INFO] server-c5c5e7ae-5ce8-7365-ed94-f13d1d6529b5: snapshot restore progress: id=2-15-1761537779938 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:59.938Z [INFO] server-c5c5e7ae-5ce8-7365-ed94-f13d1d6529b5: removing all old logs from log store -2025-10-27T04:02:59.938Z [INFO] server-c5c5e7ae-5ce8-7365-ed94-f13d1d6529b5: compacting logs: from=1 to=2 -2025-10-27T04:02:59.938Z [INFO] server-c5c5e7ae-5ce8-7365-ed94-f13d1d6529b5: restored user snapshot: index=15 -2025-10-27T04:02:59.938Z [ERROR] server-c5c5e7ae-5ce8-7365-ed94-f13d1d6529b5: failed to get log: index=2 error="log not found" -2025-10-27T04:02:59.938Z [INFO] server-c5c5e7ae-5ce8-7365-ed94-f13d1d6529b5: aborting pipeline replication: peer="{Voter server-726accb5-9e40-80c1-f32f-2b579edc5be4 726accb5-9e40-80c1-f32f-2b579edc5be4}" -2025-10-27T04:02:59.938Z [ERROR] server-c5c5e7ae-5ce8-7365-ed94-f13d1d6529b5: failed to get log: index=2 error="log not found" -2025-10-27T04:02:59.938Z [INFO] server-c5c5e7ae-5ce8-7365-ed94-f13d1d6529b5: aborting pipeline replication: peer="{Voter server-c44118ed-1cdf-c4df-7f82-bb9a951c5815 c44118ed-1cdf-c4df-7f82-bb9a951c5815}" -2025-10-27T04:02:59.938Z [ERROR] server-c5c5e7ae-5ce8-7365-ed94-f13d1d6529b5: failed to get log: index=2 error="log not found" -2025-10-27T04:02:59.939Z [INFO] creating new snapshot: path=/tmp/raft85918418/snapshots/2-15-1761537779939.tmp -2025-10-27T04:02:59.939Z [INFO] server-c44118ed-1cdf-c4df-7f82-bb9a951c5815: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:59.939Z [INFO] server-c44118ed-1cdf-c4df-7f82-bb9a951c5815: copied to local snapshot: bytes=71 -2025-10-27T04:02:59.939Z [INFO] server-c44118ed-1cdf-c4df-7f82-bb9a951c5815: snapshot restore progress: id=2-15-1761537779939 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:59.939Z [INFO] server-c44118ed-1cdf-c4df-7f82-bb9a951c5815: removing all old logs from log store -2025-10-27T04:02:59.939Z [INFO] server-c44118ed-1cdf-c4df-7f82-bb9a951c5815: compacting logs: from=1 to=2 -2025-10-27T04:02:59.939Z [INFO] server-c44118ed-1cdf-c4df-7f82-bb9a951c5815: Installed remote snapshot -2025-10-27T04:02:59.939Z [INFO] server-c5c5e7ae-5ce8-7365-ed94-f13d1d6529b5: pipelining replication: peer="{Voter server-c44118ed-1cdf-c4df-7f82-bb9a951c5815 c44118ed-1cdf-c4df-7f82-bb9a951c5815}" -2025-10-27T04:02:59.939Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:02:59.948Z [ERROR] server-c5c5e7ae-5ce8-7365-ed94-f13d1d6529b5: failed to get log: index=2 error="log not found" -2025-10-27T04:02:59.949Z [INFO] creating new snapshot: path=/tmp/raft992199261/snapshots/2-15-1761537779949.tmp -2025-10-27T04:02:59.949Z [INFO] server-726accb5-9e40-80c1-f32f-2b579edc5be4: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:59.949Z [INFO] server-726accb5-9e40-80c1-f32f-2b579edc5be4: copied to local snapshot: bytes=71 -2025-10-27T04:02:59.949Z [INFO] server-726accb5-9e40-80c1-f32f-2b579edc5be4: snapshot restore progress: id=2-15-1761537779949 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:02:59.949Z [INFO] server-726accb5-9e40-80c1-f32f-2b579edc5be4: removing all old logs from log store -2025-10-27T04:02:59.949Z [INFO] server-726accb5-9e40-80c1-f32f-2b579edc5be4: compacting logs: from=1 to=2 -2025-10-27T04:02:59.949Z [INFO] server-726accb5-9e40-80c1-f32f-2b579edc5be4: Installed remote snapshot -2025-10-27T04:02:59.949Z [INFO] server-c5c5e7ae-5ce8-7365-ed94-f13d1d6529b5: pipelining replication: peer="{Voter server-726accb5-9e40-80c1-f32f-2b579edc5be4 726accb5-9e40-80c1-f32f-2b579edc5be4}" -2025-10-27T04:03:00.944Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:02:59.939679884 +0000 UTC m=+54.934552322 (2 nodes), 680ns from start of poll, 1.625458027s from cluster start. Timeout at 2025-10-27 04:03:00.944686164 +0000 UTC m=+55.939558602, 1.00500628s after stability -2025-10-27T04:03:00.957Z [INFO] server-92eb2197-a588-7d38-8795-0d1fac785c4c: aborting pipeline replication: peer="{Voter server-76b1e6fc-5a7a-9a33-6262-50de1eaa633b 76b1e6fc-5a7a-9a33-6262-50de1eaa633b}" -2025-10-27T04:03:00.957Z [INFO] server-92eb2197-a588-7d38-8795-0d1fac785c4c: aborting pipeline replication: peer="{Voter server-de74a030-0b7c-0969-7a64-6d61aec5f540 de74a030-0b7c-0969-7a64-6d61aec5f540}" +2026-11-29T10:28:59.621Z [INFO] server-d2f78837-4f63-deba-30cf-c0878f613cf0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d2f78837-4f63-deba-30cf-c0878f613cf0 Address:d2f78837-4f63-deba-30cf-c0878f613cf0} {Suffrage:Voter ID:server-d3697a77-41e3-8d3a-9453-93e3a3d7348d Address:d3697a77-41e3-8d3a-9453-93e3a3d7348d} {Suffrage:Voter ID:server-56a874cc-2f94-bab6-7a04-9f0216ef8e5b Address:56a874cc-2f94-bab6-7a04-9f0216ef8e5b}]" +2026-11-29T10:28:59.621Z [INFO] server-d3697a77-41e3-8d3a-9453-93e3a3d7348d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d2f78837-4f63-deba-30cf-c0878f613cf0 Address:d2f78837-4f63-deba-30cf-c0878f613cf0} {Suffrage:Voter ID:server-d3697a77-41e3-8d3a-9453-93e3a3d7348d Address:d3697a77-41e3-8d3a-9453-93e3a3d7348d} {Suffrage:Voter ID:server-56a874cc-2f94-bab6-7a04-9f0216ef8e5b Address:56a874cc-2f94-bab6-7a04-9f0216ef8e5b}]" +2026-11-29T10:28:59.621Z [INFO] server-56a874cc-2f94-bab6-7a04-9f0216ef8e5b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d2f78837-4f63-deba-30cf-c0878f613cf0 Address:d2f78837-4f63-deba-30cf-c0878f613cf0} {Suffrage:Voter ID:server-d3697a77-41e3-8d3a-9453-93e3a3d7348d Address:d3697a77-41e3-8d3a-9453-93e3a3d7348d} {Suffrage:Voter ID:server-56a874cc-2f94-bab6-7a04-9f0216ef8e5b Address:56a874cc-2f94-bab6-7a04-9f0216ef8e5b}]" +2026-11-29T10:28:59.621Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:28:59.621Z [INFO] server-d2f78837-4f63-deba-30cf-c0878f613cf0: entering follower state: follower="Node at d2f78837-4f63-deba-30cf-c0878f613cf0 [Follower]" leader-address= leader-id= +2026-11-29T10:28:59.621Z [INFO] server-d3697a77-41e3-8d3a-9453-93e3a3d7348d: entering follower state: follower="Node at d3697a77-41e3-8d3a-9453-93e3a3d7348d [Follower]" leader-address= leader-id= +2026-11-29T10:28:59.621Z [INFO] server-56a874cc-2f94-bab6-7a04-9f0216ef8e5b: entering follower state: follower="Node at 56a874cc-2f94-bab6-7a04-9f0216ef8e5b [Follower]" leader-address= leader-id= +2026-11-29T10:29:00.130Z [WARN] server-56a874cc-2f94-bab6-7a04-9f0216ef8e5b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:00.130Z [INFO] server-56a874cc-2f94-bab6-7a04-9f0216ef8e5b: entering candidate state: node="Node at 56a874cc-2f94-bab6-7a04-9f0216ef8e5b [Candidate]" term=2 +2026-11-29T10:29:00.130Z [INFO] server-56a874cc-2f94-bab6-7a04-9f0216ef8e5b: election won: term=2 tally=2 +2026-11-29T10:29:00.130Z [INFO] server-56a874cc-2f94-bab6-7a04-9f0216ef8e5b: entering leader state: leader="Node at 56a874cc-2f94-bab6-7a04-9f0216ef8e5b [Leader]" +2026-11-29T10:29:00.130Z [INFO] server-56a874cc-2f94-bab6-7a04-9f0216ef8e5b: added peer, starting replication: peer=server-d2f78837-4f63-deba-30cf-c0878f613cf0 +2026-11-29T10:29:00.130Z [INFO] server-56a874cc-2f94-bab6-7a04-9f0216ef8e5b: added peer, starting replication: peer=server-d3697a77-41e3-8d3a-9453-93e3a3d7348d +2026-11-29T10:29:00.130Z [INFO] server-56a874cc-2f94-bab6-7a04-9f0216ef8e5b: pipelining replication: peer="{Voter server-d3697a77-41e3-8d3a-9453-93e3a3d7348d d3697a77-41e3-8d3a-9453-93e3a3d7348d}" +2026-11-29T10:29:00.131Z [INFO] server-56a874cc-2f94-bab6-7a04-9f0216ef8e5b: pipelining replication: peer="{Voter server-d2f78837-4f63-deba-30cf-c0878f613cf0 d2f78837-4f63-deba-30cf-c0878f613cf0}" +2026-11-29T10:29:01.136Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:00.130932859 +0000 UTC m=+50.865986961 (1 nodes), 509.02552ms from start of poll, 509.19848ms from cluster start. Timeout at 2026-11-29 10:29:01.135933939 +0000 UTC m=+51.870988081, 1.00500112s after stability +2026-11-29T10:29:01.136Z [INFO] server-56a874cc-2f94-bab6-7a04-9f0216ef8e5b: starting snapshot up to: index=12 +2026-11-29T10:29:01.136Z [INFO] creating new snapshot: path=/tmp/raft337961436/snapshots/2-12-1795948141136.tmp +2026-11-29T10:29:01.136Z [INFO] server-56a874cc-2f94-bab6-7a04-9f0216ef8e5b: snapshot complete up to: index=12 +2026-11-29T10:29:01.137Z [INFO] server-4c666558-fad3-ebd5-cd98-5f612afcfdd9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4c666558-fad3-ebd5-cd98-5f612afcfdd9 Address:4c666558-fad3-ebd5-cd98-5f612afcfdd9} {Suffrage:Voter ID:server-1dd88b6b-8025-0d35-d481-a90ee0ad8d25 Address:1dd88b6b-8025-0d35-d481-a90ee0ad8d25} {Suffrage:Voter ID:server-290dab43-a86e-86c5-5078-6a35fc1bc419 Address:290dab43-a86e-86c5-5078-6a35fc1bc419}]" +2026-11-29T10:29:01.137Z [INFO] server-1dd88b6b-8025-0d35-d481-a90ee0ad8d25: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4c666558-fad3-ebd5-cd98-5f612afcfdd9 Address:4c666558-fad3-ebd5-cd98-5f612afcfdd9} {Suffrage:Voter ID:server-1dd88b6b-8025-0d35-d481-a90ee0ad8d25 Address:1dd88b6b-8025-0d35-d481-a90ee0ad8d25} {Suffrage:Voter ID:server-290dab43-a86e-86c5-5078-6a35fc1bc419 Address:290dab43-a86e-86c5-5078-6a35fc1bc419}]" +2026-11-29T10:29:01.137Z [INFO] server-290dab43-a86e-86c5-5078-6a35fc1bc419: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4c666558-fad3-ebd5-cd98-5f612afcfdd9 Address:4c666558-fad3-ebd5-cd98-5f612afcfdd9} {Suffrage:Voter ID:server-1dd88b6b-8025-0d35-d481-a90ee0ad8d25 Address:1dd88b6b-8025-0d35-d481-a90ee0ad8d25} {Suffrage:Voter ID:server-290dab43-a86e-86c5-5078-6a35fc1bc419 Address:290dab43-a86e-86c5-5078-6a35fc1bc419}]" +2026-11-29T10:29:01.137Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:01.137Z [INFO] server-4c666558-fad3-ebd5-cd98-5f612afcfdd9: entering follower state: follower="Node at 4c666558-fad3-ebd5-cd98-5f612afcfdd9 [Follower]" leader-address= leader-id= +2026-11-29T10:29:01.137Z [INFO] server-1dd88b6b-8025-0d35-d481-a90ee0ad8d25: entering follower state: follower="Node at 1dd88b6b-8025-0d35-d481-a90ee0ad8d25 [Follower]" leader-address= leader-id= +2026-11-29T10:29:01.137Z [INFO] server-290dab43-a86e-86c5-5078-6a35fc1bc419: entering follower state: follower="Node at 290dab43-a86e-86c5-5078-6a35fc1bc419 [Follower]" leader-address= leader-id= +2026-11-29T10:29:01.748Z [WARN] server-1dd88b6b-8025-0d35-d481-a90ee0ad8d25: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:01.748Z [INFO] server-1dd88b6b-8025-0d35-d481-a90ee0ad8d25: entering candidate state: node="Node at 1dd88b6b-8025-0d35-d481-a90ee0ad8d25 [Candidate]" term=2 +2026-11-29T10:29:01.748Z [INFO] server-1dd88b6b-8025-0d35-d481-a90ee0ad8d25: election won: term=2 tally=2 +2026-11-29T10:29:01.748Z [INFO] server-1dd88b6b-8025-0d35-d481-a90ee0ad8d25: entering leader state: leader="Node at 1dd88b6b-8025-0d35-d481-a90ee0ad8d25 [Leader]" +2026-11-29T10:29:01.748Z [INFO] server-1dd88b6b-8025-0d35-d481-a90ee0ad8d25: added peer, starting replication: peer=server-4c666558-fad3-ebd5-cd98-5f612afcfdd9 +2026-11-29T10:29:01.748Z [INFO] server-1dd88b6b-8025-0d35-d481-a90ee0ad8d25: added peer, starting replication: peer=server-290dab43-a86e-86c5-5078-6a35fc1bc419 +2026-11-29T10:29:01.748Z [INFO] server-1dd88b6b-8025-0d35-d481-a90ee0ad8d25: pipelining replication: peer="{Voter server-4c666558-fad3-ebd5-cd98-5f612afcfdd9 4c666558-fad3-ebd5-cd98-5f612afcfdd9}" +2026-11-29T10:29:01.748Z [INFO] server-1dd88b6b-8025-0d35-d481-a90ee0ad8d25: pipelining replication: peer="{Voter server-290dab43-a86e-86c5-5078-6a35fc1bc419 290dab43-a86e-86c5-5078-6a35fc1bc419}" +2026-11-29T10:29:02.754Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:01.748402899 +0000 UTC m=+52.483457001 (1 nodes), 610.80564ms from start of poll, 610.97864ms from cluster start. Timeout at 2026-11-29 10:29:02.753408819 +0000 UTC m=+53.488462921, 1.00500592s after stability +2026-11-29T10:29:02.754Z [INFO] creating new snapshot: path=/tmp/raft1329571489/snapshots/2-15-1795948142754.tmp +2026-11-29T10:29:02.754Z [INFO] server-1dd88b6b-8025-0d35-d481-a90ee0ad8d25: copied to local snapshot: bytes=71 +2026-11-29T10:29:02.754Z [INFO] server-1dd88b6b-8025-0d35-d481-a90ee0ad8d25: snapshot restore progress: id=2-15-1795948142754 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:02.754Z [INFO] server-1dd88b6b-8025-0d35-d481-a90ee0ad8d25: removing all old logs from log store +2026-11-29T10:29:02.754Z [INFO] server-1dd88b6b-8025-0d35-d481-a90ee0ad8d25: compacting logs: from=1 to=2 +2026-11-29T10:29:02.754Z [INFO] server-1dd88b6b-8025-0d35-d481-a90ee0ad8d25: restored user snapshot: index=15 +2026-11-29T10:29:02.754Z [ERROR] server-1dd88b6b-8025-0d35-d481-a90ee0ad8d25: failed to get log: index=2 error="log not found" +2026-11-29T10:29:02.754Z [INFO] server-1dd88b6b-8025-0d35-d481-a90ee0ad8d25: aborting pipeline replication: peer="{Voter server-290dab43-a86e-86c5-5078-6a35fc1bc419 290dab43-a86e-86c5-5078-6a35fc1bc419}" +2026-11-29T10:29:02.754Z [ERROR] server-1dd88b6b-8025-0d35-d481-a90ee0ad8d25: failed to get log: index=2 error="log not found" +2026-11-29T10:29:02.754Z [INFO] server-1dd88b6b-8025-0d35-d481-a90ee0ad8d25: aborting pipeline replication: peer="{Voter server-4c666558-fad3-ebd5-cd98-5f612afcfdd9 4c666558-fad3-ebd5-cd98-5f612afcfdd9}" +2026-11-29T10:29:02.760Z [ERROR] server-1dd88b6b-8025-0d35-d481-a90ee0ad8d25: failed to get log: index=2 error="log not found" +2026-11-29T10:29:02.761Z [ERROR] server-1dd88b6b-8025-0d35-d481-a90ee0ad8d25: failed to get log: index=2 error="log not found" +2026-11-29T10:29:02.761Z [INFO] creating new snapshot: path=/tmp/raft1074852486/snapshots/2-15-1795948142761.tmp +2026-11-29T10:29:02.761Z [INFO] creating new snapshot: path=/tmp/raft3813487386/snapshots/2-15-1795948142761.tmp +2026-11-29T10:29:02.761Z [INFO] server-290dab43-a86e-86c5-5078-6a35fc1bc419: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:02.761Z [INFO] server-4c666558-fad3-ebd5-cd98-5f612afcfdd9: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:02.761Z [INFO] server-290dab43-a86e-86c5-5078-6a35fc1bc419: copied to local snapshot: bytes=71 +2026-11-29T10:29:02.761Z [INFO] server-4c666558-fad3-ebd5-cd98-5f612afcfdd9: copied to local snapshot: bytes=71 +2026-11-29T10:29:02.761Z [INFO] server-290dab43-a86e-86c5-5078-6a35fc1bc419: snapshot restore progress: id=2-15-1795948142761 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:02.761Z [INFO] server-290dab43-a86e-86c5-5078-6a35fc1bc419: removing all old logs from log store +2026-11-29T10:29:02.761Z [INFO] server-290dab43-a86e-86c5-5078-6a35fc1bc419: compacting logs: from=1 to=2 +2026-11-29T10:29:02.761Z [INFO] server-290dab43-a86e-86c5-5078-6a35fc1bc419: Installed remote snapshot +2026-11-29T10:29:02.761Z [INFO] server-4c666558-fad3-ebd5-cd98-5f612afcfdd9: snapshot restore progress: id=2-15-1795948142761 last-index=15 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:02.761Z [INFO] server-1dd88b6b-8025-0d35-d481-a90ee0ad8d25: pipelining replication: peer="{Voter server-290dab43-a86e-86c5-5078-6a35fc1bc419 290dab43-a86e-86c5-5078-6a35fc1bc419}" +2026-11-29T10:29:02.761Z [INFO] server-4c666558-fad3-ebd5-cd98-5f612afcfdd9: removing all old logs from log store +2026-11-29T10:29:02.761Z [INFO] server-4c666558-fad3-ebd5-cd98-5f612afcfdd9: compacting logs: from=1 to=2 +2026-11-29T10:29:02.761Z [INFO] server-4c666558-fad3-ebd5-cd98-5f612afcfdd9: Installed remote snapshot +2026-11-29T10:29:02.761Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:02.761Z [INFO] server-1dd88b6b-8025-0d35-d481-a90ee0ad8d25: pipelining replication: peer="{Voter server-4c666558-fad3-ebd5-cd98-5f612afcfdd9 4c666558-fad3-ebd5-cd98-5f612afcfdd9}" +2026-11-29T10:29:03.767Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:02.761902379 +0000 UTC m=+53.496956481 (2 nodes), 680ns from start of poll, 1.62447812s from cluster start. Timeout at 2026-11-29 10:29:03.766915339 +0000 UTC m=+54.501969401, 1.00501292s after stability +2026-11-29T10:29:03.778Z [INFO] server-56a874cc-2f94-bab6-7a04-9f0216ef8e5b: aborting pipeline replication: peer="{Voter server-d3697a77-41e3-8d3a-9453-93e3a3d7348d d3697a77-41e3-8d3a-9453-93e3a3d7348d}" +2026-11-29T10:29:03.778Z [INFO] server-56a874cc-2f94-bab6-7a04-9f0216ef8e5b: aborting pipeline replication: peer="{Voter server-d2f78837-4f63-deba-30cf-c0878f613cf0 d2f78837-4f63-deba-30cf-c0878f613cf0}" === RUN TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_false -2025-10-27T04:03:00.958Z [INFO] server-30441ac4-6d03-9651-a92d-460edc888e2e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-30441ac4-6d03-9651-a92d-460edc888e2e Address:30441ac4-6d03-9651-a92d-460edc888e2e} {Suffrage:Voter ID:server-f0b6df46-9ec2-e19a-4285-c7e520ac4b91 Address:f0b6df46-9ec2-e19a-4285-c7e520ac4b91} {Suffrage:Voter ID:server-b9733971-1d45-f127-b376-f5a2304e346b Address:b9733971-1d45-f127-b376-f5a2304e346b}]" -2025-10-27T04:03:00.958Z [INFO] server-f0b6df46-9ec2-e19a-4285-c7e520ac4b91: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-30441ac4-6d03-9651-a92d-460edc888e2e Address:30441ac4-6d03-9651-a92d-460edc888e2e} {Suffrage:Voter ID:server-f0b6df46-9ec2-e19a-4285-c7e520ac4b91 Address:f0b6df46-9ec2-e19a-4285-c7e520ac4b91} {Suffrage:Voter ID:server-b9733971-1d45-f127-b376-f5a2304e346b Address:b9733971-1d45-f127-b376-f5a2304e346b}]" -2025-10-27T04:03:00.958Z [INFO] server-b9733971-1d45-f127-b376-f5a2304e346b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-30441ac4-6d03-9651-a92d-460edc888e2e Address:30441ac4-6d03-9651-a92d-460edc888e2e} {Suffrage:Voter ID:server-f0b6df46-9ec2-e19a-4285-c7e520ac4b91 Address:f0b6df46-9ec2-e19a-4285-c7e520ac4b91} {Suffrage:Voter ID:server-b9733971-1d45-f127-b376-f5a2304e346b Address:b9733971-1d45-f127-b376-f5a2304e346b}]" -2025-10-27T04:03:00.958Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:00.958Z [INFO] server-30441ac4-6d03-9651-a92d-460edc888e2e: entering follower state: follower="Node at 30441ac4-6d03-9651-a92d-460edc888e2e [Follower]" leader-address= leader-id= -2025-10-27T04:03:00.958Z [INFO] server-f0b6df46-9ec2-e19a-4285-c7e520ac4b91: entering follower state: follower="Node at f0b6df46-9ec2-e19a-4285-c7e520ac4b91 [Follower]" leader-address= leader-id= -2025-10-27T04:03:00.958Z [INFO] server-b9733971-1d45-f127-b376-f5a2304e346b: entering follower state: follower="Node at b9733971-1d45-f127-b376-f5a2304e346b [Follower]" leader-address= leader-id= -2025-10-27T04:03:01.751Z [WARN] server-30441ac4-6d03-9651-a92d-460edc888e2e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:01.751Z [INFO] server-30441ac4-6d03-9651-a92d-460edc888e2e: entering candidate state: node="Node at 30441ac4-6d03-9651-a92d-460edc888e2e [Candidate]" term=2 -2025-10-27T04:03:01.751Z [INFO] server-30441ac4-6d03-9651-a92d-460edc888e2e: election won: term=2 tally=2 -2025-10-27T04:03:01.751Z [INFO] server-30441ac4-6d03-9651-a92d-460edc888e2e: entering leader state: leader="Node at 30441ac4-6d03-9651-a92d-460edc888e2e [Leader]" -2025-10-27T04:03:01.751Z [INFO] server-30441ac4-6d03-9651-a92d-460edc888e2e: added peer, starting replication: peer=server-f0b6df46-9ec2-e19a-4285-c7e520ac4b91 -2025-10-27T04:03:01.751Z [INFO] server-30441ac4-6d03-9651-a92d-460edc888e2e: added peer, starting replication: peer=server-b9733971-1d45-f127-b376-f5a2304e346b -2025-10-27T04:03:01.751Z [INFO] server-30441ac4-6d03-9651-a92d-460edc888e2e: pipelining replication: peer="{Voter server-b9733971-1d45-f127-b376-f5a2304e346b b9733971-1d45-f127-b376-f5a2304e346b}" -2025-10-27T04:03:01.751Z [INFO] server-30441ac4-6d03-9651-a92d-460edc888e2e: pipelining replication: peer="{Voter server-f0b6df46-9ec2-e19a-4285-c7e520ac4b91 f0b6df46-9ec2-e19a-4285-c7e520ac4b91}" -2025-10-27T04:03:02.757Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:01.751840467 +0000 UTC m=+56.746712905 (1 nodes), 793.530462ms from start of poll, 793.738264ms from cluster start. Timeout at 2025-10-27 04:03:02.756844547 +0000 UTC m=+57.751716985, 1.00500408s after stability -2025-10-27T04:03:02.757Z [INFO] server-30441ac4-6d03-9651-a92d-460edc888e2e: starting snapshot up to: index=12 -2025-10-27T04:03:02.757Z [INFO] creating new snapshot: path=/tmp/raft3643372382/snapshots/2-12-1761537782757.tmp -2025-10-27T04:03:02.760Z [INFO] server-30441ac4-6d03-9651-a92d-460edc888e2e: snapshot complete up to: index=12 -2025-10-27T04:03:02.765Z [INFO] creating new snapshot: path=/tmp/raft3643372382/snapshots/2-113-1761537782765.tmp -2025-10-27T04:03:02.765Z [INFO] server-30441ac4-6d03-9651-a92d-460edc888e2e: copied to local snapshot: bytes=71 -2025-10-27T04:03:02.765Z [INFO] server-30441ac4-6d03-9651-a92d-460edc888e2e: snapshot restore progress: id=2-113-1761537782765 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:02.765Z [INFO] server-30441ac4-6d03-9651-a92d-460edc888e2e: restored user snapshot: index=113 -2025-10-27T04:03:02.765Z [ERROR] server-30441ac4-6d03-9651-a92d-460edc888e2e: failed to get log: index=23 error="log not found" -2025-10-27T04:03:02.765Z [INFO] server-30441ac4-6d03-9651-a92d-460edc888e2e: aborting pipeline replication: peer="{Voter server-b9733971-1d45-f127-b376-f5a2304e346b b9733971-1d45-f127-b376-f5a2304e346b}" -2025-10-27T04:03:02.765Z [ERROR] server-30441ac4-6d03-9651-a92d-460edc888e2e: failed to get log: index=23 error="log not found" -2025-10-27T04:03:02.765Z [INFO] server-30441ac4-6d03-9651-a92d-460edc888e2e: aborting pipeline replication: peer="{Voter server-f0b6df46-9ec2-e19a-4285-c7e520ac4b91 f0b6df46-9ec2-e19a-4285-c7e520ac4b91}" -2025-10-27T04:03:02.773Z [ERROR] server-30441ac4-6d03-9651-a92d-460edc888e2e: failed to get log: index=23 error="log not found" -2025-10-27T04:03:02.773Z [INFO] creating new snapshot: path=/tmp/raft30823866/snapshots/2-113-1761537782773.tmp -2025-10-27T04:03:02.773Z [INFO] server-b9733971-1d45-f127-b376-f5a2304e346b: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:02.773Z [INFO] server-b9733971-1d45-f127-b376-f5a2304e346b: copied to local snapshot: bytes=71 -2025-10-27T04:03:02.773Z [INFO] server-b9733971-1d45-f127-b376-f5a2304e346b: snapshot restore progress: id=2-113-1761537782773 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:02.773Z [INFO] server-b9733971-1d45-f127-b376-f5a2304e346b: Installed remote snapshot -2025-10-27T04:03:02.773Z [WARN] server-b9733971-1d45-f127-b376-f5a2304e346b: skipping application of old log: index=22 -2025-10-27T04:03:02.774Z [INFO] server-30441ac4-6d03-9651-a92d-460edc888e2e: pipelining replication: peer="{Voter server-b9733971-1d45-f127-b376-f5a2304e346b b9733971-1d45-f127-b376-f5a2304e346b}" -2025-10-27T04:03:02.774Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:03:02.776Z [ERROR] server-30441ac4-6d03-9651-a92d-460edc888e2e: failed to get log: index=23 error="log not found" -2025-10-27T04:03:02.776Z [INFO] creating new snapshot: path=/tmp/raft3015337962/snapshots/2-113-1761537782776.tmp -2025-10-27T04:03:02.777Z [INFO] server-f0b6df46-9ec2-e19a-4285-c7e520ac4b91: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:02.777Z [INFO] server-f0b6df46-9ec2-e19a-4285-c7e520ac4b91: copied to local snapshot: bytes=71 -2025-10-27T04:03:02.777Z [INFO] server-f0b6df46-9ec2-e19a-4285-c7e520ac4b91: snapshot restore progress: id=2-113-1761537782776 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:02.777Z [INFO] server-f0b6df46-9ec2-e19a-4285-c7e520ac4b91: Installed remote snapshot -2025-10-27T04:03:02.777Z [INFO] server-30441ac4-6d03-9651-a92d-460edc888e2e: pipelining replication: peer="{Voter server-f0b6df46-9ec2-e19a-4285-c7e520ac4b91 f0b6df46-9ec2-e19a-4285-c7e520ac4b91}" -2025-10-27T04:03:03.779Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:03:02.774092972 +0000 UTC m=+57.768965410 (2 nodes), 520ns from start of poll, 1.815990769s from cluster start. Timeout at 2025-10-27 04:03:03.779105692 +0000 UTC m=+58.773978130, 1.00501272s after stability -2025-10-27T04:03:03.791Z [INFO] server-30441ac4-6d03-9651-a92d-460edc888e2e: aborting pipeline replication: peer="{Voter server-b9733971-1d45-f127-b376-f5a2304e346b b9733971-1d45-f127-b376-f5a2304e346b}" -2025-10-27T04:03:03.791Z [INFO] server-30441ac4-6d03-9651-a92d-460edc888e2e: aborting pipeline replication: peer="{Voter server-f0b6df46-9ec2-e19a-4285-c7e520ac4b91 f0b6df46-9ec2-e19a-4285-c7e520ac4b91}" +2026-11-29T10:29:03.778Z [INFO] server-daed4bd2-2a47-0c12-e694-c72d421d9532: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-daed4bd2-2a47-0c12-e694-c72d421d9532 Address:daed4bd2-2a47-0c12-e694-c72d421d9532} {Suffrage:Voter ID:server-2b397ab5-62d9-e8a4-5376-a8d0329ba840 Address:2b397ab5-62d9-e8a4-5376-a8d0329ba840} {Suffrage:Voter ID:server-ae57d575-13ac-c529-ceb1-21b2071153f0 Address:ae57d575-13ac-c529-ceb1-21b2071153f0}]" +2026-11-29T10:29:03.778Z [INFO] server-2b397ab5-62d9-e8a4-5376-a8d0329ba840: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-daed4bd2-2a47-0c12-e694-c72d421d9532 Address:daed4bd2-2a47-0c12-e694-c72d421d9532} {Suffrage:Voter ID:server-2b397ab5-62d9-e8a4-5376-a8d0329ba840 Address:2b397ab5-62d9-e8a4-5376-a8d0329ba840} {Suffrage:Voter ID:server-ae57d575-13ac-c529-ceb1-21b2071153f0 Address:ae57d575-13ac-c529-ceb1-21b2071153f0}]" +2026-11-29T10:29:03.779Z [INFO] server-ae57d575-13ac-c529-ceb1-21b2071153f0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-daed4bd2-2a47-0c12-e694-c72d421d9532 Address:daed4bd2-2a47-0c12-e694-c72d421d9532} {Suffrage:Voter ID:server-2b397ab5-62d9-e8a4-5376-a8d0329ba840 Address:2b397ab5-62d9-e8a4-5376-a8d0329ba840} {Suffrage:Voter ID:server-ae57d575-13ac-c529-ceb1-21b2071153f0 Address:ae57d575-13ac-c529-ceb1-21b2071153f0}]" +2026-11-29T10:29:03.778Z [INFO] server-daed4bd2-2a47-0c12-e694-c72d421d9532: entering follower state: follower="Node at daed4bd2-2a47-0c12-e694-c72d421d9532 [Follower]" leader-address= leader-id= +2026-11-29T10:29:03.779Z [INFO] server-2b397ab5-62d9-e8a4-5376-a8d0329ba840: entering follower state: follower="Node at 2b397ab5-62d9-e8a4-5376-a8d0329ba840 [Follower]" leader-address= leader-id= +2026-11-29T10:29:03.779Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:03.779Z [INFO] server-ae57d575-13ac-c529-ceb1-21b2071153f0: entering follower state: follower="Node at ae57d575-13ac-c529-ceb1-21b2071153f0 [Follower]" leader-address= leader-id= +2026-11-29T10:29:04.544Z [WARN] server-2b397ab5-62d9-e8a4-5376-a8d0329ba840: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:04.544Z [INFO] server-2b397ab5-62d9-e8a4-5376-a8d0329ba840: entering candidate state: node="Node at 2b397ab5-62d9-e8a4-5376-a8d0329ba840 [Candidate]" term=2 +2026-11-29T10:29:04.544Z [INFO] server-2b397ab5-62d9-e8a4-5376-a8d0329ba840: election won: term=2 tally=2 +2026-11-29T10:29:04.544Z [INFO] server-2b397ab5-62d9-e8a4-5376-a8d0329ba840: entering leader state: leader="Node at 2b397ab5-62d9-e8a4-5376-a8d0329ba840 [Leader]" +2026-11-29T10:29:04.544Z [INFO] server-2b397ab5-62d9-e8a4-5376-a8d0329ba840: added peer, starting replication: peer=server-daed4bd2-2a47-0c12-e694-c72d421d9532 +2026-11-29T10:29:04.544Z [INFO] server-2b397ab5-62d9-e8a4-5376-a8d0329ba840: added peer, starting replication: peer=server-ae57d575-13ac-c529-ceb1-21b2071153f0 +2026-11-29T10:29:04.544Z [INFO] server-2b397ab5-62d9-e8a4-5376-a8d0329ba840: pipelining replication: peer="{Voter server-ae57d575-13ac-c529-ceb1-21b2071153f0 ae57d575-13ac-c529-ceb1-21b2071153f0}" +2026-11-29T10:29:04.544Z [INFO] server-2b397ab5-62d9-e8a4-5376-a8d0329ba840: pipelining replication: peer="{Voter server-daed4bd2-2a47-0c12-e694-c72d421d9532 daed4bd2-2a47-0c12-e694-c72d421d9532}" +2026-11-29T10:29:05.550Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:04.544898139 +0000 UTC m=+55.279952241 (1 nodes), 765.85396ms from start of poll, 766.0678ms from cluster start. Timeout at 2026-11-29 10:29:05.549904819 +0000 UTC m=+56.284958921, 1.00500668s after stability +2026-11-29T10:29:05.550Z [INFO] server-2b397ab5-62d9-e8a4-5376-a8d0329ba840: starting snapshot up to: index=12 +2026-11-29T10:29:05.550Z [INFO] creating new snapshot: path=/tmp/raft857082962/snapshots/2-12-1795948145550.tmp +2026-11-29T10:29:05.551Z [INFO] server-2b397ab5-62d9-e8a4-5376-a8d0329ba840: snapshot complete up to: index=12 +2026-11-29T10:29:05.551Z [INFO] creating new snapshot: path=/tmp/raft857082962/snapshots/2-113-1795948145551.tmp +2026-11-29T10:29:05.552Z [INFO] server-2b397ab5-62d9-e8a4-5376-a8d0329ba840: copied to local snapshot: bytes=71 +2026-11-29T10:29:05.552Z [INFO] server-2b397ab5-62d9-e8a4-5376-a8d0329ba840: snapshot restore progress: id=2-113-1795948145551 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:05.552Z [INFO] server-2b397ab5-62d9-e8a4-5376-a8d0329ba840: restored user snapshot: index=113 +2026-11-29T10:29:05.552Z [ERROR] server-2b397ab5-62d9-e8a4-5376-a8d0329ba840: failed to get log: index=23 error="log not found" +2026-11-29T10:29:05.552Z [INFO] server-2b397ab5-62d9-e8a4-5376-a8d0329ba840: aborting pipeline replication: peer="{Voter server-ae57d575-13ac-c529-ceb1-21b2071153f0 ae57d575-13ac-c529-ceb1-21b2071153f0}" +2026-11-29T10:29:05.552Z [ERROR] server-2b397ab5-62d9-e8a4-5376-a8d0329ba840: failed to get log: index=23 error="log not found" +2026-11-29T10:29:05.552Z [INFO] server-2b397ab5-62d9-e8a4-5376-a8d0329ba840: aborting pipeline replication: peer="{Voter server-daed4bd2-2a47-0c12-e694-c72d421d9532 daed4bd2-2a47-0c12-e694-c72d421d9532}" +2026-11-29T10:29:05.560Z [ERROR] server-2b397ab5-62d9-e8a4-5376-a8d0329ba840: failed to get log: index=23 error="log not found" +2026-11-29T10:29:05.560Z [INFO] creating new snapshot: path=/tmp/raft3580044949/snapshots/2-113-1795948145560.tmp +2026-11-29T10:29:05.561Z [INFO] server-daed4bd2-2a47-0c12-e694-c72d421d9532: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:05.561Z [INFO] server-daed4bd2-2a47-0c12-e694-c72d421d9532: copied to local snapshot: bytes=71 +2026-11-29T10:29:05.561Z [INFO] server-daed4bd2-2a47-0c12-e694-c72d421d9532: snapshot restore progress: id=2-113-1795948145560 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:05.561Z [INFO] server-daed4bd2-2a47-0c12-e694-c72d421d9532: Installed remote snapshot +2026-11-29T10:29:05.561Z [WARN] server-daed4bd2-2a47-0c12-e694-c72d421d9532: skipping application of old log: index=22 +2026-11-29T10:29:05.561Z [INFO] server-2b397ab5-62d9-e8a4-5376-a8d0329ba840: pipelining replication: peer="{Voter server-daed4bd2-2a47-0c12-e694-c72d421d9532 daed4bd2-2a47-0c12-e694-c72d421d9532}" +2026-11-29T10:29:05.561Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:05.561Z [ERROR] server-2b397ab5-62d9-e8a4-5376-a8d0329ba840: failed to get log: index=23 error="log not found" +2026-11-29T10:29:05.561Z [INFO] creating new snapshot: path=/tmp/raft3092082667/snapshots/2-113-1795948145561.tmp +2026-11-29T10:29:05.561Z [INFO] server-ae57d575-13ac-c529-ceb1-21b2071153f0: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:05.562Z [INFO] server-ae57d575-13ac-c529-ceb1-21b2071153f0: copied to local snapshot: bytes=71 +2026-11-29T10:29:05.562Z [INFO] server-ae57d575-13ac-c529-ceb1-21b2071153f0: snapshot restore progress: id=2-113-1795948145561 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:05.562Z [INFO] server-ae57d575-13ac-c529-ceb1-21b2071153f0: Installed remote snapshot +2026-11-29T10:29:05.562Z [INFO] server-2b397ab5-62d9-e8a4-5376-a8d0329ba840: pipelining replication: peer="{Voter server-ae57d575-13ac-c529-ceb1-21b2071153f0 ae57d575-13ac-c529-ceb1-21b2071153f0}" +2026-11-29T10:29:06.566Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:05.561545779 +0000 UTC m=+56.296599881 (2 nodes), 600ns from start of poll, 1.78271544s from cluster start. Timeout at 2026-11-29 10:29:06.566557059 +0000 UTC m=+57.301611161, 1.00501128s after stability +2026-11-29T10:29:06.577Z [INFO] server-2b397ab5-62d9-e8a4-5376-a8d0329ba840: aborting pipeline replication: peer="{Voter server-ae57d575-13ac-c529-ceb1-21b2071153f0 ae57d575-13ac-c529-ceb1-21b2071153f0}" +2026-11-29T10:29:06.577Z [INFO] server-2b397ab5-62d9-e8a4-5376-a8d0329ba840: aborting pipeline replication: peer="{Voter server-daed4bd2-2a47-0c12-e694-c72d421d9532 daed4bd2-2a47-0c12-e694-c72d421d9532}" === RUN TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_false -2025-10-27T04:03:03.792Z [INFO] server-7c15738f-2b22-134a-8106-216b05a292d5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7c15738f-2b22-134a-8106-216b05a292d5 Address:7c15738f-2b22-134a-8106-216b05a292d5} {Suffrage:Voter ID:server-95bd4532-c906-dc99-d549-05ddd4f0502e Address:95bd4532-c906-dc99-d549-05ddd4f0502e} {Suffrage:Voter ID:server-d6d88c39-a5e5-43fd-0aad-685ee0cf06eb Address:d6d88c39-a5e5-43fd-0aad-685ee0cf06eb}]" -2025-10-27T04:03:03.792Z [INFO] server-95bd4532-c906-dc99-d549-05ddd4f0502e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7c15738f-2b22-134a-8106-216b05a292d5 Address:7c15738f-2b22-134a-8106-216b05a292d5} {Suffrage:Voter ID:server-95bd4532-c906-dc99-d549-05ddd4f0502e Address:95bd4532-c906-dc99-d549-05ddd4f0502e} {Suffrage:Voter ID:server-d6d88c39-a5e5-43fd-0aad-685ee0cf06eb Address:d6d88c39-a5e5-43fd-0aad-685ee0cf06eb}]" -2025-10-27T04:03:03.792Z [INFO] server-7c15738f-2b22-134a-8106-216b05a292d5: entering follower state: follower="Node at 7c15738f-2b22-134a-8106-216b05a292d5 [Follower]" leader-address= leader-id= -2025-10-27T04:03:03.792Z [INFO] server-95bd4532-c906-dc99-d549-05ddd4f0502e: entering follower state: follower="Node at 95bd4532-c906-dc99-d549-05ddd4f0502e [Follower]" leader-address= leader-id= -2025-10-27T04:03:03.792Z [INFO] server-d6d88c39-a5e5-43fd-0aad-685ee0cf06eb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7c15738f-2b22-134a-8106-216b05a292d5 Address:7c15738f-2b22-134a-8106-216b05a292d5} {Suffrage:Voter ID:server-95bd4532-c906-dc99-d549-05ddd4f0502e Address:95bd4532-c906-dc99-d549-05ddd4f0502e} {Suffrage:Voter ID:server-d6d88c39-a5e5-43fd-0aad-685ee0cf06eb Address:d6d88c39-a5e5-43fd-0aad-685ee0cf06eb}]" -2025-10-27T04:03:03.792Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:03.792Z [INFO] server-d6d88c39-a5e5-43fd-0aad-685ee0cf06eb: entering follower state: follower="Node at d6d88c39-a5e5-43fd-0aad-685ee0cf06eb [Follower]" leader-address= leader-id= -2025-10-27T04:03:04.373Z [WARN] server-d6d88c39-a5e5-43fd-0aad-685ee0cf06eb: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:04.373Z [INFO] server-d6d88c39-a5e5-43fd-0aad-685ee0cf06eb: entering candidate state: node="Node at d6d88c39-a5e5-43fd-0aad-685ee0cf06eb [Candidate]" term=2 -2025-10-27T04:03:04.373Z [INFO] server-d6d88c39-a5e5-43fd-0aad-685ee0cf06eb: election won: term=2 tally=2 -2025-10-27T04:03:04.373Z [INFO] server-d6d88c39-a5e5-43fd-0aad-685ee0cf06eb: entering leader state: leader="Node at d6d88c39-a5e5-43fd-0aad-685ee0cf06eb [Leader]" -2025-10-27T04:03:04.373Z [INFO] server-d6d88c39-a5e5-43fd-0aad-685ee0cf06eb: added peer, starting replication: peer=server-7c15738f-2b22-134a-8106-216b05a292d5 -2025-10-27T04:03:04.373Z [INFO] server-d6d88c39-a5e5-43fd-0aad-685ee0cf06eb: added peer, starting replication: peer=server-95bd4532-c906-dc99-d549-05ddd4f0502e -2025-10-27T04:03:04.373Z [INFO] server-d6d88c39-a5e5-43fd-0aad-685ee0cf06eb: pipelining replication: peer="{Voter server-95bd4532-c906-dc99-d549-05ddd4f0502e 95bd4532-c906-dc99-d549-05ddd4f0502e}" -2025-10-27T04:03:04.373Z [INFO] server-d6d88c39-a5e5-43fd-0aad-685ee0cf06eb: pipelining replication: peer="{Voter server-7c15738f-2b22-134a-8106-216b05a292d5 7c15738f-2b22-134a-8106-216b05a292d5}" -2025-10-27T04:03:05.379Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:04.373843464 +0000 UTC m=+59.368715902 (1 nodes), 581.313012ms from start of poll, 581.531774ms from cluster start. Timeout at 2025-10-27 04:03:05.378844904 +0000 UTC m=+60.373717382, 1.00500148s after stability -2025-10-27T04:03:05.380Z [INFO] server-d6d88c39-a5e5-43fd-0aad-685ee0cf06eb: starting snapshot up to: index=12 -2025-10-27T04:03:05.380Z [INFO] creating new snapshot: path=/tmp/raft3163889688/snapshots/2-12-1761537785380.tmp -2025-10-27T04:03:05.380Z [INFO] server-d6d88c39-a5e5-43fd-0aad-685ee0cf06eb: snapshot complete up to: index=12 -2025-10-27T04:03:05.380Z [INFO] creating new snapshot: path=/tmp/raft3163889688/snapshots/2-113-1761537785380.tmp -2025-10-27T04:03:05.381Z [INFO] server-d6d88c39-a5e5-43fd-0aad-685ee0cf06eb: copied to local snapshot: bytes=71 -2025-10-27T04:03:05.381Z [INFO] server-d6d88c39-a5e5-43fd-0aad-685ee0cf06eb: snapshot restore progress: id=2-113-1761537785380 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:05.381Z [INFO] server-d6d88c39-a5e5-43fd-0aad-685ee0cf06eb: removing all old logs from log store -2025-10-27T04:03:05.381Z [INFO] server-d6d88c39-a5e5-43fd-0aad-685ee0cf06eb: compacting logs: from=1 to=22 -2025-10-27T04:03:05.381Z [INFO] server-d6d88c39-a5e5-43fd-0aad-685ee0cf06eb: restored user snapshot: index=113 -2025-10-27T04:03:05.381Z [ERROR] server-d6d88c39-a5e5-43fd-0aad-685ee0cf06eb: failed to get log: index=22 error="log not found" -2025-10-27T04:03:05.381Z [INFO] server-d6d88c39-a5e5-43fd-0aad-685ee0cf06eb: aborting pipeline replication: peer="{Voter server-95bd4532-c906-dc99-d549-05ddd4f0502e 95bd4532-c906-dc99-d549-05ddd4f0502e}" -2025-10-27T04:03:05.381Z [ERROR] server-d6d88c39-a5e5-43fd-0aad-685ee0cf06eb: failed to get log: index=22 error="log not found" -2025-10-27T04:03:05.382Z [INFO] server-d6d88c39-a5e5-43fd-0aad-685ee0cf06eb: aborting pipeline replication: peer="{Voter server-7c15738f-2b22-134a-8106-216b05a292d5 7c15738f-2b22-134a-8106-216b05a292d5}" -2025-10-27T04:03:05.389Z [ERROR] server-d6d88c39-a5e5-43fd-0aad-685ee0cf06eb: failed to get log: index=22 error="log not found" -2025-10-27T04:03:05.389Z [INFO] creating new snapshot: path=/tmp/raft3213798598/snapshots/2-113-1761537785389.tmp -2025-10-27T04:03:05.389Z [INFO] server-95bd4532-c906-dc99-d549-05ddd4f0502e: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:05.389Z [INFO] server-95bd4532-c906-dc99-d549-05ddd4f0502e: copied to local snapshot: bytes=71 -2025-10-27T04:03:05.389Z [ERROR] server-d6d88c39-a5e5-43fd-0aad-685ee0cf06eb: failed to get log: index=22 error="log not found" -2025-10-27T04:03:05.390Z [INFO] creating new snapshot: path=/tmp/raft773167406/snapshots/2-113-1761537785390.tmp -2025-10-27T04:03:05.390Z [INFO] server-95bd4532-c906-dc99-d549-05ddd4f0502e: snapshot restore progress: id=2-113-1761537785389 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:05.390Z [INFO] server-95bd4532-c906-dc99-d549-05ddd4f0502e: removing all old logs from log store -2025-10-27T04:03:05.390Z [INFO] server-95bd4532-c906-dc99-d549-05ddd4f0502e: compacting logs: from=1 to=22 -2025-10-27T04:03:05.390Z [INFO] server-95bd4532-c906-dc99-d549-05ddd4f0502e: Installed remote snapshot -2025-10-27T04:03:05.390Z [INFO] server-7c15738f-2b22-134a-8106-216b05a292d5: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:05.390Z [INFO] server-d6d88c39-a5e5-43fd-0aad-685ee0cf06eb: pipelining replication: peer="{Voter server-95bd4532-c906-dc99-d549-05ddd4f0502e 95bd4532-c906-dc99-d549-05ddd4f0502e}" -2025-10-27T04:03:05.390Z [INFO] server-7c15738f-2b22-134a-8106-216b05a292d5: copied to local snapshot: bytes=71 -2025-10-27T04:03:05.390Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:03:05.390Z [INFO] server-7c15738f-2b22-134a-8106-216b05a292d5: snapshot restore progress: id=2-113-1761537785390 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:05.390Z [INFO] server-7c15738f-2b22-134a-8106-216b05a292d5: removing all old logs from log store -2025-10-27T04:03:05.390Z [INFO] server-7c15738f-2b22-134a-8106-216b05a292d5: compacting logs: from=1 to=22 -2025-10-27T04:03:05.390Z [INFO] server-7c15738f-2b22-134a-8106-216b05a292d5: Installed remote snapshot -2025-10-27T04:03:05.390Z [INFO] server-d6d88c39-a5e5-43fd-0aad-685ee0cf06eb: pipelining replication: peer="{Voter server-7c15738f-2b22-134a-8106-216b05a292d5 7c15738f-2b22-134a-8106-216b05a292d5}" -2025-10-27T04:03:06.395Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:03:05.390247209 +0000 UTC m=+60.385119607 (2 nodes), 560ns from start of poll, 1.597935479s from cluster start. Timeout at 2025-10-27 04:03:06.39526533 +0000 UTC m=+61.390137768, 1.005018161s after stability -2025-10-27T04:03:06.407Z [INFO] server-d6d88c39-a5e5-43fd-0aad-685ee0cf06eb: aborting pipeline replication: peer="{Voter server-95bd4532-c906-dc99-d549-05ddd4f0502e 95bd4532-c906-dc99-d549-05ddd4f0502e}" -2025-10-27T04:03:06.407Z [INFO] server-d6d88c39-a5e5-43fd-0aad-685ee0cf06eb: aborting pipeline replication: peer="{Voter server-7c15738f-2b22-134a-8106-216b05a292d5 7c15738f-2b22-134a-8106-216b05a292d5}" +2026-11-29T10:29:06.578Z [INFO] server-325a8fd4-d6e7-fd81-1a40-6cb19aabc911: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-325a8fd4-d6e7-fd81-1a40-6cb19aabc911 Address:325a8fd4-d6e7-fd81-1a40-6cb19aabc911} {Suffrage:Voter ID:server-c9aae2f8-6fa0-7454-860e-8351615186cb Address:c9aae2f8-6fa0-7454-860e-8351615186cb} {Suffrage:Voter ID:server-35ba8c15-4220-f863-a46c-f934685ace33 Address:35ba8c15-4220-f863-a46c-f934685ace33}]" +2026-11-29T10:29:06.578Z [INFO] server-c9aae2f8-6fa0-7454-860e-8351615186cb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-325a8fd4-d6e7-fd81-1a40-6cb19aabc911 Address:325a8fd4-d6e7-fd81-1a40-6cb19aabc911} {Suffrage:Voter ID:server-c9aae2f8-6fa0-7454-860e-8351615186cb Address:c9aae2f8-6fa0-7454-860e-8351615186cb} {Suffrage:Voter ID:server-35ba8c15-4220-f863-a46c-f934685ace33 Address:35ba8c15-4220-f863-a46c-f934685ace33}]" +2026-11-29T10:29:06.578Z [INFO] server-325a8fd4-d6e7-fd81-1a40-6cb19aabc911: entering follower state: follower="Node at 325a8fd4-d6e7-fd81-1a40-6cb19aabc911 [Follower]" leader-address= leader-id= +2026-11-29T10:29:06.578Z [INFO] server-c9aae2f8-6fa0-7454-860e-8351615186cb: entering follower state: follower="Node at c9aae2f8-6fa0-7454-860e-8351615186cb [Follower]" leader-address= leader-id= +2026-11-29T10:29:06.578Z [INFO] server-35ba8c15-4220-f863-a46c-f934685ace33: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-325a8fd4-d6e7-fd81-1a40-6cb19aabc911 Address:325a8fd4-d6e7-fd81-1a40-6cb19aabc911} {Suffrage:Voter ID:server-c9aae2f8-6fa0-7454-860e-8351615186cb Address:c9aae2f8-6fa0-7454-860e-8351615186cb} {Suffrage:Voter ID:server-35ba8c15-4220-f863-a46c-f934685ace33 Address:35ba8c15-4220-f863-a46c-f934685ace33}]" +2026-11-29T10:29:06.578Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:06.578Z [INFO] server-35ba8c15-4220-f863-a46c-f934685ace33: entering follower state: follower="Node at 35ba8c15-4220-f863-a46c-f934685ace33 [Follower]" leader-address= leader-id= +2026-11-29T10:29:07.083Z [WARN] server-c9aae2f8-6fa0-7454-860e-8351615186cb: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:07.083Z [INFO] server-c9aae2f8-6fa0-7454-860e-8351615186cb: entering candidate state: node="Node at c9aae2f8-6fa0-7454-860e-8351615186cb [Candidate]" term=2 +2026-11-29T10:29:07.083Z [INFO] server-c9aae2f8-6fa0-7454-860e-8351615186cb: election won: term=2 tally=2 +2026-11-29T10:29:07.083Z [INFO] server-c9aae2f8-6fa0-7454-860e-8351615186cb: entering leader state: leader="Node at c9aae2f8-6fa0-7454-860e-8351615186cb [Leader]" +2026-11-29T10:29:07.083Z [INFO] server-c9aae2f8-6fa0-7454-860e-8351615186cb: added peer, starting replication: peer=server-325a8fd4-d6e7-fd81-1a40-6cb19aabc911 +2026-11-29T10:29:07.083Z [INFO] server-c9aae2f8-6fa0-7454-860e-8351615186cb: added peer, starting replication: peer=server-35ba8c15-4220-f863-a46c-f934685ace33 +2026-11-29T10:29:07.083Z [INFO] server-c9aae2f8-6fa0-7454-860e-8351615186cb: pipelining replication: peer="{Voter server-325a8fd4-d6e7-fd81-1a40-6cb19aabc911 325a8fd4-d6e7-fd81-1a40-6cb19aabc911}" +2026-11-29T10:29:07.083Z [INFO] server-c9aae2f8-6fa0-7454-860e-8351615186cb: pipelining replication: peer="{Voter server-35ba8c15-4220-f863-a46c-f934685ace33 35ba8c15-4220-f863-a46c-f934685ace33}" +2026-11-29T10:29:08.088Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:07.083224979 +0000 UTC m=+57.818279081 (1 nodes), 504.51428ms from start of poll, 504.76264ms from cluster start. Timeout at 2026-11-29 10:29:08.088232899 +0000 UTC m=+58.823287001, 1.00500792s after stability +2026-11-29T10:29:08.088Z [INFO] server-c9aae2f8-6fa0-7454-860e-8351615186cb: starting snapshot up to: index=12 +2026-11-29T10:29:08.088Z [INFO] creating new snapshot: path=/tmp/raft32290024/snapshots/2-12-1795948148088.tmp +2026-11-29T10:29:08.089Z [INFO] server-c9aae2f8-6fa0-7454-860e-8351615186cb: snapshot complete up to: index=12 +2026-11-29T10:29:08.089Z [INFO] creating new snapshot: path=/tmp/raft32290024/snapshots/2-113-1795948148089.tmp +2026-11-29T10:29:08.089Z [INFO] server-c9aae2f8-6fa0-7454-860e-8351615186cb: copied to local snapshot: bytes=71 +2026-11-29T10:29:08.089Z [INFO] server-c9aae2f8-6fa0-7454-860e-8351615186cb: snapshot restore progress: id=2-113-1795948148089 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:08.089Z [INFO] server-c9aae2f8-6fa0-7454-860e-8351615186cb: removing all old logs from log store +2026-11-29T10:29:08.089Z [INFO] server-c9aae2f8-6fa0-7454-860e-8351615186cb: compacting logs: from=1 to=22 +2026-11-29T10:29:08.089Z [INFO] server-c9aae2f8-6fa0-7454-860e-8351615186cb: restored user snapshot: index=113 +2026-11-29T10:29:08.089Z [ERROR] server-c9aae2f8-6fa0-7454-860e-8351615186cb: failed to get log: index=22 error="log not found" +2026-11-29T10:29:08.089Z [INFO] server-c9aae2f8-6fa0-7454-860e-8351615186cb: aborting pipeline replication: peer="{Voter server-35ba8c15-4220-f863-a46c-f934685ace33 35ba8c15-4220-f863-a46c-f934685ace33}" +2026-11-29T10:29:08.089Z [ERROR] server-c9aae2f8-6fa0-7454-860e-8351615186cb: failed to get log: index=22 error="log not found" +2026-11-29T10:29:08.089Z [INFO] server-c9aae2f8-6fa0-7454-860e-8351615186cb: aborting pipeline replication: peer="{Voter server-325a8fd4-d6e7-fd81-1a40-6cb19aabc911 325a8fd4-d6e7-fd81-1a40-6cb19aabc911}" +2026-11-29T10:29:08.096Z [ERROR] server-c9aae2f8-6fa0-7454-860e-8351615186cb: failed to get log: index=22 error="log not found" +2026-11-29T10:29:08.096Z [INFO] creating new snapshot: path=/tmp/raft668702767/snapshots/2-113-1795948148096.tmp +2026-11-29T10:29:08.096Z [INFO] server-325a8fd4-d6e7-fd81-1a40-6cb19aabc911: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:08.096Z [INFO] server-325a8fd4-d6e7-fd81-1a40-6cb19aabc911: copied to local snapshot: bytes=71 +2026-11-29T10:29:08.096Z [INFO] server-325a8fd4-d6e7-fd81-1a40-6cb19aabc911: snapshot restore progress: id=2-113-1795948148096 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:08.097Z [INFO] server-325a8fd4-d6e7-fd81-1a40-6cb19aabc911: removing all old logs from log store +2026-11-29T10:29:08.097Z [INFO] server-325a8fd4-d6e7-fd81-1a40-6cb19aabc911: compacting logs: from=1 to=22 +2026-11-29T10:29:08.097Z [INFO] server-325a8fd4-d6e7-fd81-1a40-6cb19aabc911: Installed remote snapshot +2026-11-29T10:29:08.097Z [WARN] server-325a8fd4-d6e7-fd81-1a40-6cb19aabc911: skipping application of old log: index=22 +2026-11-29T10:29:08.097Z [INFO] server-c9aae2f8-6fa0-7454-860e-8351615186cb: pipelining replication: peer="{Voter server-325a8fd4-d6e7-fd81-1a40-6cb19aabc911 325a8fd4-d6e7-fd81-1a40-6cb19aabc911}" +2026-11-29T10:29:08.097Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:08.098Z [ERROR] server-c9aae2f8-6fa0-7454-860e-8351615186cb: failed to get log: index=22 error="log not found" +2026-11-29T10:29:08.098Z [INFO] creating new snapshot: path=/tmp/raft3426317206/snapshots/2-113-1795948148098.tmp +2026-11-29T10:29:08.098Z [INFO] server-35ba8c15-4220-f863-a46c-f934685ace33: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:08.098Z [INFO] server-35ba8c15-4220-f863-a46c-f934685ace33: copied to local snapshot: bytes=71 +2026-11-29T10:29:08.098Z [INFO] server-35ba8c15-4220-f863-a46c-f934685ace33: snapshot restore progress: id=2-113-1795948148098 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:08.099Z [INFO] server-35ba8c15-4220-f863-a46c-f934685ace33: removing all old logs from log store +2026-11-29T10:29:08.099Z [INFO] server-35ba8c15-4220-f863-a46c-f934685ace33: compacting logs: from=1 to=22 +2026-11-29T10:29:08.099Z [INFO] server-35ba8c15-4220-f863-a46c-f934685ace33: Installed remote snapshot +2026-11-29T10:29:08.099Z [INFO] server-c9aae2f8-6fa0-7454-860e-8351615186cb: pipelining replication: peer="{Voter server-35ba8c15-4220-f863-a46c-f934685ace33 35ba8c15-4220-f863-a46c-f934685ace33}" +2026-11-29T10:29:09.103Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:08.097228699 +0000 UTC m=+58.832282801 (2 nodes), 760ns from start of poll, 1.51876636s from cluster start. Timeout at 2026-11-29 10:29:09.102244139 +0000 UTC m=+59.837298201, 1.0050154s after stability +2026-11-29T10:29:09.115Z [INFO] server-c9aae2f8-6fa0-7454-860e-8351615186cb: aborting pipeline replication: peer="{Voter server-35ba8c15-4220-f863-a46c-f934685ace33 35ba8c15-4220-f863-a46c-f934685ace33}" +2026-11-29T10:29:09.115Z [INFO] server-c9aae2f8-6fa0-7454-860e-8351615186cb: aborting pipeline replication: peer="{Voter server-325a8fd4-d6e7-fd81-1a40-6cb19aabc911 325a8fd4-d6e7-fd81-1a40-6cb19aabc911}" +2026-11-29T10:29:09.615Z [ERROR] server-c9aae2f8-6fa0-7454-860e-8351615186cb: failed to heartbeat to: peer=35ba8c15-4220-f863-a46c-f934685ace33 backoff time=10ms error="command timed out" === RUN TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_true -2025-10-27T04:03:06.410Z [INFO] server-2957832c-4b14-808d-80d8-ec5b53156265: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2957832c-4b14-808d-80d8-ec5b53156265 Address:2957832c-4b14-808d-80d8-ec5b53156265} {Suffrage:Voter ID:server-7de98435-7dd2-086d-3b2f-a3be36a97cba Address:7de98435-7dd2-086d-3b2f-a3be36a97cba} {Suffrage:Voter ID:server-c6dd5a7e-9000-9bc0-665a-4351061b8f32 Address:c6dd5a7e-9000-9bc0-665a-4351061b8f32}]" -2025-10-27T04:03:06.410Z [INFO] server-7de98435-7dd2-086d-3b2f-a3be36a97cba: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2957832c-4b14-808d-80d8-ec5b53156265 Address:2957832c-4b14-808d-80d8-ec5b53156265} {Suffrage:Voter ID:server-7de98435-7dd2-086d-3b2f-a3be36a97cba Address:7de98435-7dd2-086d-3b2f-a3be36a97cba} {Suffrage:Voter ID:server-c6dd5a7e-9000-9bc0-665a-4351061b8f32 Address:c6dd5a7e-9000-9bc0-665a-4351061b8f32}]" -2025-10-27T04:03:06.410Z [INFO] server-c6dd5a7e-9000-9bc0-665a-4351061b8f32: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2957832c-4b14-808d-80d8-ec5b53156265 Address:2957832c-4b14-808d-80d8-ec5b53156265} {Suffrage:Voter ID:server-7de98435-7dd2-086d-3b2f-a3be36a97cba Address:7de98435-7dd2-086d-3b2f-a3be36a97cba} {Suffrage:Voter ID:server-c6dd5a7e-9000-9bc0-665a-4351061b8f32 Address:c6dd5a7e-9000-9bc0-665a-4351061b8f32}]" -2025-10-27T04:03:06.410Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:06.410Z [INFO] server-2957832c-4b14-808d-80d8-ec5b53156265: entering follower state: follower="Node at 2957832c-4b14-808d-80d8-ec5b53156265 [Follower]" leader-address= leader-id= -2025-10-27T04:03:06.410Z [INFO] server-7de98435-7dd2-086d-3b2f-a3be36a97cba: entering follower state: follower="Node at 7de98435-7dd2-086d-3b2f-a3be36a97cba [Follower]" leader-address= leader-id= -2025-10-27T04:03:06.410Z [INFO] server-c6dd5a7e-9000-9bc0-665a-4351061b8f32: entering follower state: follower="Node at c6dd5a7e-9000-9bc0-665a-4351061b8f32 [Follower]" leader-address= leader-id= -2025-10-27T04:03:07.311Z [WARN] server-2957832c-4b14-808d-80d8-ec5b53156265: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:07.311Z [INFO] server-2957832c-4b14-808d-80d8-ec5b53156265: entering candidate state: node="Node at 2957832c-4b14-808d-80d8-ec5b53156265 [Candidate]" term=2 -2025-10-27T04:03:07.311Z [INFO] server-2957832c-4b14-808d-80d8-ec5b53156265: election won: term=2 tally=2 -2025-10-27T04:03:07.311Z [INFO] server-2957832c-4b14-808d-80d8-ec5b53156265: entering leader state: leader="Node at 2957832c-4b14-808d-80d8-ec5b53156265 [Leader]" -2025-10-27T04:03:07.311Z [INFO] server-2957832c-4b14-808d-80d8-ec5b53156265: added peer, starting replication: peer=server-7de98435-7dd2-086d-3b2f-a3be36a97cba -2025-10-27T04:03:07.311Z [INFO] server-2957832c-4b14-808d-80d8-ec5b53156265: added peer, starting replication: peer=server-c6dd5a7e-9000-9bc0-665a-4351061b8f32 -2025-10-27T04:03:07.311Z [INFO] server-2957832c-4b14-808d-80d8-ec5b53156265: pipelining replication: peer="{Voter server-7de98435-7dd2-086d-3b2f-a3be36a97cba 7de98435-7dd2-086d-3b2f-a3be36a97cba}" -2025-10-27T04:03:07.311Z [INFO] server-2957832c-4b14-808d-80d8-ec5b53156265: pipelining replication: peer="{Voter server-c6dd5a7e-9000-9bc0-665a-4351061b8f32 c6dd5a7e-9000-9bc0-665a-4351061b8f32}" -2025-10-27T04:03:08.317Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:07.311567858 +0000 UTC m=+62.306440336 (1 nodes), 901.181238ms from start of poll, 903.836416ms from cluster start. Timeout at 2025-10-27 04:03:08.316569378 +0000 UTC m=+63.311441816, 1.00500148s after stability -2025-10-27T04:03:08.318Z [INFO] server-2957832c-4b14-808d-80d8-ec5b53156265: starting snapshot up to: index=12 -2025-10-27T04:03:08.318Z [INFO] creating new snapshot: path=/tmp/raft1494243017/snapshots/2-12-1761537788318.tmp -2025-10-27T04:03:08.318Z [INFO] server-2957832c-4b14-808d-80d8-ec5b53156265: snapshot complete up to: index=12 -2025-10-27T04:03:08.319Z [INFO] server-cff587fb-9b3f-bb10-e160-1b1574702331: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cff587fb-9b3f-bb10-e160-1b1574702331 Address:cff587fb-9b3f-bb10-e160-1b1574702331} {Suffrage:Voter ID:server-26a1ff3f-bf41-47a5-e0ac-6871c87b5520 Address:26a1ff3f-bf41-47a5-e0ac-6871c87b5520} {Suffrage:Voter ID:server-e41f1fe9-ff76-d61c-b924-d5a6a9cc43e1 Address:e41f1fe9-ff76-d61c-b924-d5a6a9cc43e1}]" -2025-10-27T04:03:08.319Z [INFO] server-26a1ff3f-bf41-47a5-e0ac-6871c87b5520: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cff587fb-9b3f-bb10-e160-1b1574702331 Address:cff587fb-9b3f-bb10-e160-1b1574702331} {Suffrage:Voter ID:server-26a1ff3f-bf41-47a5-e0ac-6871c87b5520 Address:26a1ff3f-bf41-47a5-e0ac-6871c87b5520} {Suffrage:Voter ID:server-e41f1fe9-ff76-d61c-b924-d5a6a9cc43e1 Address:e41f1fe9-ff76-d61c-b924-d5a6a9cc43e1}]" -2025-10-27T04:03:08.319Z [INFO] server-cff587fb-9b3f-bb10-e160-1b1574702331: entering follower state: follower="Node at cff587fb-9b3f-bb10-e160-1b1574702331 [Follower]" leader-address= leader-id= -2025-10-27T04:03:08.319Z [INFO] server-e41f1fe9-ff76-d61c-b924-d5a6a9cc43e1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cff587fb-9b3f-bb10-e160-1b1574702331 Address:cff587fb-9b3f-bb10-e160-1b1574702331} {Suffrage:Voter ID:server-26a1ff3f-bf41-47a5-e0ac-6871c87b5520 Address:26a1ff3f-bf41-47a5-e0ac-6871c87b5520} {Suffrage:Voter ID:server-e41f1fe9-ff76-d61c-b924-d5a6a9cc43e1 Address:e41f1fe9-ff76-d61c-b924-d5a6a9cc43e1}]" -2025-10-27T04:03:08.319Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:08.319Z [INFO] server-e41f1fe9-ff76-d61c-b924-d5a6a9cc43e1: entering follower state: follower="Node at e41f1fe9-ff76-d61c-b924-d5a6a9cc43e1 [Follower]" leader-address= leader-id= -2025-10-27T04:03:08.319Z [INFO] server-26a1ff3f-bf41-47a5-e0ac-6871c87b5520: entering follower state: follower="Node at 26a1ff3f-bf41-47a5-e0ac-6871c87b5520 [Follower]" leader-address= leader-id= -2025-10-27T04:03:09.048Z [WARN] server-e41f1fe9-ff76-d61c-b924-d5a6a9cc43e1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:09.048Z [INFO] server-e41f1fe9-ff76-d61c-b924-d5a6a9cc43e1: entering candidate state: node="Node at e41f1fe9-ff76-d61c-b924-d5a6a9cc43e1 [Candidate]" term=2 -2025-10-27T04:03:09.048Z [INFO] server-e41f1fe9-ff76-d61c-b924-d5a6a9cc43e1: election won: term=2 tally=2 -2025-10-27T04:03:09.048Z [INFO] server-e41f1fe9-ff76-d61c-b924-d5a6a9cc43e1: entering leader state: leader="Node at e41f1fe9-ff76-d61c-b924-d5a6a9cc43e1 [Leader]" -2025-10-27T04:03:09.048Z [INFO] server-e41f1fe9-ff76-d61c-b924-d5a6a9cc43e1: added peer, starting replication: peer=server-cff587fb-9b3f-bb10-e160-1b1574702331 -2025-10-27T04:03:09.048Z [INFO] server-e41f1fe9-ff76-d61c-b924-d5a6a9cc43e1: added peer, starting replication: peer=server-26a1ff3f-bf41-47a5-e0ac-6871c87b5520 -2025-10-27T04:03:09.048Z [INFO] server-e41f1fe9-ff76-d61c-b924-d5a6a9cc43e1: pipelining replication: peer="{Voter server-26a1ff3f-bf41-47a5-e0ac-6871c87b5520 26a1ff3f-bf41-47a5-e0ac-6871c87b5520}" -2025-10-27T04:03:09.048Z [INFO] server-e41f1fe9-ff76-d61c-b924-d5a6a9cc43e1: pipelining replication: peer="{Voter server-cff587fb-9b3f-bb10-e160-1b1574702331 cff587fb-9b3f-bb10-e160-1b1574702331}" -2025-10-27T04:03:10.053Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:09.048375205 +0000 UTC m=+64.043247643 (1 nodes), 729.056221ms from start of poll, 729.299943ms from cluster start. Timeout at 2025-10-27 04:03:10.053388045 +0000 UTC m=+65.048260483, 1.00501284s after stability -2025-10-27T04:03:10.053Z [INFO] creating new snapshot: path=/tmp/raft3399703593/snapshots/2-113-1761537790053.tmp -2025-10-27T04:03:10.053Z [INFO] server-e41f1fe9-ff76-d61c-b924-d5a6a9cc43e1: copied to local snapshot: bytes=71 -2025-10-27T04:03:10.054Z [INFO] server-e41f1fe9-ff76-d61c-b924-d5a6a9cc43e1: snapshot restore progress: id=2-113-1761537790053 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:10.054Z [INFO] server-e41f1fe9-ff76-d61c-b924-d5a6a9cc43e1: restored user snapshot: index=113 -2025-10-27T04:03:10.054Z [ERROR] server-e41f1fe9-ff76-d61c-b924-d5a6a9cc43e1: failed to get log: index=3 error="log not found" -2025-10-27T04:03:10.054Z [INFO] server-e41f1fe9-ff76-d61c-b924-d5a6a9cc43e1: aborting pipeline replication: peer="{Voter server-cff587fb-9b3f-bb10-e160-1b1574702331 cff587fb-9b3f-bb10-e160-1b1574702331}" -2025-10-27T04:03:10.054Z [ERROR] server-e41f1fe9-ff76-d61c-b924-d5a6a9cc43e1: failed to get log: index=3 error="log not found" -2025-10-27T04:03:10.054Z [INFO] server-e41f1fe9-ff76-d61c-b924-d5a6a9cc43e1: aborting pipeline replication: peer="{Voter server-26a1ff3f-bf41-47a5-e0ac-6871c87b5520 26a1ff3f-bf41-47a5-e0ac-6871c87b5520}" -2025-10-27T04:03:10.062Z [ERROR] server-e41f1fe9-ff76-d61c-b924-d5a6a9cc43e1: failed to get log: index=3 error="log not found" -2025-10-27T04:03:10.062Z [INFO] creating new snapshot: path=/tmp/raft1655078517/snapshots/2-113-1761537790062.tmp -2025-10-27T04:03:10.062Z [INFO] server-26a1ff3f-bf41-47a5-e0ac-6871c87b5520: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:10.062Z [INFO] server-26a1ff3f-bf41-47a5-e0ac-6871c87b5520: copied to local snapshot: bytes=71 -2025-10-27T04:03:10.062Z [INFO] server-26a1ff3f-bf41-47a5-e0ac-6871c87b5520: snapshot restore progress: id=2-113-1761537790062 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:10.062Z [INFO] server-26a1ff3f-bf41-47a5-e0ac-6871c87b5520: Installed remote snapshot -2025-10-27T04:03:10.062Z [INFO] server-e41f1fe9-ff76-d61c-b924-d5a6a9cc43e1: pipelining replication: peer="{Voter server-26a1ff3f-bf41-47a5-e0ac-6871c87b5520 26a1ff3f-bf41-47a5-e0ac-6871c87b5520}" -2025-10-27T04:03:10.063Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:03:10.064Z [ERROR] server-e41f1fe9-ff76-d61c-b924-d5a6a9cc43e1: failed to get log: index=3 error="log not found" -2025-10-27T04:03:10.064Z [INFO] creating new snapshot: path=/tmp/raft3469721545/snapshots/2-113-1761537790064.tmp -2025-10-27T04:03:10.064Z [INFO] server-cff587fb-9b3f-bb10-e160-1b1574702331: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:10.065Z [INFO] server-cff587fb-9b3f-bb10-e160-1b1574702331: copied to local snapshot: bytes=71 -2025-10-27T04:03:10.065Z [INFO] server-cff587fb-9b3f-bb10-e160-1b1574702331: snapshot restore progress: id=2-113-1761537790064 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:10.065Z [INFO] server-cff587fb-9b3f-bb10-e160-1b1574702331: Installed remote snapshot -2025-10-27T04:03:10.065Z [INFO] server-e41f1fe9-ff76-d61c-b924-d5a6a9cc43e1: pipelining replication: peer="{Voter server-cff587fb-9b3f-bb10-e160-1b1574702331 cff587fb-9b3f-bb10-e160-1b1574702331}" -2025-10-27T04:03:11.069Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:03:10.063046339 +0000 UTC m=+65.057918777 (2 nodes), 480ns from start of poll, 1.743971077s from cluster start. Timeout at 2025-10-27 04:03:11.068060099 +0000 UTC m=+66.062932537, 1.00501376s after stability -2025-10-27T04:03:11.080Z [INFO] server-2957832c-4b14-808d-80d8-ec5b53156265: aborting pipeline replication: peer="{Voter server-7de98435-7dd2-086d-3b2f-a3be36a97cba 7de98435-7dd2-086d-3b2f-a3be36a97cba}" -2025-10-27T04:03:11.080Z [INFO] server-2957832c-4b14-808d-80d8-ec5b53156265: aborting pipeline replication: peer="{Voter server-c6dd5a7e-9000-9bc0-665a-4351061b8f32 c6dd5a7e-9000-9bc0-665a-4351061b8f32}" +2026-11-29T10:29:09.616Z [INFO] server-3d20655f-8894-df2c-b354-3a4edd39cd38: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3d20655f-8894-df2c-b354-3a4edd39cd38 Address:3d20655f-8894-df2c-b354-3a4edd39cd38} {Suffrage:Voter ID:server-1e9aa434-3ec6-ad64-c0c1-69bd158acf5d Address:1e9aa434-3ec6-ad64-c0c1-69bd158acf5d} {Suffrage:Voter ID:server-7e647547-781a-494b-3855-4bc5644d8592 Address:7e647547-781a-494b-3855-4bc5644d8592}]" +2026-11-29T10:29:09.616Z [INFO] server-3d20655f-8894-df2c-b354-3a4edd39cd38: entering follower state: follower="Node at 3d20655f-8894-df2c-b354-3a4edd39cd38 [Follower]" leader-address= leader-id= +2026-11-29T10:29:09.616Z [INFO] server-1e9aa434-3ec6-ad64-c0c1-69bd158acf5d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3d20655f-8894-df2c-b354-3a4edd39cd38 Address:3d20655f-8894-df2c-b354-3a4edd39cd38} {Suffrage:Voter ID:server-1e9aa434-3ec6-ad64-c0c1-69bd158acf5d Address:1e9aa434-3ec6-ad64-c0c1-69bd158acf5d} {Suffrage:Voter ID:server-7e647547-781a-494b-3855-4bc5644d8592 Address:7e647547-781a-494b-3855-4bc5644d8592}]" +2026-11-29T10:29:09.616Z [INFO] server-7e647547-781a-494b-3855-4bc5644d8592: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3d20655f-8894-df2c-b354-3a4edd39cd38 Address:3d20655f-8894-df2c-b354-3a4edd39cd38} {Suffrage:Voter ID:server-1e9aa434-3ec6-ad64-c0c1-69bd158acf5d Address:1e9aa434-3ec6-ad64-c0c1-69bd158acf5d} {Suffrage:Voter ID:server-7e647547-781a-494b-3855-4bc5644d8592 Address:7e647547-781a-494b-3855-4bc5644d8592}]" +2026-11-29T10:29:09.616Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:09.616Z [INFO] server-1e9aa434-3ec6-ad64-c0c1-69bd158acf5d: entering follower state: follower="Node at 1e9aa434-3ec6-ad64-c0c1-69bd158acf5d [Follower]" leader-address= leader-id= +2026-11-29T10:29:09.616Z [INFO] server-7e647547-781a-494b-3855-4bc5644d8592: entering follower state: follower="Node at 7e647547-781a-494b-3855-4bc5644d8592 [Follower]" leader-address= leader-id= +2026-11-29T10:29:10.134Z [WARN] server-3d20655f-8894-df2c-b354-3a4edd39cd38: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:10.134Z [INFO] server-3d20655f-8894-df2c-b354-3a4edd39cd38: entering candidate state: node="Node at 3d20655f-8894-df2c-b354-3a4edd39cd38 [Candidate]" term=2 +2026-11-29T10:29:10.134Z [INFO] server-3d20655f-8894-df2c-b354-3a4edd39cd38: election won: term=2 tally=2 +2026-11-29T10:29:10.134Z [INFO] server-3d20655f-8894-df2c-b354-3a4edd39cd38: entering leader state: leader="Node at 3d20655f-8894-df2c-b354-3a4edd39cd38 [Leader]" +2026-11-29T10:29:10.134Z [INFO] server-3d20655f-8894-df2c-b354-3a4edd39cd38: added peer, starting replication: peer=server-1e9aa434-3ec6-ad64-c0c1-69bd158acf5d +2026-11-29T10:29:10.134Z [INFO] server-3d20655f-8894-df2c-b354-3a4edd39cd38: added peer, starting replication: peer=server-7e647547-781a-494b-3855-4bc5644d8592 +2026-11-29T10:29:10.134Z [INFO] server-3d20655f-8894-df2c-b354-3a4edd39cd38: pipelining replication: peer="{Voter server-7e647547-781a-494b-3855-4bc5644d8592 7e647547-781a-494b-3855-4bc5644d8592}" +2026-11-29T10:29:10.134Z [INFO] server-3d20655f-8894-df2c-b354-3a4edd39cd38: pipelining replication: peer="{Voter server-1e9aa434-3ec6-ad64-c0c1-69bd158acf5d 1e9aa434-3ec6-ad64-c0c1-69bd158acf5d}" +2026-11-29T10:29:11.142Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:10.134720059 +0000 UTC m=+60.869774161 (1 nodes), 518.00576ms from start of poll, 518.35604ms from cluster start. Timeout at 2026-11-29 10:29:11.139799419 +0000 UTC m=+61.874853521, 1.00507936s after stability +2026-11-29T10:29:11.142Z [INFO] server-3d20655f-8894-df2c-b354-3a4edd39cd38: starting snapshot up to: index=12 +2026-11-29T10:29:11.142Z [INFO] creating new snapshot: path=/tmp/raft2025301635/snapshots/2-12-1795948151142.tmp +2026-11-29T10:29:11.143Z [INFO] server-3d20655f-8894-df2c-b354-3a4edd39cd38: snapshot complete up to: index=12 +2026-11-29T10:29:11.143Z [INFO] server-043516c5-4ec9-dbe7-4e23-57a03acef719: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-043516c5-4ec9-dbe7-4e23-57a03acef719 Address:043516c5-4ec9-dbe7-4e23-57a03acef719} {Suffrage:Voter ID:server-c1cdfcaa-f7d5-17d2-25d7-0a6ccb70eab2 Address:c1cdfcaa-f7d5-17d2-25d7-0a6ccb70eab2} {Suffrage:Voter ID:server-c300d831-401a-1055-6b36-6275e584472a Address:c300d831-401a-1055-6b36-6275e584472a}]" +2026-11-29T10:29:11.143Z [INFO] server-c1cdfcaa-f7d5-17d2-25d7-0a6ccb70eab2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-043516c5-4ec9-dbe7-4e23-57a03acef719 Address:043516c5-4ec9-dbe7-4e23-57a03acef719} {Suffrage:Voter ID:server-c1cdfcaa-f7d5-17d2-25d7-0a6ccb70eab2 Address:c1cdfcaa-f7d5-17d2-25d7-0a6ccb70eab2} {Suffrage:Voter ID:server-c300d831-401a-1055-6b36-6275e584472a Address:c300d831-401a-1055-6b36-6275e584472a}]" +2026-11-29T10:29:11.143Z [INFO] server-c300d831-401a-1055-6b36-6275e584472a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-043516c5-4ec9-dbe7-4e23-57a03acef719 Address:043516c5-4ec9-dbe7-4e23-57a03acef719} {Suffrage:Voter ID:server-c1cdfcaa-f7d5-17d2-25d7-0a6ccb70eab2 Address:c1cdfcaa-f7d5-17d2-25d7-0a6ccb70eab2} {Suffrage:Voter ID:server-c300d831-401a-1055-6b36-6275e584472a Address:c300d831-401a-1055-6b36-6275e584472a}]" +2026-11-29T10:29:11.143Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:11.143Z [INFO] server-043516c5-4ec9-dbe7-4e23-57a03acef719: entering follower state: follower="Node at 043516c5-4ec9-dbe7-4e23-57a03acef719 [Follower]" leader-address= leader-id= +2026-11-29T10:29:11.143Z [INFO] server-c1cdfcaa-f7d5-17d2-25d7-0a6ccb70eab2: entering follower state: follower="Node at c1cdfcaa-f7d5-17d2-25d7-0a6ccb70eab2 [Follower]" leader-address= leader-id= +2026-11-29T10:29:11.143Z [INFO] server-c300d831-401a-1055-6b36-6275e584472a: entering follower state: follower="Node at c300d831-401a-1055-6b36-6275e584472a [Follower]" leader-address= leader-id= +2026-11-29T10:29:11.742Z [WARN] server-c300d831-401a-1055-6b36-6275e584472a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:11.742Z [INFO] server-c300d831-401a-1055-6b36-6275e584472a: entering candidate state: node="Node at c300d831-401a-1055-6b36-6275e584472a [Candidate]" term=2 +2026-11-29T10:29:11.742Z [INFO] server-c300d831-401a-1055-6b36-6275e584472a: election won: term=2 tally=2 +2026-11-29T10:29:11.742Z [INFO] server-c300d831-401a-1055-6b36-6275e584472a: entering leader state: leader="Node at c300d831-401a-1055-6b36-6275e584472a [Leader]" +2026-11-29T10:29:11.742Z [INFO] server-c300d831-401a-1055-6b36-6275e584472a: added peer, starting replication: peer=server-043516c5-4ec9-dbe7-4e23-57a03acef719 +2026-11-29T10:29:11.742Z [INFO] server-c300d831-401a-1055-6b36-6275e584472a: added peer, starting replication: peer=server-c1cdfcaa-f7d5-17d2-25d7-0a6ccb70eab2 +2026-11-29T10:29:11.742Z [INFO] server-c300d831-401a-1055-6b36-6275e584472a: pipelining replication: peer="{Voter server-c1cdfcaa-f7d5-17d2-25d7-0a6ccb70eab2 c1cdfcaa-f7d5-17d2-25d7-0a6ccb70eab2}" +2026-11-29T10:29:11.742Z [INFO] server-c300d831-401a-1055-6b36-6275e584472a: pipelining replication: peer="{Voter server-043516c5-4ec9-dbe7-4e23-57a03acef719 043516c5-4ec9-dbe7-4e23-57a03acef719}" +2026-11-29T10:29:12.747Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:11.742391459 +0000 UTC m=+62.477445561 (1 nodes), 598.84048ms from start of poll, 598.98576ms from cluster start. Timeout at 2026-11-29 10:29:12.747401419 +0000 UTC m=+63.482455561, 1.00501s after stability +2026-11-29T10:29:12.747Z [INFO] creating new snapshot: path=/tmp/raft719030215/snapshots/2-113-1795948152747.tmp +2026-11-29T10:29:12.748Z [INFO] server-c300d831-401a-1055-6b36-6275e584472a: copied to local snapshot: bytes=71 +2026-11-29T10:29:12.748Z [INFO] server-c300d831-401a-1055-6b36-6275e584472a: snapshot restore progress: id=2-113-1795948152747 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:12.748Z [INFO] server-c300d831-401a-1055-6b36-6275e584472a: restored user snapshot: index=113 +2026-11-29T10:29:12.748Z [ERROR] server-c300d831-401a-1055-6b36-6275e584472a: failed to get log: index=3 error="log not found" +2026-11-29T10:29:12.748Z [INFO] server-c300d831-401a-1055-6b36-6275e584472a: aborting pipeline replication: peer="{Voter server-c1cdfcaa-f7d5-17d2-25d7-0a6ccb70eab2 c1cdfcaa-f7d5-17d2-25d7-0a6ccb70eab2}" +2026-11-29T10:29:12.748Z [ERROR] server-c300d831-401a-1055-6b36-6275e584472a: failed to get log: index=3 error="log not found" +2026-11-29T10:29:12.748Z [INFO] server-c300d831-401a-1055-6b36-6275e584472a: aborting pipeline replication: peer="{Voter server-043516c5-4ec9-dbe7-4e23-57a03acef719 043516c5-4ec9-dbe7-4e23-57a03acef719}" +2026-11-29T10:29:12.755Z [ERROR] server-c300d831-401a-1055-6b36-6275e584472a: failed to get log: index=3 error="log not found" +2026-11-29T10:29:12.755Z [INFO] creating new snapshot: path=/tmp/raft1499654143/snapshots/2-113-1795948152755.tmp +2026-11-29T10:29:12.755Z [INFO] server-043516c5-4ec9-dbe7-4e23-57a03acef719: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:12.755Z [INFO] server-043516c5-4ec9-dbe7-4e23-57a03acef719: copied to local snapshot: bytes=71 +2026-11-29T10:29:12.755Z [INFO] server-043516c5-4ec9-dbe7-4e23-57a03acef719: snapshot restore progress: id=2-113-1795948152755 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:12.755Z [INFO] server-043516c5-4ec9-dbe7-4e23-57a03acef719: Installed remote snapshot +2026-11-29T10:29:12.756Z [INFO] server-c300d831-401a-1055-6b36-6275e584472a: pipelining replication: peer="{Voter server-043516c5-4ec9-dbe7-4e23-57a03acef719 043516c5-4ec9-dbe7-4e23-57a03acef719}" +2026-11-29T10:29:12.756Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:12.758Z [ERROR] server-c300d831-401a-1055-6b36-6275e584472a: failed to get log: index=3 error="log not found" +2026-11-29T10:29:12.759Z [INFO] creating new snapshot: path=/tmp/raft636742175/snapshots/2-113-1795948152759.tmp +2026-11-29T10:29:12.759Z [INFO] server-c1cdfcaa-f7d5-17d2-25d7-0a6ccb70eab2: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:12.759Z [INFO] server-c1cdfcaa-f7d5-17d2-25d7-0a6ccb70eab2: copied to local snapshot: bytes=71 +2026-11-29T10:29:12.759Z [INFO] server-c1cdfcaa-f7d5-17d2-25d7-0a6ccb70eab2: snapshot restore progress: id=2-113-1795948152759 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:12.759Z [INFO] server-c1cdfcaa-f7d5-17d2-25d7-0a6ccb70eab2: Installed remote snapshot +2026-11-29T10:29:12.759Z [INFO] server-c300d831-401a-1055-6b36-6275e584472a: pipelining replication: peer="{Voter server-c1cdfcaa-f7d5-17d2-25d7-0a6ccb70eab2 c1cdfcaa-f7d5-17d2-25d7-0a6ccb70eab2}" +2026-11-29T10:29:13.762Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:12.756065779 +0000 UTC m=+63.491119881 (2 nodes), 480ns from start of poll, 1.61266008s from cluster start. Timeout at 2026-11-29 10:29:13.761083579 +0000 UTC m=+64.496137761, 1.00501788s after stability +2026-11-29T10:29:13.773Z [INFO] server-3d20655f-8894-df2c-b354-3a4edd39cd38: aborting pipeline replication: peer="{Voter server-7e647547-781a-494b-3855-4bc5644d8592 7e647547-781a-494b-3855-4bc5644d8592}" +2026-11-29T10:29:13.774Z [INFO] server-3d20655f-8894-df2c-b354-3a4edd39cd38: aborting pipeline replication: peer="{Voter server-1e9aa434-3ec6-ad64-c0c1-69bd158acf5d 1e9aa434-3ec6-ad64-c0c1-69bd158acf5d}" === RUN TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_true -2025-10-27T04:03:11.080Z [INFO] server-0db2f8fc-3201-cc6b-120d-038b0875c934: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0db2f8fc-3201-cc6b-120d-038b0875c934 Address:0db2f8fc-3201-cc6b-120d-038b0875c934} {Suffrage:Voter ID:server-5f0bc7a3-9e71-8eca-5ec8-46e66a38be38 Address:5f0bc7a3-9e71-8eca-5ec8-46e66a38be38} {Suffrage:Voter ID:server-a1b76a63-f1be-5406-dea6-bab5d1856074 Address:a1b76a63-f1be-5406-dea6-bab5d1856074}]" -2025-10-27T04:03:11.080Z [INFO] server-5f0bc7a3-9e71-8eca-5ec8-46e66a38be38: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0db2f8fc-3201-cc6b-120d-038b0875c934 Address:0db2f8fc-3201-cc6b-120d-038b0875c934} {Suffrage:Voter ID:server-5f0bc7a3-9e71-8eca-5ec8-46e66a38be38 Address:5f0bc7a3-9e71-8eca-5ec8-46e66a38be38} {Suffrage:Voter ID:server-a1b76a63-f1be-5406-dea6-bab5d1856074 Address:a1b76a63-f1be-5406-dea6-bab5d1856074}]" -2025-10-27T04:03:11.080Z [INFO] server-0db2f8fc-3201-cc6b-120d-038b0875c934: entering follower state: follower="Node at 0db2f8fc-3201-cc6b-120d-038b0875c934 [Follower]" leader-address= leader-id= -2025-10-27T04:03:11.081Z [INFO] server-a1b76a63-f1be-5406-dea6-bab5d1856074: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0db2f8fc-3201-cc6b-120d-038b0875c934 Address:0db2f8fc-3201-cc6b-120d-038b0875c934} {Suffrage:Voter ID:server-5f0bc7a3-9e71-8eca-5ec8-46e66a38be38 Address:5f0bc7a3-9e71-8eca-5ec8-46e66a38be38} {Suffrage:Voter ID:server-a1b76a63-f1be-5406-dea6-bab5d1856074 Address:a1b76a63-f1be-5406-dea6-bab5d1856074}]" -2025-10-27T04:03:11.081Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:11.081Z [INFO] server-5f0bc7a3-9e71-8eca-5ec8-46e66a38be38: entering follower state: follower="Node at 5f0bc7a3-9e71-8eca-5ec8-46e66a38be38 [Follower]" leader-address= leader-id= -2025-10-27T04:03:11.081Z [INFO] server-a1b76a63-f1be-5406-dea6-bab5d1856074: entering follower state: follower="Node at a1b76a63-f1be-5406-dea6-bab5d1856074 [Follower]" leader-address= leader-id= -2025-10-27T04:03:11.977Z [WARN] server-5f0bc7a3-9e71-8eca-5ec8-46e66a38be38: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:11.977Z [INFO] server-5f0bc7a3-9e71-8eca-5ec8-46e66a38be38: entering candidate state: node="Node at 5f0bc7a3-9e71-8eca-5ec8-46e66a38be38 [Candidate]" term=2 -2025-10-27T04:03:11.977Z [INFO] server-5f0bc7a3-9e71-8eca-5ec8-46e66a38be38: election won: term=2 tally=2 -2025-10-27T04:03:11.977Z [INFO] server-5f0bc7a3-9e71-8eca-5ec8-46e66a38be38: entering leader state: leader="Node at 5f0bc7a3-9e71-8eca-5ec8-46e66a38be38 [Leader]" -2025-10-27T04:03:11.977Z [INFO] server-5f0bc7a3-9e71-8eca-5ec8-46e66a38be38: added peer, starting replication: peer=server-0db2f8fc-3201-cc6b-120d-038b0875c934 -2025-10-27T04:03:11.977Z [INFO] server-5f0bc7a3-9e71-8eca-5ec8-46e66a38be38: added peer, starting replication: peer=server-a1b76a63-f1be-5406-dea6-bab5d1856074 -2025-10-27T04:03:11.977Z [INFO] server-5f0bc7a3-9e71-8eca-5ec8-46e66a38be38: pipelining replication: peer="{Voter server-a1b76a63-f1be-5406-dea6-bab5d1856074 a1b76a63-f1be-5406-dea6-bab5d1856074}" -2025-10-27T04:03:11.977Z [INFO] server-5f0bc7a3-9e71-8eca-5ec8-46e66a38be38: pipelining replication: peer="{Voter server-0db2f8fc-3201-cc6b-120d-038b0875c934 0db2f8fc-3201-cc6b-120d-038b0875c934}" -2025-10-27T04:03:12.983Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:11.977629621 +0000 UTC m=+66.972502059 (1 nodes), 896.575846ms from start of poll, 896.772448ms from cluster start. Timeout at 2025-10-27 04:03:12.982640861 +0000 UTC m=+67.977513379, 1.00501132s after stability -2025-10-27T04:03:12.983Z [INFO] server-5f0bc7a3-9e71-8eca-5ec8-46e66a38be38: starting snapshot up to: index=12 -2025-10-27T04:03:12.983Z [INFO] creating new snapshot: path=/tmp/raft3259926656/snapshots/2-12-1761537792983.tmp -2025-10-27T04:03:12.984Z [INFO] server-5f0bc7a3-9e71-8eca-5ec8-46e66a38be38: snapshot complete up to: index=12 -2025-10-27T04:03:12.984Z [INFO] server-dfc2ec75-8cd5-9ea7-7493-0e17de94e0ce: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dfc2ec75-8cd5-9ea7-7493-0e17de94e0ce Address:dfc2ec75-8cd5-9ea7-7493-0e17de94e0ce} {Suffrage:Voter ID:server-d12b7387-fb28-4a4c-8297-7a47f147722e Address:d12b7387-fb28-4a4c-8297-7a47f147722e} {Suffrage:Voter ID:server-2cdae8d6-51c9-e25d-e177-0cee8fd2a1a2 Address:2cdae8d6-51c9-e25d-e177-0cee8fd2a1a2}]" -2025-10-27T04:03:12.984Z [INFO] server-d12b7387-fb28-4a4c-8297-7a47f147722e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dfc2ec75-8cd5-9ea7-7493-0e17de94e0ce Address:dfc2ec75-8cd5-9ea7-7493-0e17de94e0ce} {Suffrage:Voter ID:server-d12b7387-fb28-4a4c-8297-7a47f147722e Address:d12b7387-fb28-4a4c-8297-7a47f147722e} {Suffrage:Voter ID:server-2cdae8d6-51c9-e25d-e177-0cee8fd2a1a2 Address:2cdae8d6-51c9-e25d-e177-0cee8fd2a1a2}]" -2025-10-27T04:03:12.985Z [INFO] server-2cdae8d6-51c9-e25d-e177-0cee8fd2a1a2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dfc2ec75-8cd5-9ea7-7493-0e17de94e0ce Address:dfc2ec75-8cd5-9ea7-7493-0e17de94e0ce} {Suffrage:Voter ID:server-d12b7387-fb28-4a4c-8297-7a47f147722e Address:d12b7387-fb28-4a4c-8297-7a47f147722e} {Suffrage:Voter ID:server-2cdae8d6-51c9-e25d-e177-0cee8fd2a1a2 Address:2cdae8d6-51c9-e25d-e177-0cee8fd2a1a2}]" -2025-10-27T04:03:12.985Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:12.985Z [INFO] server-dfc2ec75-8cd5-9ea7-7493-0e17de94e0ce: entering follower state: follower="Node at dfc2ec75-8cd5-9ea7-7493-0e17de94e0ce [Follower]" leader-address= leader-id= -2025-10-27T04:03:12.985Z [INFO] server-d12b7387-fb28-4a4c-8297-7a47f147722e: entering follower state: follower="Node at d12b7387-fb28-4a4c-8297-7a47f147722e [Follower]" leader-address= leader-id= -2025-10-27T04:03:12.985Z [INFO] server-2cdae8d6-51c9-e25d-e177-0cee8fd2a1a2: entering follower state: follower="Node at 2cdae8d6-51c9-e25d-e177-0cee8fd2a1a2 [Follower]" leader-address= leader-id= -2025-10-27T04:03:13.609Z [WARN] server-d12b7387-fb28-4a4c-8297-7a47f147722e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:13.609Z [INFO] server-d12b7387-fb28-4a4c-8297-7a47f147722e: entering candidate state: node="Node at d12b7387-fb28-4a4c-8297-7a47f147722e [Candidate]" term=2 -2025-10-27T04:03:13.611Z [INFO] server-d12b7387-fb28-4a4c-8297-7a47f147722e: election won: term=2 tally=2 -2025-10-27T04:03:13.612Z [INFO] server-d12b7387-fb28-4a4c-8297-7a47f147722e: entering leader state: leader="Node at d12b7387-fb28-4a4c-8297-7a47f147722e [Leader]" -2025-10-27T04:03:13.612Z [INFO] server-d12b7387-fb28-4a4c-8297-7a47f147722e: added peer, starting replication: peer=server-dfc2ec75-8cd5-9ea7-7493-0e17de94e0ce -2025-10-27T04:03:13.612Z [INFO] server-d12b7387-fb28-4a4c-8297-7a47f147722e: added peer, starting replication: peer=server-2cdae8d6-51c9-e25d-e177-0cee8fd2a1a2 -2025-10-27T04:03:13.612Z [INFO] server-d12b7387-fb28-4a4c-8297-7a47f147722e: pipelining replication: peer="{Voter server-dfc2ec75-8cd5-9ea7-7493-0e17de94e0ce dfc2ec75-8cd5-9ea7-7493-0e17de94e0ce}" -2025-10-27T04:03:13.612Z [INFO] server-d12b7387-fb28-4a4c-8297-7a47f147722e: pipelining replication: peer="{Voter server-2cdae8d6-51c9-e25d-e177-0cee8fd2a1a2 2cdae8d6-51c9-e25d-e177-0cee8fd2a1a2}" -2025-10-27T04:03:14.617Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:13.612501912 +0000 UTC m=+68.607374350 (1 nodes), 627.447287ms from start of poll, 627.636209ms from cluster start. Timeout at 2025-10-27 04:03:14.617507432 +0000 UTC m=+69.612379870, 1.00500552s after stability -2025-10-27T04:03:14.620Z [INFO] creating new snapshot: path=/tmp/raft2726457016/snapshots/2-113-1761537794620.tmp -2025-10-27T04:03:14.621Z [INFO] server-d12b7387-fb28-4a4c-8297-7a47f147722e: copied to local snapshot: bytes=71 -2025-10-27T04:03:14.623Z [INFO] server-d12b7387-fb28-4a4c-8297-7a47f147722e: snapshot restore progress: id=2-113-1761537794620 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:14.623Z [INFO] server-d12b7387-fb28-4a4c-8297-7a47f147722e: removing all old logs from log store -2025-10-27T04:03:14.623Z [INFO] server-d12b7387-fb28-4a4c-8297-7a47f147722e: compacting logs: from=1 to=2 -2025-10-27T04:03:14.623Z [INFO] server-d12b7387-fb28-4a4c-8297-7a47f147722e: restored user snapshot: index=113 -2025-10-27T04:03:14.624Z [ERROR] server-d12b7387-fb28-4a4c-8297-7a47f147722e: failed to get log: index=2 error="log not found" -2025-10-27T04:03:14.624Z [INFO] server-d12b7387-fb28-4a4c-8297-7a47f147722e: aborting pipeline replication: peer="{Voter server-2cdae8d6-51c9-e25d-e177-0cee8fd2a1a2 2cdae8d6-51c9-e25d-e177-0cee8fd2a1a2}" -2025-10-27T04:03:14.624Z [ERROR] server-d12b7387-fb28-4a4c-8297-7a47f147722e: failed to get log: index=2 error="log not found" -2025-10-27T04:03:14.624Z [INFO] server-d12b7387-fb28-4a4c-8297-7a47f147722e: aborting pipeline replication: peer="{Voter server-dfc2ec75-8cd5-9ea7-7493-0e17de94e0ce dfc2ec75-8cd5-9ea7-7493-0e17de94e0ce}" -2025-10-27T04:03:14.632Z [ERROR] server-d12b7387-fb28-4a4c-8297-7a47f147722e: failed to get log: index=2 error="log not found" -2025-10-27T04:03:14.632Z [INFO] creating new snapshot: path=/tmp/raft2399496098/snapshots/2-113-1761537794632.tmp -2025-10-27T04:03:14.632Z [ERROR] server-d12b7387-fb28-4a4c-8297-7a47f147722e: failed to get log: index=2 error="log not found" -2025-10-27T04:03:14.632Z [INFO] creating new snapshot: path=/tmp/raft2949882322/snapshots/2-113-1761537794632.tmp -2025-10-27T04:03:14.633Z [INFO] server-dfc2ec75-8cd5-9ea7-7493-0e17de94e0ce: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:14.633Z [INFO] server-dfc2ec75-8cd5-9ea7-7493-0e17de94e0ce: copied to local snapshot: bytes=71 -2025-10-27T04:03:14.633Z [INFO] server-2cdae8d6-51c9-e25d-e177-0cee8fd2a1a2: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:14.633Z [INFO] server-2cdae8d6-51c9-e25d-e177-0cee8fd2a1a2: copied to local snapshot: bytes=71 -2025-10-27T04:03:14.634Z [INFO] server-dfc2ec75-8cd5-9ea7-7493-0e17de94e0ce: snapshot restore progress: id=2-113-1761537794632 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:14.634Z [INFO] server-dfc2ec75-8cd5-9ea7-7493-0e17de94e0ce: removing all old logs from log store -2025-10-27T04:03:14.634Z [INFO] server-dfc2ec75-8cd5-9ea7-7493-0e17de94e0ce: compacting logs: from=1 to=2 -2025-10-27T04:03:14.634Z [INFO] server-dfc2ec75-8cd5-9ea7-7493-0e17de94e0ce: Installed remote snapshot -2025-10-27T04:03:14.634Z [INFO] server-d12b7387-fb28-4a4c-8297-7a47f147722e: pipelining replication: peer="{Voter server-dfc2ec75-8cd5-9ea7-7493-0e17de94e0ce dfc2ec75-8cd5-9ea7-7493-0e17de94e0ce}" -2025-10-27T04:03:14.634Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:03:14.634Z [INFO] server-2cdae8d6-51c9-e25d-e177-0cee8fd2a1a2: snapshot restore progress: id=2-113-1761537794632 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:14.634Z [INFO] server-2cdae8d6-51c9-e25d-e177-0cee8fd2a1a2: removing all old logs from log store -2025-10-27T04:03:14.634Z [INFO] server-2cdae8d6-51c9-e25d-e177-0cee8fd2a1a2: compacting logs: from=1 to=2 -2025-10-27T04:03:14.634Z [INFO] server-2cdae8d6-51c9-e25d-e177-0cee8fd2a1a2: Installed remote snapshot -2025-10-27T04:03:14.634Z [INFO] server-d12b7387-fb28-4a4c-8297-7a47f147722e: pipelining replication: peer="{Voter server-2cdae8d6-51c9-e25d-e177-0cee8fd2a1a2 2cdae8d6-51c9-e25d-e177-0cee8fd2a1a2}" -2025-10-27T04:03:15.639Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:03:14.634307454 +0000 UTC m=+69.629179892 (2 nodes), 760ns from start of poll, 1.649441751s from cluster start. Timeout at 2025-10-27 04:03:15.639341374 +0000 UTC m=+70.634213852, 1.00503396s after stability -2025-10-27T04:03:15.651Z [INFO] server-5f0bc7a3-9e71-8eca-5ec8-46e66a38be38: aborting pipeline replication: peer="{Voter server-a1b76a63-f1be-5406-dea6-bab5d1856074 a1b76a63-f1be-5406-dea6-bab5d1856074}" -2025-10-27T04:03:15.651Z [INFO] server-5f0bc7a3-9e71-8eca-5ec8-46e66a38be38: aborting pipeline replication: peer="{Voter server-0db2f8fc-3201-cc6b-120d-038b0875c934 0db2f8fc-3201-cc6b-120d-038b0875c934}" +2026-11-29T10:29:13.774Z [INFO] server-1e0b8054-3344-03e9-338b-80101a0b4e4a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1e0b8054-3344-03e9-338b-80101a0b4e4a Address:1e0b8054-3344-03e9-338b-80101a0b4e4a} {Suffrage:Voter ID:server-b087bec1-7dff-003d-e780-eda404b95bb2 Address:b087bec1-7dff-003d-e780-eda404b95bb2} {Suffrage:Voter ID:server-683d0600-a978-ec03-392c-7d5a393ca524 Address:683d0600-a978-ec03-392c-7d5a393ca524}]" +2026-11-29T10:29:13.774Z [INFO] server-b087bec1-7dff-003d-e780-eda404b95bb2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1e0b8054-3344-03e9-338b-80101a0b4e4a Address:1e0b8054-3344-03e9-338b-80101a0b4e4a} {Suffrage:Voter ID:server-b087bec1-7dff-003d-e780-eda404b95bb2 Address:b087bec1-7dff-003d-e780-eda404b95bb2} {Suffrage:Voter ID:server-683d0600-a978-ec03-392c-7d5a393ca524 Address:683d0600-a978-ec03-392c-7d5a393ca524}]" +2026-11-29T10:29:13.774Z [INFO] server-1e0b8054-3344-03e9-338b-80101a0b4e4a: entering follower state: follower="Node at 1e0b8054-3344-03e9-338b-80101a0b4e4a [Follower]" leader-address= leader-id= +2026-11-29T10:29:13.774Z [INFO] server-683d0600-a978-ec03-392c-7d5a393ca524: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1e0b8054-3344-03e9-338b-80101a0b4e4a Address:1e0b8054-3344-03e9-338b-80101a0b4e4a} {Suffrage:Voter ID:server-b087bec1-7dff-003d-e780-eda404b95bb2 Address:b087bec1-7dff-003d-e780-eda404b95bb2} {Suffrage:Voter ID:server-683d0600-a978-ec03-392c-7d5a393ca524 Address:683d0600-a978-ec03-392c-7d5a393ca524}]" +2026-11-29T10:29:13.774Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:13.774Z [INFO] server-b087bec1-7dff-003d-e780-eda404b95bb2: entering follower state: follower="Node at b087bec1-7dff-003d-e780-eda404b95bb2 [Follower]" leader-address= leader-id= +2026-11-29T10:29:13.774Z [INFO] server-683d0600-a978-ec03-392c-7d5a393ca524: entering follower state: follower="Node at 683d0600-a978-ec03-392c-7d5a393ca524 [Follower]" leader-address= leader-id= +2026-11-29T10:29:14.362Z [WARN] server-683d0600-a978-ec03-392c-7d5a393ca524: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:14.362Z [INFO] server-683d0600-a978-ec03-392c-7d5a393ca524: entering candidate state: node="Node at 683d0600-a978-ec03-392c-7d5a393ca524 [Candidate]" term=2 +2026-11-29T10:29:14.362Z [INFO] server-683d0600-a978-ec03-392c-7d5a393ca524: election won: term=2 tally=2 +2026-11-29T10:29:14.362Z [INFO] server-683d0600-a978-ec03-392c-7d5a393ca524: entering leader state: leader="Node at 683d0600-a978-ec03-392c-7d5a393ca524 [Leader]" +2026-11-29T10:29:14.362Z [INFO] server-683d0600-a978-ec03-392c-7d5a393ca524: added peer, starting replication: peer=server-1e0b8054-3344-03e9-338b-80101a0b4e4a +2026-11-29T10:29:14.362Z [INFO] server-683d0600-a978-ec03-392c-7d5a393ca524: added peer, starting replication: peer=server-b087bec1-7dff-003d-e780-eda404b95bb2 +2026-11-29T10:29:14.362Z [INFO] server-683d0600-a978-ec03-392c-7d5a393ca524: pipelining replication: peer="{Voter server-1e0b8054-3344-03e9-338b-80101a0b4e4a 1e0b8054-3344-03e9-338b-80101a0b4e4a}" +2026-11-29T10:29:14.362Z [INFO] server-683d0600-a978-ec03-392c-7d5a393ca524: pipelining replication: peer="{Voter server-b087bec1-7dff-003d-e780-eda404b95bb2 b087bec1-7dff-003d-e780-eda404b95bb2}" +2026-11-29T10:29:15.367Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:14.362383939 +0000 UTC m=+65.097438081 (1 nodes), 587.50328ms from start of poll, 587.71244ms from cluster start. Timeout at 2026-11-29 10:29:15.367389459 +0000 UTC m=+66.102443561, 1.00500548s after stability +2026-11-29T10:29:15.369Z [INFO] server-683d0600-a978-ec03-392c-7d5a393ca524: starting snapshot up to: index=12 +2026-11-29T10:29:15.369Z [INFO] creating new snapshot: path=/tmp/raft922911375/snapshots/2-12-1795948155369.tmp +2026-11-29T10:29:15.369Z [INFO] server-683d0600-a978-ec03-392c-7d5a393ca524: snapshot complete up to: index=12 +2026-11-29T10:29:15.370Z [INFO] server-f13af205-cd9e-22f1-ba57-da86b52a2be8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f13af205-cd9e-22f1-ba57-da86b52a2be8 Address:f13af205-cd9e-22f1-ba57-da86b52a2be8} {Suffrage:Voter ID:server-540df27b-1dd1-f7d2-ebf5-790b8be33b10 Address:540df27b-1dd1-f7d2-ebf5-790b8be33b10} {Suffrage:Voter ID:server-f076cd26-33aa-3669-6c54-815fc8bf3f9a Address:f076cd26-33aa-3669-6c54-815fc8bf3f9a}]" +2026-11-29T10:29:15.370Z [INFO] server-f13af205-cd9e-22f1-ba57-da86b52a2be8: entering follower state: follower="Node at f13af205-cd9e-22f1-ba57-da86b52a2be8 [Follower]" leader-address= leader-id= +2026-11-29T10:29:15.370Z [INFO] server-540df27b-1dd1-f7d2-ebf5-790b8be33b10: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f13af205-cd9e-22f1-ba57-da86b52a2be8 Address:f13af205-cd9e-22f1-ba57-da86b52a2be8} {Suffrage:Voter ID:server-540df27b-1dd1-f7d2-ebf5-790b8be33b10 Address:540df27b-1dd1-f7d2-ebf5-790b8be33b10} {Suffrage:Voter ID:server-f076cd26-33aa-3669-6c54-815fc8bf3f9a Address:f076cd26-33aa-3669-6c54-815fc8bf3f9a}]" +2026-11-29T10:29:15.370Z [INFO] server-540df27b-1dd1-f7d2-ebf5-790b8be33b10: entering follower state: follower="Node at 540df27b-1dd1-f7d2-ebf5-790b8be33b10 [Follower]" leader-address= leader-id= +2026-11-29T10:29:15.370Z [INFO] server-f076cd26-33aa-3669-6c54-815fc8bf3f9a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f13af205-cd9e-22f1-ba57-da86b52a2be8 Address:f13af205-cd9e-22f1-ba57-da86b52a2be8} {Suffrage:Voter ID:server-540df27b-1dd1-f7d2-ebf5-790b8be33b10 Address:540df27b-1dd1-f7d2-ebf5-790b8be33b10} {Suffrage:Voter ID:server-f076cd26-33aa-3669-6c54-815fc8bf3f9a Address:f076cd26-33aa-3669-6c54-815fc8bf3f9a}]" +2026-11-29T10:29:15.370Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:15.371Z [INFO] server-f076cd26-33aa-3669-6c54-815fc8bf3f9a: entering follower state: follower="Node at f076cd26-33aa-3669-6c54-815fc8bf3f9a [Follower]" leader-address= leader-id= +2026-11-29T10:29:15.994Z [WARN] server-f13af205-cd9e-22f1-ba57-da86b52a2be8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:15.994Z [INFO] server-f13af205-cd9e-22f1-ba57-da86b52a2be8: entering candidate state: node="Node at f13af205-cd9e-22f1-ba57-da86b52a2be8 [Candidate]" term=2 +2026-11-29T10:29:15.994Z [INFO] server-f13af205-cd9e-22f1-ba57-da86b52a2be8: election won: term=2 tally=2 +2026-11-29T10:29:15.994Z [INFO] server-f13af205-cd9e-22f1-ba57-da86b52a2be8: entering leader state: leader="Node at f13af205-cd9e-22f1-ba57-da86b52a2be8 [Leader]" +2026-11-29T10:29:15.994Z [INFO] server-f13af205-cd9e-22f1-ba57-da86b52a2be8: added peer, starting replication: peer=server-540df27b-1dd1-f7d2-ebf5-790b8be33b10 +2026-11-29T10:29:15.994Z [INFO] server-f13af205-cd9e-22f1-ba57-da86b52a2be8: added peer, starting replication: peer=server-f076cd26-33aa-3669-6c54-815fc8bf3f9a +2026-11-29T10:29:15.994Z [INFO] server-f13af205-cd9e-22f1-ba57-da86b52a2be8: pipelining replication: peer="{Voter server-540df27b-1dd1-f7d2-ebf5-790b8be33b10 540df27b-1dd1-f7d2-ebf5-790b8be33b10}" +2026-11-29T10:29:15.994Z [INFO] server-f13af205-cd9e-22f1-ba57-da86b52a2be8: pipelining replication: peer="{Voter server-f076cd26-33aa-3669-6c54-815fc8bf3f9a f076cd26-33aa-3669-6c54-815fc8bf3f9a}" +2026-11-29T10:29:17.000Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:15.994767819 +0000 UTC m=+66.729821961 (1 nodes), 623.77688ms from start of poll, 624.26716ms from cluster start. Timeout at 2026-11-29 10:29:16.999769699 +0000 UTC m=+67.734823801, 1.00500184s after stability +2026-11-29T10:29:17.000Z [INFO] creating new snapshot: path=/tmp/raft3331473770/snapshots/2-113-1795948157000.tmp +2026-11-29T10:29:17.000Z [INFO] server-f13af205-cd9e-22f1-ba57-da86b52a2be8: copied to local snapshot: bytes=71 +2026-11-29T10:29:17.001Z [INFO] server-f13af205-cd9e-22f1-ba57-da86b52a2be8: snapshot restore progress: id=2-113-1795948157000 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:17.001Z [INFO] server-f13af205-cd9e-22f1-ba57-da86b52a2be8: removing all old logs from log store +2026-11-29T10:29:17.001Z [INFO] server-f13af205-cd9e-22f1-ba57-da86b52a2be8: compacting logs: from=1 to=2 +2026-11-29T10:29:17.001Z [INFO] server-f13af205-cd9e-22f1-ba57-da86b52a2be8: restored user snapshot: index=113 +2026-11-29T10:29:17.001Z [ERROR] server-f13af205-cd9e-22f1-ba57-da86b52a2be8: failed to get log: index=2 error="log not found" +2026-11-29T10:29:17.001Z [INFO] server-f13af205-cd9e-22f1-ba57-da86b52a2be8: aborting pipeline replication: peer="{Voter server-f076cd26-33aa-3669-6c54-815fc8bf3f9a f076cd26-33aa-3669-6c54-815fc8bf3f9a}" +2026-11-29T10:29:17.001Z [ERROR] server-f13af205-cd9e-22f1-ba57-da86b52a2be8: failed to get log: index=2 error="log not found" +2026-11-29T10:29:17.001Z [INFO] server-f13af205-cd9e-22f1-ba57-da86b52a2be8: aborting pipeline replication: peer="{Voter server-540df27b-1dd1-f7d2-ebf5-790b8be33b10 540df27b-1dd1-f7d2-ebf5-790b8be33b10}" +2026-11-29T10:29:17.007Z [ERROR] server-f13af205-cd9e-22f1-ba57-da86b52a2be8: failed to get log: index=2 error="log not found" +2026-11-29T10:29:17.007Z [ERROR] server-f13af205-cd9e-22f1-ba57-da86b52a2be8: failed to get log: index=2 error="log not found" +2026-11-29T10:29:17.007Z [INFO] creating new snapshot: path=/tmp/raft1838867612/snapshots/2-113-1795948157007.tmp +2026-11-29T10:29:17.007Z [INFO] server-f076cd26-33aa-3669-6c54-815fc8bf3f9a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:17.007Z [INFO] creating new snapshot: path=/tmp/raft743048607/snapshots/2-113-1795948157007.tmp +2026-11-29T10:29:17.007Z [INFO] server-f076cd26-33aa-3669-6c54-815fc8bf3f9a: copied to local snapshot: bytes=71 +2026-11-29T10:29:17.008Z [INFO] server-f076cd26-33aa-3669-6c54-815fc8bf3f9a: snapshot restore progress: id=2-113-1795948157007 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:17.008Z [INFO] server-540df27b-1dd1-f7d2-ebf5-790b8be33b10: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:17.008Z [INFO] server-f076cd26-33aa-3669-6c54-815fc8bf3f9a: removing all old logs from log store +2026-11-29T10:29:17.008Z [INFO] server-f076cd26-33aa-3669-6c54-815fc8bf3f9a: compacting logs: from=1 to=2 +2026-11-29T10:29:17.008Z [INFO] server-f076cd26-33aa-3669-6c54-815fc8bf3f9a: Installed remote snapshot +2026-11-29T10:29:17.008Z [INFO] server-f13af205-cd9e-22f1-ba57-da86b52a2be8: pipelining replication: peer="{Voter server-f076cd26-33aa-3669-6c54-815fc8bf3f9a f076cd26-33aa-3669-6c54-815fc8bf3f9a}" +2026-11-29T10:29:17.008Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:17.008Z [INFO] server-540df27b-1dd1-f7d2-ebf5-790b8be33b10: copied to local snapshot: bytes=71 +2026-11-29T10:29:17.008Z [INFO] server-540df27b-1dd1-f7d2-ebf5-790b8be33b10: snapshot restore progress: id=2-113-1795948157007 last-index=113 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:17.008Z [INFO] server-540df27b-1dd1-f7d2-ebf5-790b8be33b10: removing all old logs from log store +2026-11-29T10:29:17.008Z [INFO] server-540df27b-1dd1-f7d2-ebf5-790b8be33b10: compacting logs: from=1 to=2 +2026-11-29T10:29:17.008Z [INFO] server-540df27b-1dd1-f7d2-ebf5-790b8be33b10: Installed remote snapshot +2026-11-29T10:29:17.008Z [INFO] server-f13af205-cd9e-22f1-ba57-da86b52a2be8: pipelining replication: peer="{Voter server-540df27b-1dd1-f7d2-ebf5-790b8be33b10 540df27b-1dd1-f7d2-ebf5-790b8be33b10}" +2026-11-29T10:29:18.013Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:17.008376379 +0000 UTC m=+67.743430481 (2 nodes), 600ns from start of poll, 1.63787568s from cluster start. Timeout at 2026-11-29 10:29:18.013393059 +0000 UTC m=+68.748447161, 1.00501668s after stability +2026-11-29T10:29:18.026Z [INFO] server-683d0600-a978-ec03-392c-7d5a393ca524: aborting pipeline replication: peer="{Voter server-b087bec1-7dff-003d-e780-eda404b95bb2 b087bec1-7dff-003d-e780-eda404b95bb2}" +2026-11-29T10:29:18.026Z [INFO] server-683d0600-a978-ec03-392c-7d5a393ca524: aborting pipeline replication: peer="{Voter server-1e0b8054-3344-03e9-338b-80101a0b4e4a 1e0b8054-3344-03e9-338b-80101a0b4e4a}" === RUN TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_false -2025-10-27T04:03:15.651Z [INFO] server-7c51c78f-a1f3-f322-4d5b-733172d3e4ac: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7c51c78f-a1f3-f322-4d5b-733172d3e4ac Address:7c51c78f-a1f3-f322-4d5b-733172d3e4ac} {Suffrage:Voter ID:server-9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16 Address:9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16} {Suffrage:Voter ID:server-b8afccdc-cf79-93d7-a9cf-410e269721fb Address:b8afccdc-cf79-93d7-a9cf-410e269721fb}]" -2025-10-27T04:03:15.652Z [INFO] server-9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7c51c78f-a1f3-f322-4d5b-733172d3e4ac Address:7c51c78f-a1f3-f322-4d5b-733172d3e4ac} {Suffrage:Voter ID:server-9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16 Address:9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16} {Suffrage:Voter ID:server-b8afccdc-cf79-93d7-a9cf-410e269721fb Address:b8afccdc-cf79-93d7-a9cf-410e269721fb}]" -2025-10-27T04:03:15.651Z [INFO] server-7c51c78f-a1f3-f322-4d5b-733172d3e4ac: entering follower state: follower="Node at 7c51c78f-a1f3-f322-4d5b-733172d3e4ac [Follower]" leader-address= leader-id= -2025-10-27T04:03:15.652Z [INFO] server-b8afccdc-cf79-93d7-a9cf-410e269721fb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7c51c78f-a1f3-f322-4d5b-733172d3e4ac Address:7c51c78f-a1f3-f322-4d5b-733172d3e4ac} {Suffrage:Voter ID:server-9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16 Address:9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16} {Suffrage:Voter ID:server-b8afccdc-cf79-93d7-a9cf-410e269721fb Address:b8afccdc-cf79-93d7-a9cf-410e269721fb}]" -2025-10-27T04:03:15.652Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:15.652Z [INFO] server-b8afccdc-cf79-93d7-a9cf-410e269721fb: entering follower state: follower="Node at b8afccdc-cf79-93d7-a9cf-410e269721fb [Follower]" leader-address= leader-id= -2025-10-27T04:03:15.652Z [INFO] server-9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16: entering follower state: follower="Node at 9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16 [Follower]" leader-address= leader-id= -2025-10-27T04:03:16.177Z [WARN] server-9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:16.177Z [INFO] server-9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16: entering candidate state: node="Node at 9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16 [Candidate]" term=2 -2025-10-27T04:03:16.177Z [INFO] server-9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16: election won: term=2 tally=2 -2025-10-27T04:03:16.177Z [INFO] server-9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16: entering leader state: leader="Node at 9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16 [Leader]" -2025-10-27T04:03:16.177Z [INFO] server-9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16: added peer, starting replication: peer=server-7c51c78f-a1f3-f322-4d5b-733172d3e4ac -2025-10-27T04:03:16.177Z [INFO] server-9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16: added peer, starting replication: peer=server-b8afccdc-cf79-93d7-a9cf-410e269721fb -2025-10-27T04:03:16.177Z [INFO] server-9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16: pipelining replication: peer="{Voter server-b8afccdc-cf79-93d7-a9cf-410e269721fb b8afccdc-cf79-93d7-a9cf-410e269721fb}" -2025-10-27T04:03:16.177Z [INFO] server-9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16: pipelining replication: peer="{Voter server-7c51c78f-a1f3-f322-4d5b-733172d3e4ac 7c51c78f-a1f3-f322-4d5b-733172d3e4ac}" -2025-10-27T04:03:17.183Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:16.177700239 +0000 UTC m=+71.172572677 (1 nodes), 525.640751ms from start of poll, 525.840593ms from cluster start. Timeout at 2025-10-27 04:03:17.182709399 +0000 UTC m=+72.177581877, 1.0050092s after stability -2025-10-27T04:03:17.183Z [INFO] server-9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16: starting snapshot up to: index=12 -2025-10-27T04:03:17.183Z [INFO] creating new snapshot: path=/tmp/raft3758425222/snapshots/2-12-1761537797183.tmp -2025-10-27T04:03:17.184Z [INFO] server-9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16: snapshot complete up to: index=12 -2025-10-27T04:03:17.184Z [INFO] creating new snapshot: path=/tmp/raft3758425222/snapshots/2-1013-1761537797184.tmp -2025-10-27T04:03:17.184Z [INFO] server-9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16: copied to local snapshot: bytes=71 -2025-10-27T04:03:17.185Z [INFO] server-9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16: snapshot restore progress: id=2-1013-1761537797184 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:17.185Z [INFO] server-9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16: restored user snapshot: index=1013 -2025-10-27T04:03:17.185Z [ERROR] server-9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16: failed to get log: index=23 error="log not found" -2025-10-27T04:03:17.185Z [INFO] server-9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16: aborting pipeline replication: peer="{Voter server-b8afccdc-cf79-93d7-a9cf-410e269721fb b8afccdc-cf79-93d7-a9cf-410e269721fb}" -2025-10-27T04:03:17.185Z [ERROR] server-9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16: failed to get log: index=23 error="log not found" -2025-10-27T04:03:17.185Z [INFO] server-9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16: aborting pipeline replication: peer="{Voter server-7c51c78f-a1f3-f322-4d5b-733172d3e4ac 7c51c78f-a1f3-f322-4d5b-733172d3e4ac}" -2025-10-27T04:03:17.192Z [ERROR] server-9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16: failed to get log: index=23 error="log not found" -2025-10-27T04:03:17.193Z [INFO] creating new snapshot: path=/tmp/raft1285316399/snapshots/2-1013-1761537797193.tmp -2025-10-27T04:03:17.193Z [INFO] server-b8afccdc-cf79-93d7-a9cf-410e269721fb: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:17.193Z [INFO] server-b8afccdc-cf79-93d7-a9cf-410e269721fb: copied to local snapshot: bytes=71 -2025-10-27T04:03:17.193Z [INFO] server-b8afccdc-cf79-93d7-a9cf-410e269721fb: snapshot restore progress: id=2-1013-1761537797193 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:17.193Z [INFO] server-b8afccdc-cf79-93d7-a9cf-410e269721fb: Installed remote snapshot -2025-10-27T04:03:17.193Z [WARN] server-b8afccdc-cf79-93d7-a9cf-410e269721fb: skipping application of old log: index=22 -2025-10-27T04:03:17.193Z [INFO] server-9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16: pipelining replication: peer="{Voter server-b8afccdc-cf79-93d7-a9cf-410e269721fb b8afccdc-cf79-93d7-a9cf-410e269721fb}" -2025-10-27T04:03:17.192Z [ERROR] server-9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16: failed to get log: index=23 error="log not found" -2025-10-27T04:03:17.193Z [INFO] creating new snapshot: path=/tmp/raft3584380464/snapshots/2-1013-1761537797193.tmp -2025-10-27T04:03:17.193Z [INFO] server-7c51c78f-a1f3-f322-4d5b-733172d3e4ac: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:17.193Z [INFO] server-7c51c78f-a1f3-f322-4d5b-733172d3e4ac: copied to local snapshot: bytes=71 -2025-10-27T04:03:17.193Z [INFO] server-7c51c78f-a1f3-f322-4d5b-733172d3e4ac: snapshot restore progress: id=2-1013-1761537797193 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:17.194Z [INFO] server-7c51c78f-a1f3-f322-4d5b-733172d3e4ac: Installed remote snapshot -2025-10-27T04:03:17.194Z [WARN] server-7c51c78f-a1f3-f322-4d5b-733172d3e4ac: skipping application of old log: index=22 -2025-10-27T04:03:17.194Z [INFO] server-9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16: pipelining replication: peer="{Voter server-7c51c78f-a1f3-f322-4d5b-733172d3e4ac 7c51c78f-a1f3-f322-4d5b-733172d3e4ac}" -2025-10-27T04:03:17.194Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:03:18.199Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:03:17.194097264 +0000 UTC m=+72.188969702 (2 nodes), 440ns from start of poll, 1.542237618s from cluster start. Timeout at 2025-10-27 04:03:18.199188385 +0000 UTC m=+73.194060783, 1.005091081s after stability -2025-10-27T04:03:18.213Z [INFO] server-9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16: aborting pipeline replication: peer="{Voter server-b8afccdc-cf79-93d7-a9cf-410e269721fb b8afccdc-cf79-93d7-a9cf-410e269721fb}" -2025-10-27T04:03:18.213Z [INFO] server-9bc5ec1c-3ba9-28d3-e3bb-f25e0e366a16: aborting pipeline replication: peer="{Voter server-7c51c78f-a1f3-f322-4d5b-733172d3e4ac 7c51c78f-a1f3-f322-4d5b-733172d3e4ac}" +2026-11-29T10:29:18.027Z [INFO] server-25e0ddf1-44ca-aa15-7478-dd1434b13a63: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-25e0ddf1-44ca-aa15-7478-dd1434b13a63 Address:25e0ddf1-44ca-aa15-7478-dd1434b13a63} {Suffrage:Voter ID:server-68f51379-5454-fb01-5ee2-cc95cf1f5508 Address:68f51379-5454-fb01-5ee2-cc95cf1f5508} {Suffrage:Voter ID:server-12e494c9-1f40-34d8-d188-db2e6575530a Address:12e494c9-1f40-34d8-d188-db2e6575530a}]" +2026-11-29T10:29:18.027Z [INFO] server-25e0ddf1-44ca-aa15-7478-dd1434b13a63: entering follower state: follower="Node at 25e0ddf1-44ca-aa15-7478-dd1434b13a63 [Follower]" leader-address= leader-id= +2026-11-29T10:29:18.027Z [INFO] server-68f51379-5454-fb01-5ee2-cc95cf1f5508: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-25e0ddf1-44ca-aa15-7478-dd1434b13a63 Address:25e0ddf1-44ca-aa15-7478-dd1434b13a63} {Suffrage:Voter ID:server-68f51379-5454-fb01-5ee2-cc95cf1f5508 Address:68f51379-5454-fb01-5ee2-cc95cf1f5508} {Suffrage:Voter ID:server-12e494c9-1f40-34d8-d188-db2e6575530a Address:12e494c9-1f40-34d8-d188-db2e6575530a}]" +2026-11-29T10:29:18.027Z [INFO] server-68f51379-5454-fb01-5ee2-cc95cf1f5508: entering follower state: follower="Node at 68f51379-5454-fb01-5ee2-cc95cf1f5508 [Follower]" leader-address= leader-id= +2026-11-29T10:29:18.027Z [INFO] server-12e494c9-1f40-34d8-d188-db2e6575530a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-25e0ddf1-44ca-aa15-7478-dd1434b13a63 Address:25e0ddf1-44ca-aa15-7478-dd1434b13a63} {Suffrage:Voter ID:server-68f51379-5454-fb01-5ee2-cc95cf1f5508 Address:68f51379-5454-fb01-5ee2-cc95cf1f5508} {Suffrage:Voter ID:server-12e494c9-1f40-34d8-d188-db2e6575530a Address:12e494c9-1f40-34d8-d188-db2e6575530a}]" +2026-11-29T10:29:18.027Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:18.027Z [INFO] server-12e494c9-1f40-34d8-d188-db2e6575530a: entering follower state: follower="Node at 12e494c9-1f40-34d8-d188-db2e6575530a [Follower]" leader-address= leader-id= +2026-11-29T10:29:18.536Z [WARN] server-68f51379-5454-fb01-5ee2-cc95cf1f5508: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:18.536Z [INFO] server-68f51379-5454-fb01-5ee2-cc95cf1f5508: entering candidate state: node="Node at 68f51379-5454-fb01-5ee2-cc95cf1f5508 [Candidate]" term=2 +2026-11-29T10:29:18.536Z [INFO] server-68f51379-5454-fb01-5ee2-cc95cf1f5508: election won: term=2 tally=2 +2026-11-29T10:29:18.536Z [INFO] server-68f51379-5454-fb01-5ee2-cc95cf1f5508: entering leader state: leader="Node at 68f51379-5454-fb01-5ee2-cc95cf1f5508 [Leader]" +2026-11-29T10:29:18.536Z [INFO] server-68f51379-5454-fb01-5ee2-cc95cf1f5508: added peer, starting replication: peer=server-25e0ddf1-44ca-aa15-7478-dd1434b13a63 +2026-11-29T10:29:18.536Z [INFO] server-68f51379-5454-fb01-5ee2-cc95cf1f5508: added peer, starting replication: peer=server-12e494c9-1f40-34d8-d188-db2e6575530a +2026-11-29T10:29:18.536Z [INFO] server-68f51379-5454-fb01-5ee2-cc95cf1f5508: pipelining replication: peer="{Voter server-12e494c9-1f40-34d8-d188-db2e6575530a 12e494c9-1f40-34d8-d188-db2e6575530a}" +2026-11-29T10:29:18.536Z [INFO] server-68f51379-5454-fb01-5ee2-cc95cf1f5508: pipelining replication: peer="{Voter server-25e0ddf1-44ca-aa15-7478-dd1434b13a63 25e0ddf1-44ca-aa15-7478-dd1434b13a63}" +2026-11-29T10:29:19.542Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:18.536383899 +0000 UTC m=+69.271438041 (1 nodes), 508.53912ms from start of poll, 509.09996ms from cluster start. Timeout at 2026-11-29 10:29:19.541386699 +0000 UTC m=+70.276440761, 1.00500272s after stability +2026-11-29T10:29:19.542Z [INFO] server-68f51379-5454-fb01-5ee2-cc95cf1f5508: starting snapshot up to: index=12 +2026-11-29T10:29:19.542Z [INFO] creating new snapshot: path=/tmp/raft3775169602/snapshots/2-12-1795948159542.tmp +2026-11-29T10:29:19.543Z [INFO] server-68f51379-5454-fb01-5ee2-cc95cf1f5508: snapshot complete up to: index=12 +2026-11-29T10:29:19.544Z [INFO] creating new snapshot: path=/tmp/raft3775169602/snapshots/2-1013-1795948159544.tmp +2026-11-29T10:29:19.544Z [INFO] server-68f51379-5454-fb01-5ee2-cc95cf1f5508: copied to local snapshot: bytes=71 +2026-11-29T10:29:19.544Z [INFO] server-68f51379-5454-fb01-5ee2-cc95cf1f5508: snapshot restore progress: id=2-1013-1795948159544 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:19.544Z [INFO] server-68f51379-5454-fb01-5ee2-cc95cf1f5508: restored user snapshot: index=1013 +2026-11-29T10:29:19.545Z [ERROR] server-68f51379-5454-fb01-5ee2-cc95cf1f5508: failed to get log: index=23 error="log not found" +2026-11-29T10:29:19.545Z [INFO] server-68f51379-5454-fb01-5ee2-cc95cf1f5508: aborting pipeline replication: peer="{Voter server-25e0ddf1-44ca-aa15-7478-dd1434b13a63 25e0ddf1-44ca-aa15-7478-dd1434b13a63}" +2026-11-29T10:29:19.545Z [ERROR] server-68f51379-5454-fb01-5ee2-cc95cf1f5508: failed to get log: index=23 error="log not found" +2026-11-29T10:29:19.545Z [INFO] server-68f51379-5454-fb01-5ee2-cc95cf1f5508: aborting pipeline replication: peer="{Voter server-12e494c9-1f40-34d8-d188-db2e6575530a 12e494c9-1f40-34d8-d188-db2e6575530a}" +2026-11-29T10:29:19.551Z [ERROR] server-68f51379-5454-fb01-5ee2-cc95cf1f5508: failed to get log: index=23 error="log not found" +2026-11-29T10:29:19.552Z [INFO] creating new snapshot: path=/tmp/raft2666642142/snapshots/2-1013-1795948159552.tmp +2026-11-29T10:29:19.552Z [INFO] server-12e494c9-1f40-34d8-d188-db2e6575530a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:19.552Z [INFO] server-12e494c9-1f40-34d8-d188-db2e6575530a: copied to local snapshot: bytes=71 +2026-11-29T10:29:19.553Z [INFO] server-12e494c9-1f40-34d8-d188-db2e6575530a: snapshot restore progress: id=2-1013-1795948159552 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:19.553Z [INFO] server-12e494c9-1f40-34d8-d188-db2e6575530a: Installed remote snapshot +2026-11-29T10:29:19.553Z [WARN] server-12e494c9-1f40-34d8-d188-db2e6575530a: skipping application of old log: index=22 +2026-11-29T10:29:19.553Z [INFO] server-68f51379-5454-fb01-5ee2-cc95cf1f5508: pipelining replication: peer="{Voter server-12e494c9-1f40-34d8-d188-db2e6575530a 12e494c9-1f40-34d8-d188-db2e6575530a}" +2026-11-29T10:29:19.553Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:19.555Z [ERROR] server-68f51379-5454-fb01-5ee2-cc95cf1f5508: failed to get log: index=23 error="log not found" +2026-11-29T10:29:19.555Z [INFO] creating new snapshot: path=/tmp/raft129473009/snapshots/2-1013-1795948159555.tmp +2026-11-29T10:29:19.556Z [INFO] server-25e0ddf1-44ca-aa15-7478-dd1434b13a63: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:19.556Z [INFO] server-25e0ddf1-44ca-aa15-7478-dd1434b13a63: copied to local snapshot: bytes=71 +2026-11-29T10:29:19.556Z [INFO] server-25e0ddf1-44ca-aa15-7478-dd1434b13a63: snapshot restore progress: id=2-1013-1795948159555 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:19.556Z [INFO] server-25e0ddf1-44ca-aa15-7478-dd1434b13a63: Installed remote snapshot +2026-11-29T10:29:19.556Z [INFO] server-68f51379-5454-fb01-5ee2-cc95cf1f5508: pipelining replication: peer="{Voter server-25e0ddf1-44ca-aa15-7478-dd1434b13a63 25e0ddf1-44ca-aa15-7478-dd1434b13a63}" +2026-11-29T10:29:20.559Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:19.553243939 +0000 UTC m=+70.288298001 (2 nodes), 520ns from start of poll, 1.52595992s from cluster start. Timeout at 2026-11-29 10:29:20.558268659 +0000 UTC m=+71.293322841, 1.00502484s after stability +2026-11-29T10:29:20.570Z [INFO] server-68f51379-5454-fb01-5ee2-cc95cf1f5508: aborting pipeline replication: peer="{Voter server-25e0ddf1-44ca-aa15-7478-dd1434b13a63 25e0ddf1-44ca-aa15-7478-dd1434b13a63}" +2026-11-29T10:29:20.570Z [INFO] server-68f51379-5454-fb01-5ee2-cc95cf1f5508: aborting pipeline replication: peer="{Voter server-12e494c9-1f40-34d8-d188-db2e6575530a 12e494c9-1f40-34d8-d188-db2e6575530a}" === RUN TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_false -2025-10-27T04:03:18.214Z [INFO] server-78c1a299-7e80-a8b9-e434-54bea00e3312: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-78c1a299-7e80-a8b9-e434-54bea00e3312 Address:78c1a299-7e80-a8b9-e434-54bea00e3312} {Suffrage:Voter ID:server-59355739-66af-29c1-56b3-e1ede7aabe54 Address:59355739-66af-29c1-56b3-e1ede7aabe54} {Suffrage:Voter ID:server-1e6b386f-4665-4043-cebd-e105151e9194 Address:1e6b386f-4665-4043-cebd-e105151e9194}]" -2025-10-27T04:03:18.214Z [INFO] server-59355739-66af-29c1-56b3-e1ede7aabe54: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-78c1a299-7e80-a8b9-e434-54bea00e3312 Address:78c1a299-7e80-a8b9-e434-54bea00e3312} {Suffrage:Voter ID:server-59355739-66af-29c1-56b3-e1ede7aabe54 Address:59355739-66af-29c1-56b3-e1ede7aabe54} {Suffrage:Voter ID:server-1e6b386f-4665-4043-cebd-e105151e9194 Address:1e6b386f-4665-4043-cebd-e105151e9194}]" -2025-10-27T04:03:18.214Z [INFO] server-78c1a299-7e80-a8b9-e434-54bea00e3312: entering follower state: follower="Node at 78c1a299-7e80-a8b9-e434-54bea00e3312 [Follower]" leader-address= leader-id= -2025-10-27T04:03:18.214Z [INFO] server-1e6b386f-4665-4043-cebd-e105151e9194: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-78c1a299-7e80-a8b9-e434-54bea00e3312 Address:78c1a299-7e80-a8b9-e434-54bea00e3312} {Suffrage:Voter ID:server-59355739-66af-29c1-56b3-e1ede7aabe54 Address:59355739-66af-29c1-56b3-e1ede7aabe54} {Suffrage:Voter ID:server-1e6b386f-4665-4043-cebd-e105151e9194 Address:1e6b386f-4665-4043-cebd-e105151e9194}]" -2025-10-27T04:03:18.214Z [INFO] server-59355739-66af-29c1-56b3-e1ede7aabe54: entering follower state: follower="Node at 59355739-66af-29c1-56b3-e1ede7aabe54 [Follower]" leader-address= leader-id= -2025-10-27T04:03:18.214Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:18.214Z [INFO] server-1e6b386f-4665-4043-cebd-e105151e9194: entering follower state: follower="Node at 1e6b386f-4665-4043-cebd-e105151e9194 [Follower]" leader-address= leader-id= -2025-10-27T04:03:18.909Z [WARN] server-59355739-66af-29c1-56b3-e1ede7aabe54: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:18.909Z [INFO] server-59355739-66af-29c1-56b3-e1ede7aabe54: entering candidate state: node="Node at 59355739-66af-29c1-56b3-e1ede7aabe54 [Candidate]" term=2 -2025-10-27T04:03:18.910Z [INFO] server-59355739-66af-29c1-56b3-e1ede7aabe54: election won: term=2 tally=2 -2025-10-27T04:03:18.910Z [INFO] server-59355739-66af-29c1-56b3-e1ede7aabe54: entering leader state: leader="Node at 59355739-66af-29c1-56b3-e1ede7aabe54 [Leader]" -2025-10-27T04:03:18.910Z [INFO] server-59355739-66af-29c1-56b3-e1ede7aabe54: added peer, starting replication: peer=server-78c1a299-7e80-a8b9-e434-54bea00e3312 -2025-10-27T04:03:18.910Z [INFO] server-59355739-66af-29c1-56b3-e1ede7aabe54: added peer, starting replication: peer=server-1e6b386f-4665-4043-cebd-e105151e9194 -2025-10-27T04:03:18.910Z [INFO] server-59355739-66af-29c1-56b3-e1ede7aabe54: pipelining replication: peer="{Voter server-1e6b386f-4665-4043-cebd-e105151e9194 1e6b386f-4665-4043-cebd-e105151e9194}" -2025-10-27T04:03:18.910Z [INFO] server-59355739-66af-29c1-56b3-e1ede7aabe54: pipelining replication: peer="{Voter server-78c1a299-7e80-a8b9-e434-54bea00e3312 78c1a299-7e80-a8b9-e434-54bea00e3312}" -2025-10-27T04:03:19.916Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:18.910114309 +0000 UTC m=+73.904986747 (1 nodes), 695.780674ms from start of poll, 695.976115ms from cluster start. Timeout at 2025-10-27 04:03:19.915116549 +0000 UTC m=+74.909989027, 1.00500228s after stability -2025-10-27T04:03:19.916Z [INFO] server-59355739-66af-29c1-56b3-e1ede7aabe54: starting snapshot up to: index=12 -2025-10-27T04:03:19.916Z [INFO] creating new snapshot: path=/tmp/raft1281478497/snapshots/2-12-1761537799916.tmp -2025-10-27T04:03:19.916Z [INFO] server-59355739-66af-29c1-56b3-e1ede7aabe54: snapshot complete up to: index=12 -2025-10-27T04:03:19.917Z [INFO] creating new snapshot: path=/tmp/raft1281478497/snapshots/2-1013-1761537799917.tmp -2025-10-27T04:03:19.917Z [INFO] server-59355739-66af-29c1-56b3-e1ede7aabe54: copied to local snapshot: bytes=71 -2025-10-27T04:03:19.917Z [INFO] server-59355739-66af-29c1-56b3-e1ede7aabe54: snapshot restore progress: id=2-1013-1761537799917 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:19.917Z [INFO] server-59355739-66af-29c1-56b3-e1ede7aabe54: removing all old logs from log store -2025-10-27T04:03:19.917Z [INFO] server-59355739-66af-29c1-56b3-e1ede7aabe54: compacting logs: from=1 to=22 -2025-10-27T04:03:19.917Z [INFO] server-59355739-66af-29c1-56b3-e1ede7aabe54: restored user snapshot: index=1013 -2025-10-27T04:03:19.917Z [ERROR] server-59355739-66af-29c1-56b3-e1ede7aabe54: failed to get log: index=22 error="log not found" -2025-10-27T04:03:19.917Z [ERROR] server-59355739-66af-29c1-56b3-e1ede7aabe54: failed to get log: index=22 error="log not found" -2025-10-27T04:03:19.917Z [INFO] server-59355739-66af-29c1-56b3-e1ede7aabe54: aborting pipeline replication: peer="{Voter server-1e6b386f-4665-4043-cebd-e105151e9194 1e6b386f-4665-4043-cebd-e105151e9194}" -2025-10-27T04:03:19.917Z [INFO] server-59355739-66af-29c1-56b3-e1ede7aabe54: aborting pipeline replication: peer="{Voter server-78c1a299-7e80-a8b9-e434-54bea00e3312 78c1a299-7e80-a8b9-e434-54bea00e3312}" -2025-10-27T04:03:19.924Z [ERROR] server-59355739-66af-29c1-56b3-e1ede7aabe54: failed to get log: index=22 error="log not found" -2025-10-27T04:03:19.924Z [INFO] creating new snapshot: path=/tmp/raft262100495/snapshots/2-1013-1761537799924.tmp -2025-10-27T04:03:19.924Z [INFO] server-1e6b386f-4665-4043-cebd-e105151e9194: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:19.925Z [INFO] server-1e6b386f-4665-4043-cebd-e105151e9194: copied to local snapshot: bytes=71 -2025-10-27T04:03:19.925Z [INFO] server-1e6b386f-4665-4043-cebd-e105151e9194: snapshot restore progress: id=2-1013-1761537799924 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:19.925Z [INFO] server-1e6b386f-4665-4043-cebd-e105151e9194: removing all old logs from log store -2025-10-27T04:03:19.925Z [INFO] server-1e6b386f-4665-4043-cebd-e105151e9194: compacting logs: from=1 to=22 -2025-10-27T04:03:19.925Z [INFO] server-1e6b386f-4665-4043-cebd-e105151e9194: Installed remote snapshot -2025-10-27T04:03:19.925Z [WARN] server-1e6b386f-4665-4043-cebd-e105151e9194: skipping application of old log: index=22 -2025-10-27T04:03:19.925Z [INFO] server-59355739-66af-29c1-56b3-e1ede7aabe54: pipelining replication: peer="{Voter server-1e6b386f-4665-4043-cebd-e105151e9194 1e6b386f-4665-4043-cebd-e105151e9194}" -2025-10-27T04:03:19.925Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:03:19.927Z [ERROR] server-59355739-66af-29c1-56b3-e1ede7aabe54: failed to get log: index=22 error="log not found" -2025-10-27T04:03:19.928Z [INFO] creating new snapshot: path=/tmp/raft718040629/snapshots/2-1013-1761537799928.tmp -2025-10-27T04:03:19.928Z [INFO] server-78c1a299-7e80-a8b9-e434-54bea00e3312: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:19.928Z [INFO] server-78c1a299-7e80-a8b9-e434-54bea00e3312: copied to local snapshot: bytes=71 -2025-10-27T04:03:19.928Z [INFO] server-78c1a299-7e80-a8b9-e434-54bea00e3312: snapshot restore progress: id=2-1013-1761537799928 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:19.928Z [INFO] server-78c1a299-7e80-a8b9-e434-54bea00e3312: removing all old logs from log store -2025-10-27T04:03:19.928Z [INFO] server-78c1a299-7e80-a8b9-e434-54bea00e3312: compacting logs: from=1 to=22 -2025-10-27T04:03:19.928Z [INFO] server-78c1a299-7e80-a8b9-e434-54bea00e3312: Installed remote snapshot -2025-10-27T04:03:19.928Z [INFO] server-59355739-66af-29c1-56b3-e1ede7aabe54: pipelining replication: peer="{Voter server-78c1a299-7e80-a8b9-e434-54bea00e3312 78c1a299-7e80-a8b9-e434-54bea00e3312}" -2025-10-27T04:03:20.931Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:03:19.925582048 +0000 UTC m=+74.920454486 (2 nodes), 520ns from start of poll, 1.711443854s from cluster start. Timeout at 2025-10-27 04:03:20.930594248 +0000 UTC m=+75.925466726, 1.00501224s after stability -2025-10-27T04:03:20.942Z [INFO] server-59355739-66af-29c1-56b3-e1ede7aabe54: aborting pipeline replication: peer="{Voter server-1e6b386f-4665-4043-cebd-e105151e9194 1e6b386f-4665-4043-cebd-e105151e9194}" -2025-10-27T04:03:20.942Z [INFO] server-59355739-66af-29c1-56b3-e1ede7aabe54: aborting pipeline replication: peer="{Voter server-78c1a299-7e80-a8b9-e434-54bea00e3312 78c1a299-7e80-a8b9-e434-54bea00e3312}" +2026-11-29T10:29:20.571Z [INFO] server-88496e43-1c57-6fed-801f-6cf8e600282f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-88496e43-1c57-6fed-801f-6cf8e600282f Address:88496e43-1c57-6fed-801f-6cf8e600282f} {Suffrage:Voter ID:server-7c2a5e55-729c-d3e5-a0d4-b06499c85737 Address:7c2a5e55-729c-d3e5-a0d4-b06499c85737} {Suffrage:Voter ID:server-eb52b02e-51a0-cd80-8a48-c43e9972d384 Address:eb52b02e-51a0-cd80-8a48-c43e9972d384}]" +2026-11-29T10:29:20.572Z [INFO] server-7c2a5e55-729c-d3e5-a0d4-b06499c85737: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-88496e43-1c57-6fed-801f-6cf8e600282f Address:88496e43-1c57-6fed-801f-6cf8e600282f} {Suffrage:Voter ID:server-7c2a5e55-729c-d3e5-a0d4-b06499c85737 Address:7c2a5e55-729c-d3e5-a0d4-b06499c85737} {Suffrage:Voter ID:server-eb52b02e-51a0-cd80-8a48-c43e9972d384 Address:eb52b02e-51a0-cd80-8a48-c43e9972d384}]" +2026-11-29T10:29:20.572Z [INFO] server-88496e43-1c57-6fed-801f-6cf8e600282f: entering follower state: follower="Node at 88496e43-1c57-6fed-801f-6cf8e600282f [Follower]" leader-address= leader-id= +2026-11-29T10:29:20.572Z [INFO] server-7c2a5e55-729c-d3e5-a0d4-b06499c85737: entering follower state: follower="Node at 7c2a5e55-729c-d3e5-a0d4-b06499c85737 [Follower]" leader-address= leader-id= +2026-11-29T10:29:20.572Z [INFO] server-eb52b02e-51a0-cd80-8a48-c43e9972d384: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-88496e43-1c57-6fed-801f-6cf8e600282f Address:88496e43-1c57-6fed-801f-6cf8e600282f} {Suffrage:Voter ID:server-7c2a5e55-729c-d3e5-a0d4-b06499c85737 Address:7c2a5e55-729c-d3e5-a0d4-b06499c85737} {Suffrage:Voter ID:server-eb52b02e-51a0-cd80-8a48-c43e9972d384 Address:eb52b02e-51a0-cd80-8a48-c43e9972d384}]" +2026-11-29T10:29:20.572Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:20.572Z [INFO] server-eb52b02e-51a0-cd80-8a48-c43e9972d384: entering follower state: follower="Node at eb52b02e-51a0-cd80-8a48-c43e9972d384 [Follower]" leader-address= leader-id= +2026-11-29T10:29:21.144Z [WARN] server-7c2a5e55-729c-d3e5-a0d4-b06499c85737: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:21.145Z [INFO] server-7c2a5e55-729c-d3e5-a0d4-b06499c85737: entering candidate state: node="Node at 7c2a5e55-729c-d3e5-a0d4-b06499c85737 [Candidate]" term=2 +2026-11-29T10:29:21.145Z [INFO] server-7c2a5e55-729c-d3e5-a0d4-b06499c85737: election won: term=2 tally=2 +2026-11-29T10:29:21.145Z [INFO] server-7c2a5e55-729c-d3e5-a0d4-b06499c85737: entering leader state: leader="Node at 7c2a5e55-729c-d3e5-a0d4-b06499c85737 [Leader]" +2026-11-29T10:29:21.145Z [INFO] server-7c2a5e55-729c-d3e5-a0d4-b06499c85737: added peer, starting replication: peer=server-88496e43-1c57-6fed-801f-6cf8e600282f +2026-11-29T10:29:21.145Z [INFO] server-7c2a5e55-729c-d3e5-a0d4-b06499c85737: added peer, starting replication: peer=server-eb52b02e-51a0-cd80-8a48-c43e9972d384 +2026-11-29T10:29:21.145Z [INFO] server-7c2a5e55-729c-d3e5-a0d4-b06499c85737: pipelining replication: peer="{Voter server-88496e43-1c57-6fed-801f-6cf8e600282f 88496e43-1c57-6fed-801f-6cf8e600282f}" +2026-11-29T10:29:21.145Z [INFO] server-7c2a5e55-729c-d3e5-a0d4-b06499c85737: pipelining replication: peer="{Voter server-eb52b02e-51a0-cd80-8a48-c43e9972d384 eb52b02e-51a0-cd80-8a48-c43e9972d384}" +2026-11-29T10:29:22.151Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:21.145462259 +0000 UTC m=+71.880516321 (1 nodes), 573.27456ms from start of poll, 573.6404ms from cluster start. Timeout at 2026-11-29 10:29:22.150464419 +0000 UTC m=+72.885518601, 1.00500228s after stability +2026-11-29T10:29:22.151Z [INFO] server-7c2a5e55-729c-d3e5-a0d4-b06499c85737: starting snapshot up to: index=12 +2026-11-29T10:29:22.151Z [INFO] creating new snapshot: path=/tmp/raft2968781243/snapshots/2-12-1795948162151.tmp +2026-11-29T10:29:22.152Z [INFO] server-7c2a5e55-729c-d3e5-a0d4-b06499c85737: snapshot complete up to: index=12 +2026-11-29T10:29:22.152Z [INFO] creating new snapshot: path=/tmp/raft2968781243/snapshots/2-1013-1795948162152.tmp +2026-11-29T10:29:22.153Z [INFO] server-7c2a5e55-729c-d3e5-a0d4-b06499c85737: copied to local snapshot: bytes=71 +2026-11-29T10:29:22.153Z [INFO] server-7c2a5e55-729c-d3e5-a0d4-b06499c85737: snapshot restore progress: id=2-1013-1795948162152 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:22.153Z [INFO] server-7c2a5e55-729c-d3e5-a0d4-b06499c85737: removing all old logs from log store +2026-11-29T10:29:22.153Z [INFO] server-7c2a5e55-729c-d3e5-a0d4-b06499c85737: compacting logs: from=1 to=22 +2026-11-29T10:29:22.153Z [INFO] server-7c2a5e55-729c-d3e5-a0d4-b06499c85737: restored user snapshot: index=1013 +2026-11-29T10:29:22.153Z [ERROR] server-7c2a5e55-729c-d3e5-a0d4-b06499c85737: failed to get log: index=22 error="log not found" +2026-11-29T10:29:22.153Z [INFO] server-7c2a5e55-729c-d3e5-a0d4-b06499c85737: aborting pipeline replication: peer="{Voter server-88496e43-1c57-6fed-801f-6cf8e600282f 88496e43-1c57-6fed-801f-6cf8e600282f}" +2026-11-29T10:29:22.153Z [ERROR] server-7c2a5e55-729c-d3e5-a0d4-b06499c85737: failed to get log: index=22 error="log not found" +2026-11-29T10:29:22.153Z [INFO] server-7c2a5e55-729c-d3e5-a0d4-b06499c85737: aborting pipeline replication: peer="{Voter server-eb52b02e-51a0-cd80-8a48-c43e9972d384 eb52b02e-51a0-cd80-8a48-c43e9972d384}" +2026-11-29T10:29:22.160Z [ERROR] server-7c2a5e55-729c-d3e5-a0d4-b06499c85737: failed to get log: index=22 error="log not found" +2026-11-29T10:29:22.160Z [INFO] creating new snapshot: path=/tmp/raft1145560464/snapshots/2-1013-1795948162160.tmp +2026-11-29T10:29:22.160Z [INFO] server-88496e43-1c57-6fed-801f-6cf8e600282f: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:22.160Z [INFO] server-88496e43-1c57-6fed-801f-6cf8e600282f: copied to local snapshot: bytes=71 +2026-11-29T10:29:22.160Z [INFO] server-88496e43-1c57-6fed-801f-6cf8e600282f: snapshot restore progress: id=2-1013-1795948162160 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:22.160Z [INFO] server-88496e43-1c57-6fed-801f-6cf8e600282f: removing all old logs from log store +2026-11-29T10:29:22.160Z [INFO] server-88496e43-1c57-6fed-801f-6cf8e600282f: compacting logs: from=1 to=22 +2026-11-29T10:29:22.160Z [INFO] server-88496e43-1c57-6fed-801f-6cf8e600282f: Installed remote snapshot +2026-11-29T10:29:22.161Z [WARN] server-88496e43-1c57-6fed-801f-6cf8e600282f: skipping application of old log: index=22 +2026-11-29T10:29:22.161Z [INFO] server-7c2a5e55-729c-d3e5-a0d4-b06499c85737: pipelining replication: peer="{Voter server-88496e43-1c57-6fed-801f-6cf8e600282f 88496e43-1c57-6fed-801f-6cf8e600282f}" +2026-11-29T10:29:22.161Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:22.162Z [ERROR] server-7c2a5e55-729c-d3e5-a0d4-b06499c85737: failed to get log: index=22 error="log not found" +2026-11-29T10:29:22.163Z [INFO] creating new snapshot: path=/tmp/raft905158099/snapshots/2-1013-1795948162163.tmp +2026-11-29T10:29:22.163Z [INFO] server-eb52b02e-51a0-cd80-8a48-c43e9972d384: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:22.164Z [INFO] server-eb52b02e-51a0-cd80-8a48-c43e9972d384: copied to local snapshot: bytes=71 +2026-11-29T10:29:22.164Z [INFO] server-eb52b02e-51a0-cd80-8a48-c43e9972d384: snapshot restore progress: id=2-1013-1795948162163 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:22.164Z [INFO] server-eb52b02e-51a0-cd80-8a48-c43e9972d384: removing all old logs from log store +2026-11-29T10:29:22.164Z [INFO] server-eb52b02e-51a0-cd80-8a48-c43e9972d384: compacting logs: from=1 to=22 +2026-11-29T10:29:22.164Z [INFO] server-eb52b02e-51a0-cd80-8a48-c43e9972d384: Installed remote snapshot +2026-11-29T10:29:22.164Z [INFO] server-7c2a5e55-729c-d3e5-a0d4-b06499c85737: pipelining replication: peer="{Voter server-eb52b02e-51a0-cd80-8a48-c43e9972d384 eb52b02e-51a0-cd80-8a48-c43e9972d384}" +2026-11-29T10:29:23.166Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:22.161122619 +0000 UTC m=+72.896176761 (2 nodes), 1.04µs from start of poll, 1.58930084s from cluster start. Timeout at 2026-11-29 10:29:23.166137739 +0000 UTC m=+73.901191841, 1.00501508s after stability +2026-11-29T10:29:23.178Z [INFO] server-7c2a5e55-729c-d3e5-a0d4-b06499c85737: aborting pipeline replication: peer="{Voter server-88496e43-1c57-6fed-801f-6cf8e600282f 88496e43-1c57-6fed-801f-6cf8e600282f}" +2026-11-29T10:29:23.178Z [INFO] server-7c2a5e55-729c-d3e5-a0d4-b06499c85737: aborting pipeline replication: peer="{Voter server-eb52b02e-51a0-cd80-8a48-c43e9972d384 eb52b02e-51a0-cd80-8a48-c43e9972d384}" === RUN TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_true -2025-10-27T04:03:20.942Z [INFO] server-7a1d3e79-5c6a-05ab-8020-e1ebee18206e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7a1d3e79-5c6a-05ab-8020-e1ebee18206e Address:7a1d3e79-5c6a-05ab-8020-e1ebee18206e} {Suffrage:Voter ID:server-fb4a45fd-c5bc-cc8b-b9a7-c2ceba549cc7 Address:fb4a45fd-c5bc-cc8b-b9a7-c2ceba549cc7} {Suffrage:Voter ID:server-d77e36c7-23f5-5aa5-a4eb-fc0b1d67ea8a Address:d77e36c7-23f5-5aa5-a4eb-fc0b1d67ea8a}]" -2025-10-27T04:03:20.942Z [INFO] server-fb4a45fd-c5bc-cc8b-b9a7-c2ceba549cc7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7a1d3e79-5c6a-05ab-8020-e1ebee18206e Address:7a1d3e79-5c6a-05ab-8020-e1ebee18206e} {Suffrage:Voter ID:server-fb4a45fd-c5bc-cc8b-b9a7-c2ceba549cc7 Address:fb4a45fd-c5bc-cc8b-b9a7-c2ceba549cc7} {Suffrage:Voter ID:server-d77e36c7-23f5-5aa5-a4eb-fc0b1d67ea8a Address:d77e36c7-23f5-5aa5-a4eb-fc0b1d67ea8a}]" -2025-10-27T04:03:20.942Z [INFO] server-d77e36c7-23f5-5aa5-a4eb-fc0b1d67ea8a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7a1d3e79-5c6a-05ab-8020-e1ebee18206e Address:7a1d3e79-5c6a-05ab-8020-e1ebee18206e} {Suffrage:Voter ID:server-fb4a45fd-c5bc-cc8b-b9a7-c2ceba549cc7 Address:fb4a45fd-c5bc-cc8b-b9a7-c2ceba549cc7} {Suffrage:Voter ID:server-d77e36c7-23f5-5aa5-a4eb-fc0b1d67ea8a Address:d77e36c7-23f5-5aa5-a4eb-fc0b1d67ea8a}]" -2025-10-27T04:03:20.942Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:20.942Z [INFO] server-7a1d3e79-5c6a-05ab-8020-e1ebee18206e: entering follower state: follower="Node at 7a1d3e79-5c6a-05ab-8020-e1ebee18206e [Follower]" leader-address= leader-id= -2025-10-27T04:03:20.942Z [INFO] server-fb4a45fd-c5bc-cc8b-b9a7-c2ceba549cc7: entering follower state: follower="Node at fb4a45fd-c5bc-cc8b-b9a7-c2ceba549cc7 [Follower]" leader-address= leader-id= -2025-10-27T04:03:20.943Z [INFO] server-d77e36c7-23f5-5aa5-a4eb-fc0b1d67ea8a: entering follower state: follower="Node at d77e36c7-23f5-5aa5-a4eb-fc0b1d67ea8a [Follower]" leader-address= leader-id= -2025-10-27T04:03:21.636Z [WARN] server-7a1d3e79-5c6a-05ab-8020-e1ebee18206e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:21.636Z [INFO] server-7a1d3e79-5c6a-05ab-8020-e1ebee18206e: entering candidate state: node="Node at 7a1d3e79-5c6a-05ab-8020-e1ebee18206e [Candidate]" term=2 -2025-10-27T04:03:21.636Z [INFO] server-7a1d3e79-5c6a-05ab-8020-e1ebee18206e: election won: term=2 tally=2 -2025-10-27T04:03:21.636Z [INFO] server-7a1d3e79-5c6a-05ab-8020-e1ebee18206e: entering leader state: leader="Node at 7a1d3e79-5c6a-05ab-8020-e1ebee18206e [Leader]" -2025-10-27T04:03:21.636Z [INFO] server-7a1d3e79-5c6a-05ab-8020-e1ebee18206e: added peer, starting replication: peer=server-fb4a45fd-c5bc-cc8b-b9a7-c2ceba549cc7 -2025-10-27T04:03:21.636Z [INFO] server-7a1d3e79-5c6a-05ab-8020-e1ebee18206e: added peer, starting replication: peer=server-d77e36c7-23f5-5aa5-a4eb-fc0b1d67ea8a -2025-10-27T04:03:21.636Z [INFO] server-7a1d3e79-5c6a-05ab-8020-e1ebee18206e: pipelining replication: peer="{Voter server-d77e36c7-23f5-5aa5-a4eb-fc0b1d67ea8a d77e36c7-23f5-5aa5-a4eb-fc0b1d67ea8a}" -2025-10-27T04:03:21.636Z [INFO] server-7a1d3e79-5c6a-05ab-8020-e1ebee18206e: pipelining replication: peer="{Voter server-fb4a45fd-c5bc-cc8b-b9a7-c2ceba549cc7 fb4a45fd-c5bc-cc8b-b9a7-c2ceba549cc7}" -2025-10-27T04:03:22.641Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:21.636508098 +0000 UTC m=+76.631380536 (1 nodes), 693.538099ms from start of poll, 693.71514ms from cluster start. Timeout at 2025-10-27 04:03:22.641509778 +0000 UTC m=+77.636382256, 1.00500172s after stability -2025-10-27T04:03:22.642Z [INFO] server-7a1d3e79-5c6a-05ab-8020-e1ebee18206e: starting snapshot up to: index=12 -2025-10-27T04:03:22.642Z [INFO] creating new snapshot: path=/tmp/raft2759321829/snapshots/2-12-1761537802642.tmp -2025-10-27T04:03:22.642Z [INFO] server-7a1d3e79-5c6a-05ab-8020-e1ebee18206e: snapshot complete up to: index=12 -2025-10-27T04:03:22.643Z [INFO] server-14fc7ac5-e491-03e9-85f4-891a07124701: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-14fc7ac5-e491-03e9-85f4-891a07124701 Address:14fc7ac5-e491-03e9-85f4-891a07124701} {Suffrage:Voter ID:server-f2dab184-5e73-4200-81c5-fafe632552a7 Address:f2dab184-5e73-4200-81c5-fafe632552a7} {Suffrage:Voter ID:server-0ef79533-f851-9577-b15d-322d76190dfc Address:0ef79533-f851-9577-b15d-322d76190dfc}]" -2025-10-27T04:03:22.643Z [INFO] server-14fc7ac5-e491-03e9-85f4-891a07124701: entering follower state: follower="Node at 14fc7ac5-e491-03e9-85f4-891a07124701 [Follower]" leader-address= leader-id= -2025-10-27T04:03:22.643Z [INFO] server-f2dab184-5e73-4200-81c5-fafe632552a7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-14fc7ac5-e491-03e9-85f4-891a07124701 Address:14fc7ac5-e491-03e9-85f4-891a07124701} {Suffrage:Voter ID:server-f2dab184-5e73-4200-81c5-fafe632552a7 Address:f2dab184-5e73-4200-81c5-fafe632552a7} {Suffrage:Voter ID:server-0ef79533-f851-9577-b15d-322d76190dfc Address:0ef79533-f851-9577-b15d-322d76190dfc}]" -2025-10-27T04:03:22.644Z [INFO] server-f2dab184-5e73-4200-81c5-fafe632552a7: entering follower state: follower="Node at f2dab184-5e73-4200-81c5-fafe632552a7 [Follower]" leader-address= leader-id= -2025-10-27T04:03:22.644Z [INFO] server-0ef79533-f851-9577-b15d-322d76190dfc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-14fc7ac5-e491-03e9-85f4-891a07124701 Address:14fc7ac5-e491-03e9-85f4-891a07124701} {Suffrage:Voter ID:server-f2dab184-5e73-4200-81c5-fafe632552a7 Address:f2dab184-5e73-4200-81c5-fafe632552a7} {Suffrage:Voter ID:server-0ef79533-f851-9577-b15d-322d76190dfc Address:0ef79533-f851-9577-b15d-322d76190dfc}]" -2025-10-27T04:03:22.644Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:22.644Z [INFO] server-0ef79533-f851-9577-b15d-322d76190dfc: entering follower state: follower="Node at 0ef79533-f851-9577-b15d-322d76190dfc [Follower]" leader-address= leader-id= -2025-10-27T04:03:23.263Z [WARN] server-0ef79533-f851-9577-b15d-322d76190dfc: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:23.263Z [INFO] server-0ef79533-f851-9577-b15d-322d76190dfc: entering candidate state: node="Node at 0ef79533-f851-9577-b15d-322d76190dfc [Candidate]" term=2 -2025-10-27T04:03:23.263Z [INFO] server-0ef79533-f851-9577-b15d-322d76190dfc: election won: term=2 tally=2 -2025-10-27T04:03:23.263Z [INFO] server-0ef79533-f851-9577-b15d-322d76190dfc: entering leader state: leader="Node at 0ef79533-f851-9577-b15d-322d76190dfc [Leader]" -2025-10-27T04:03:23.263Z [INFO] server-0ef79533-f851-9577-b15d-322d76190dfc: added peer, starting replication: peer=server-14fc7ac5-e491-03e9-85f4-891a07124701 -2025-10-27T04:03:23.263Z [INFO] server-0ef79533-f851-9577-b15d-322d76190dfc: added peer, starting replication: peer=server-f2dab184-5e73-4200-81c5-fafe632552a7 -2025-10-27T04:03:23.263Z [INFO] server-0ef79533-f851-9577-b15d-322d76190dfc: pipelining replication: peer="{Voter server-f2dab184-5e73-4200-81c5-fafe632552a7 f2dab184-5e73-4200-81c5-fafe632552a7}" -2025-10-27T04:03:23.263Z [INFO] server-0ef79533-f851-9577-b15d-322d76190dfc: pipelining replication: peer="{Voter server-14fc7ac5-e491-03e9-85f4-891a07124701 14fc7ac5-e491-03e9-85f4-891a07124701}" -2025-10-27T04:03:24.269Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:23.263288733 +0000 UTC m=+78.258161211 (1 nodes), 618.880269ms from start of poll, 619.764995ms from cluster start. Timeout at 2025-10-27 04:03:24.268291133 +0000 UTC m=+79.263163651, 1.00500244s after stability -2025-10-27T04:03:24.269Z [INFO] creating new snapshot: path=/tmp/raft456702098/snapshots/2-1013-1761537804269.tmp -2025-10-27T04:03:24.270Z [INFO] server-0ef79533-f851-9577-b15d-322d76190dfc: copied to local snapshot: bytes=71 -2025-10-27T04:03:24.270Z [INFO] server-0ef79533-f851-9577-b15d-322d76190dfc: snapshot restore progress: id=2-1013-1761537804269 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:24.270Z [INFO] server-0ef79533-f851-9577-b15d-322d76190dfc: restored user snapshot: index=1013 -2025-10-27T04:03:24.270Z [ERROR] server-0ef79533-f851-9577-b15d-322d76190dfc: failed to get log: index=3 error="log not found" -2025-10-27T04:03:24.270Z [INFO] server-0ef79533-f851-9577-b15d-322d76190dfc: aborting pipeline replication: peer="{Voter server-f2dab184-5e73-4200-81c5-fafe632552a7 f2dab184-5e73-4200-81c5-fafe632552a7}" -2025-10-27T04:03:24.270Z [ERROR] server-0ef79533-f851-9577-b15d-322d76190dfc: failed to get log: index=3 error="log not found" -2025-10-27T04:03:24.270Z [INFO] server-0ef79533-f851-9577-b15d-322d76190dfc: aborting pipeline replication: peer="{Voter server-14fc7ac5-e491-03e9-85f4-891a07124701 14fc7ac5-e491-03e9-85f4-891a07124701}" -2025-10-27T04:03:24.278Z [ERROR] server-0ef79533-f851-9577-b15d-322d76190dfc: failed to get log: index=3 error="log not found" -2025-10-27T04:03:24.279Z [ERROR] server-0ef79533-f851-9577-b15d-322d76190dfc: failed to get log: index=3 error="log not found" -2025-10-27T04:03:24.279Z [INFO] creating new snapshot: path=/tmp/raft845997696/snapshots/2-1013-1761537804279.tmp -2025-10-27T04:03:24.279Z [INFO] creating new snapshot: path=/tmp/raft3775013087/snapshots/2-1013-1761537804279.tmp -2025-10-27T04:03:24.279Z [INFO] server-f2dab184-5e73-4200-81c5-fafe632552a7: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:24.279Z [INFO] server-f2dab184-5e73-4200-81c5-fafe632552a7: copied to local snapshot: bytes=71 -2025-10-27T04:03:24.279Z [INFO] server-14fc7ac5-e491-03e9-85f4-891a07124701: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:24.279Z [INFO] server-f2dab184-5e73-4200-81c5-fafe632552a7: snapshot restore progress: id=2-1013-1761537804279 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:24.279Z [INFO] server-f2dab184-5e73-4200-81c5-fafe632552a7: Installed remote snapshot -2025-10-27T04:03:24.279Z [INFO] server-0ef79533-f851-9577-b15d-322d76190dfc: pipelining replication: peer="{Voter server-f2dab184-5e73-4200-81c5-fafe632552a7 f2dab184-5e73-4200-81c5-fafe632552a7}" -2025-10-27T04:03:24.279Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:03:24.279Z [INFO] server-14fc7ac5-e491-03e9-85f4-891a07124701: copied to local snapshot: bytes=71 -2025-10-27T04:03:24.280Z [INFO] server-14fc7ac5-e491-03e9-85f4-891a07124701: snapshot restore progress: id=2-1013-1761537804279 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:24.280Z [INFO] server-14fc7ac5-e491-03e9-85f4-891a07124701: Installed remote snapshot -2025-10-27T04:03:24.280Z [INFO] server-0ef79533-f851-9577-b15d-322d76190dfc: pipelining replication: peer="{Voter server-14fc7ac5-e491-03e9-85f4-891a07124701 14fc7ac5-e491-03e9-85f4-891a07124701}" -2025-10-27T04:03:25.285Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:03:24.279908239 +0000 UTC m=+79.274780677 (2 nodes), 1µs from start of poll, 1.636384461s from cluster start. Timeout at 2025-10-27 04:03:25.28492664 +0000 UTC m=+80.279799078, 1.005018401s after stability -2025-10-27T04:03:25.301Z [INFO] server-7a1d3e79-5c6a-05ab-8020-e1ebee18206e: aborting pipeline replication: peer="{Voter server-d77e36c7-23f5-5aa5-a4eb-fc0b1d67ea8a d77e36c7-23f5-5aa5-a4eb-fc0b1d67ea8a}" -2025-10-27T04:03:25.301Z [INFO] server-7a1d3e79-5c6a-05ab-8020-e1ebee18206e: aborting pipeline replication: peer="{Voter server-fb4a45fd-c5bc-cc8b-b9a7-c2ceba549cc7 fb4a45fd-c5bc-cc8b-b9a7-c2ceba549cc7}" +2026-11-29T10:29:23.179Z [INFO] server-af58a943-1a6c-1984-98fc-f3683d0bd263: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-af58a943-1a6c-1984-98fc-f3683d0bd263 Address:af58a943-1a6c-1984-98fc-f3683d0bd263} {Suffrage:Voter ID:server-ae915973-df19-95ab-a00e-0670546685c9 Address:ae915973-df19-95ab-a00e-0670546685c9} {Suffrage:Voter ID:server-6fe53108-2aff-89cd-9e0a-2174a010b1f9 Address:6fe53108-2aff-89cd-9e0a-2174a010b1f9}]" +2026-11-29T10:29:23.179Z [INFO] server-af58a943-1a6c-1984-98fc-f3683d0bd263: entering follower state: follower="Node at af58a943-1a6c-1984-98fc-f3683d0bd263 [Follower]" leader-address= leader-id= +2026-11-29T10:29:23.180Z [INFO] server-ae915973-df19-95ab-a00e-0670546685c9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-af58a943-1a6c-1984-98fc-f3683d0bd263 Address:af58a943-1a6c-1984-98fc-f3683d0bd263} {Suffrage:Voter ID:server-ae915973-df19-95ab-a00e-0670546685c9 Address:ae915973-df19-95ab-a00e-0670546685c9} {Suffrage:Voter ID:server-6fe53108-2aff-89cd-9e0a-2174a010b1f9 Address:6fe53108-2aff-89cd-9e0a-2174a010b1f9}]" +2026-11-29T10:29:23.180Z [INFO] server-ae915973-df19-95ab-a00e-0670546685c9: entering follower state: follower="Node at ae915973-df19-95ab-a00e-0670546685c9 [Follower]" leader-address= leader-id= +2026-11-29T10:29:23.180Z [INFO] server-6fe53108-2aff-89cd-9e0a-2174a010b1f9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-af58a943-1a6c-1984-98fc-f3683d0bd263 Address:af58a943-1a6c-1984-98fc-f3683d0bd263} {Suffrage:Voter ID:server-ae915973-df19-95ab-a00e-0670546685c9 Address:ae915973-df19-95ab-a00e-0670546685c9} {Suffrage:Voter ID:server-6fe53108-2aff-89cd-9e0a-2174a010b1f9 Address:6fe53108-2aff-89cd-9e0a-2174a010b1f9}]" +2026-11-29T10:29:23.180Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:23.180Z [INFO] server-6fe53108-2aff-89cd-9e0a-2174a010b1f9: entering follower state: follower="Node at 6fe53108-2aff-89cd-9e0a-2174a010b1f9 [Follower]" leader-address= leader-id= +2026-11-29T10:29:23.801Z [WARN] server-6fe53108-2aff-89cd-9e0a-2174a010b1f9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:23.801Z [INFO] server-6fe53108-2aff-89cd-9e0a-2174a010b1f9: entering candidate state: node="Node at 6fe53108-2aff-89cd-9e0a-2174a010b1f9 [Candidate]" term=2 +2026-11-29T10:29:23.801Z [INFO] server-6fe53108-2aff-89cd-9e0a-2174a010b1f9: election won: term=2 tally=2 +2026-11-29T10:29:23.801Z [INFO] server-6fe53108-2aff-89cd-9e0a-2174a010b1f9: entering leader state: leader="Node at 6fe53108-2aff-89cd-9e0a-2174a010b1f9 [Leader]" +2026-11-29T10:29:23.801Z [INFO] server-6fe53108-2aff-89cd-9e0a-2174a010b1f9: added peer, starting replication: peer=server-af58a943-1a6c-1984-98fc-f3683d0bd263 +2026-11-29T10:29:23.801Z [INFO] server-6fe53108-2aff-89cd-9e0a-2174a010b1f9: added peer, starting replication: peer=server-ae915973-df19-95ab-a00e-0670546685c9 +2026-11-29T10:29:23.802Z [INFO] server-6fe53108-2aff-89cd-9e0a-2174a010b1f9: pipelining replication: peer="{Voter server-ae915973-df19-95ab-a00e-0670546685c9 ae915973-df19-95ab-a00e-0670546685c9}" +2026-11-29T10:29:23.802Z [INFO] server-6fe53108-2aff-89cd-9e0a-2174a010b1f9: pipelining replication: peer="{Voter server-af58a943-1a6c-1984-98fc-f3683d0bd263 af58a943-1a6c-1984-98fc-f3683d0bd263}" +2026-11-29T10:29:24.807Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:23.801985899 +0000 UTC m=+74.537040041 (1 nodes), 621.7948ms from start of poll, 622.23236ms from cluster start. Timeout at 2026-11-29 10:29:24.806987299 +0000 UTC m=+75.542041401, 1.00500136s after stability +2026-11-29T10:29:24.808Z [INFO] server-6fe53108-2aff-89cd-9e0a-2174a010b1f9: starting snapshot up to: index=12 +2026-11-29T10:29:24.808Z [INFO] creating new snapshot: path=/tmp/raft1594842231/snapshots/2-12-1795948164808.tmp +2026-11-29T10:29:24.808Z [INFO] server-6fe53108-2aff-89cd-9e0a-2174a010b1f9: snapshot complete up to: index=12 +2026-11-29T10:29:24.809Z [INFO] server-e39a9374-5ca9-9828-50d8-a22f8407239f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e39a9374-5ca9-9828-50d8-a22f8407239f Address:e39a9374-5ca9-9828-50d8-a22f8407239f} {Suffrage:Voter ID:server-e37f4a4c-6095-cfcb-eecb-2ccabdca133c Address:e37f4a4c-6095-cfcb-eecb-2ccabdca133c} {Suffrage:Voter ID:server-3bb173fb-5d74-a511-94bc-cb289e72e35b Address:3bb173fb-5d74-a511-94bc-cb289e72e35b}]" +2026-11-29T10:29:24.809Z [INFO] server-e39a9374-5ca9-9828-50d8-a22f8407239f: entering follower state: follower="Node at e39a9374-5ca9-9828-50d8-a22f8407239f [Follower]" leader-address= leader-id= +2026-11-29T10:29:24.809Z [INFO] server-e37f4a4c-6095-cfcb-eecb-2ccabdca133c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e39a9374-5ca9-9828-50d8-a22f8407239f Address:e39a9374-5ca9-9828-50d8-a22f8407239f} {Suffrage:Voter ID:server-e37f4a4c-6095-cfcb-eecb-2ccabdca133c Address:e37f4a4c-6095-cfcb-eecb-2ccabdca133c} {Suffrage:Voter ID:server-3bb173fb-5d74-a511-94bc-cb289e72e35b Address:3bb173fb-5d74-a511-94bc-cb289e72e35b}]" +2026-11-29T10:29:24.810Z [INFO] server-3bb173fb-5d74-a511-94bc-cb289e72e35b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e39a9374-5ca9-9828-50d8-a22f8407239f Address:e39a9374-5ca9-9828-50d8-a22f8407239f} {Suffrage:Voter ID:server-e37f4a4c-6095-cfcb-eecb-2ccabdca133c Address:e37f4a4c-6095-cfcb-eecb-2ccabdca133c} {Suffrage:Voter ID:server-3bb173fb-5d74-a511-94bc-cb289e72e35b Address:3bb173fb-5d74-a511-94bc-cb289e72e35b}]" +2026-11-29T10:29:24.809Z [INFO] server-e37f4a4c-6095-cfcb-eecb-2ccabdca133c: entering follower state: follower="Node at e37f4a4c-6095-cfcb-eecb-2ccabdca133c [Follower]" leader-address= leader-id= +2026-11-29T10:29:24.810Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:24.810Z [INFO] server-3bb173fb-5d74-a511-94bc-cb289e72e35b: entering follower state: follower="Node at 3bb173fb-5d74-a511-94bc-cb289e72e35b [Follower]" leader-address= leader-id= +2026-11-29T10:29:25.423Z [WARN] server-e39a9374-5ca9-9828-50d8-a22f8407239f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:25.423Z [INFO] server-e39a9374-5ca9-9828-50d8-a22f8407239f: entering candidate state: node="Node at e39a9374-5ca9-9828-50d8-a22f8407239f [Candidate]" term=2 +2026-11-29T10:29:25.423Z [INFO] server-e39a9374-5ca9-9828-50d8-a22f8407239f: election won: term=2 tally=2 +2026-11-29T10:29:25.423Z [INFO] server-e39a9374-5ca9-9828-50d8-a22f8407239f: entering leader state: leader="Node at e39a9374-5ca9-9828-50d8-a22f8407239f [Leader]" +2026-11-29T10:29:25.423Z [INFO] server-e39a9374-5ca9-9828-50d8-a22f8407239f: added peer, starting replication: peer=server-e37f4a4c-6095-cfcb-eecb-2ccabdca133c +2026-11-29T10:29:25.423Z [INFO] server-e39a9374-5ca9-9828-50d8-a22f8407239f: added peer, starting replication: peer=server-3bb173fb-5d74-a511-94bc-cb289e72e35b +2026-11-29T10:29:25.423Z [INFO] server-e39a9374-5ca9-9828-50d8-a22f8407239f: pipelining replication: peer="{Voter server-3bb173fb-5d74-a511-94bc-cb289e72e35b 3bb173fb-5d74-a511-94bc-cb289e72e35b}" +2026-11-29T10:29:25.423Z [INFO] server-e39a9374-5ca9-9828-50d8-a22f8407239f: pipelining replication: peer="{Voter server-e37f4a4c-6095-cfcb-eecb-2ccabdca133c e37f4a4c-6095-cfcb-eecb-2ccabdca133c}" +2026-11-29T10:29:26.428Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:25.423810699 +0000 UTC m=+76.158864841 (1 nodes), 613.73388ms from start of poll, 614.10532ms from cluster start. Timeout at 2026-11-29 10:29:26.428816499 +0000 UTC m=+77.163870561, 1.00500572s after stability +2026-11-29T10:29:26.429Z [INFO] creating new snapshot: path=/tmp/raft4122725401/snapshots/2-1013-1795948166429.tmp +2026-11-29T10:29:26.429Z [INFO] server-e39a9374-5ca9-9828-50d8-a22f8407239f: copied to local snapshot: bytes=71 +2026-11-29T10:29:26.429Z [INFO] server-e39a9374-5ca9-9828-50d8-a22f8407239f: snapshot restore progress: id=2-1013-1795948166429 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:26.429Z [INFO] server-e39a9374-5ca9-9828-50d8-a22f8407239f: restored user snapshot: index=1013 +2026-11-29T10:29:26.429Z [ERROR] server-e39a9374-5ca9-9828-50d8-a22f8407239f: failed to get log: index=3 error="log not found" +2026-11-29T10:29:26.429Z [INFO] server-e39a9374-5ca9-9828-50d8-a22f8407239f: aborting pipeline replication: peer="{Voter server-3bb173fb-5d74-a511-94bc-cb289e72e35b 3bb173fb-5d74-a511-94bc-cb289e72e35b}" +2026-11-29T10:29:26.429Z [ERROR] server-e39a9374-5ca9-9828-50d8-a22f8407239f: failed to get log: index=3 error="log not found" +2026-11-29T10:29:26.429Z [INFO] server-e39a9374-5ca9-9828-50d8-a22f8407239f: aborting pipeline replication: peer="{Voter server-e37f4a4c-6095-cfcb-eecb-2ccabdca133c e37f4a4c-6095-cfcb-eecb-2ccabdca133c}" +2026-11-29T10:29:26.439Z [ERROR] server-e39a9374-5ca9-9828-50d8-a22f8407239f: failed to get log: index=3 error="log not found" +2026-11-29T10:29:26.439Z [ERROR] server-e39a9374-5ca9-9828-50d8-a22f8407239f: failed to get log: index=3 error="log not found" +2026-11-29T10:29:26.439Z [INFO] creating new snapshot: path=/tmp/raft4040341519/snapshots/2-1013-1795948166439.tmp +2026-11-29T10:29:26.439Z [INFO] creating new snapshot: path=/tmp/raft1010032094/snapshots/2-1013-1795948166439.tmp +2026-11-29T10:29:26.440Z [INFO] server-3bb173fb-5d74-a511-94bc-cb289e72e35b: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:26.440Z [INFO] server-3bb173fb-5d74-a511-94bc-cb289e72e35b: copied to local snapshot: bytes=71 +2026-11-29T10:29:26.441Z [INFO] server-3bb173fb-5d74-a511-94bc-cb289e72e35b: snapshot restore progress: id=2-1013-1795948166439 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:26.441Z [INFO] server-3bb173fb-5d74-a511-94bc-cb289e72e35b: Installed remote snapshot +2026-11-29T10:29:26.441Z [INFO] server-e39a9374-5ca9-9828-50d8-a22f8407239f: pipelining replication: peer="{Voter server-3bb173fb-5d74-a511-94bc-cb289e72e35b 3bb173fb-5d74-a511-94bc-cb289e72e35b}" +2026-11-29T10:29:26.441Z [INFO] server-e37f4a4c-6095-cfcb-eecb-2ccabdca133c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:26.441Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:26.441Z [INFO] server-e37f4a4c-6095-cfcb-eecb-2ccabdca133c: copied to local snapshot: bytes=71 +2026-11-29T10:29:26.442Z [INFO] server-e37f4a4c-6095-cfcb-eecb-2ccabdca133c: snapshot restore progress: id=2-1013-1795948166439 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:26.442Z [INFO] server-e37f4a4c-6095-cfcb-eecb-2ccabdca133c: Installed remote snapshot +2026-11-29T10:29:26.442Z [INFO] server-e39a9374-5ca9-9828-50d8-a22f8407239f: pipelining replication: peer="{Voter server-e37f4a4c-6095-cfcb-eecb-2ccabdca133c e37f4a4c-6095-cfcb-eecb-2ccabdca133c}" +2026-11-29T10:29:27.446Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:26.441184339 +0000 UTC m=+77.176238401 (2 nodes), 360ns from start of poll, 1.63147888s from cluster start. Timeout at 2026-11-29 10:29:27.446207979 +0000 UTC m=+78.181262161, 1.00502376s after stability +2026-11-29T10:29:27.458Z [INFO] server-6fe53108-2aff-89cd-9e0a-2174a010b1f9: aborting pipeline replication: peer="{Voter server-ae915973-df19-95ab-a00e-0670546685c9 ae915973-df19-95ab-a00e-0670546685c9}" +2026-11-29T10:29:27.458Z [INFO] server-6fe53108-2aff-89cd-9e0a-2174a010b1f9: aborting pipeline replication: peer="{Voter server-af58a943-1a6c-1984-98fc-f3683d0bd263 af58a943-1a6c-1984-98fc-f3683d0bd263}" === RUN TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_true -2025-10-27T04:03:25.302Z [INFO] server-a3bf5301-9f81-5083-9f79-14bf60db99d4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a3bf5301-9f81-5083-9f79-14bf60db99d4 Address:a3bf5301-9f81-5083-9f79-14bf60db99d4} {Suffrage:Voter ID:server-5d975827-46b8-d55f-6d05-79ee3deceb6d Address:5d975827-46b8-d55f-6d05-79ee3deceb6d} {Suffrage:Voter ID:server-f892f9eb-43c7-836f-62f2-750af82387a0 Address:f892f9eb-43c7-836f-62f2-750af82387a0}]" -2025-10-27T04:03:25.302Z [INFO] server-5d975827-46b8-d55f-6d05-79ee3deceb6d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a3bf5301-9f81-5083-9f79-14bf60db99d4 Address:a3bf5301-9f81-5083-9f79-14bf60db99d4} {Suffrage:Voter ID:server-5d975827-46b8-d55f-6d05-79ee3deceb6d Address:5d975827-46b8-d55f-6d05-79ee3deceb6d} {Suffrage:Voter ID:server-f892f9eb-43c7-836f-62f2-750af82387a0 Address:f892f9eb-43c7-836f-62f2-750af82387a0}]" -2025-10-27T04:03:25.302Z [INFO] server-a3bf5301-9f81-5083-9f79-14bf60db99d4: entering follower state: follower="Node at a3bf5301-9f81-5083-9f79-14bf60db99d4 [Follower]" leader-address= leader-id= -2025-10-27T04:03:25.302Z [INFO] server-f892f9eb-43c7-836f-62f2-750af82387a0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a3bf5301-9f81-5083-9f79-14bf60db99d4 Address:a3bf5301-9f81-5083-9f79-14bf60db99d4} {Suffrage:Voter ID:server-5d975827-46b8-d55f-6d05-79ee3deceb6d Address:5d975827-46b8-d55f-6d05-79ee3deceb6d} {Suffrage:Voter ID:server-f892f9eb-43c7-836f-62f2-750af82387a0 Address:f892f9eb-43c7-836f-62f2-750af82387a0}]" -2025-10-27T04:03:25.302Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:25.302Z [INFO] server-f892f9eb-43c7-836f-62f2-750af82387a0: entering follower state: follower="Node at f892f9eb-43c7-836f-62f2-750af82387a0 [Follower]" leader-address= leader-id= -2025-10-27T04:03:25.302Z [INFO] server-5d975827-46b8-d55f-6d05-79ee3deceb6d: entering follower state: follower="Node at 5d975827-46b8-d55f-6d05-79ee3deceb6d [Follower]" leader-address= leader-id= -2025-10-27T04:03:25.821Z [WARN] server-f892f9eb-43c7-836f-62f2-750af82387a0: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:25.821Z [INFO] server-f892f9eb-43c7-836f-62f2-750af82387a0: entering candidate state: node="Node at f892f9eb-43c7-836f-62f2-750af82387a0 [Candidate]" term=2 -2025-10-27T04:03:25.821Z [INFO] server-f892f9eb-43c7-836f-62f2-750af82387a0: election won: term=2 tally=2 -2025-10-27T04:03:25.821Z [INFO] server-f892f9eb-43c7-836f-62f2-750af82387a0: entering leader state: leader="Node at f892f9eb-43c7-836f-62f2-750af82387a0 [Leader]" -2025-10-27T04:03:25.821Z [INFO] server-f892f9eb-43c7-836f-62f2-750af82387a0: added peer, starting replication: peer=server-a3bf5301-9f81-5083-9f79-14bf60db99d4 -2025-10-27T04:03:25.821Z [INFO] server-f892f9eb-43c7-836f-62f2-750af82387a0: added peer, starting replication: peer=server-5d975827-46b8-d55f-6d05-79ee3deceb6d -2025-10-27T04:03:25.821Z [INFO] server-f892f9eb-43c7-836f-62f2-750af82387a0: pipelining replication: peer="{Voter server-a3bf5301-9f81-5083-9f79-14bf60db99d4 a3bf5301-9f81-5083-9f79-14bf60db99d4}" -2025-10-27T04:03:25.821Z [INFO] server-f892f9eb-43c7-836f-62f2-750af82387a0: pipelining replication: peer="{Voter server-5d975827-46b8-d55f-6d05-79ee3deceb6d 5d975827-46b8-d55f-6d05-79ee3deceb6d}" -2025-10-27T04:03:26.835Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:25.821416571 +0000 UTC m=+80.816289089 (1 nodes), 519.003466ms from start of poll, 519.262628ms from cluster start. Timeout at 2025-10-27 04:03:26.826419051 +0000 UTC m=+81.821291489, 1.0050024s after stability -2025-10-27T04:03:26.836Z [INFO] server-f892f9eb-43c7-836f-62f2-750af82387a0: starting snapshot up to: index=12 -2025-10-27T04:03:26.836Z [INFO] creating new snapshot: path=/tmp/raft1874666782/snapshots/2-12-1761537806836.tmp -2025-10-27T04:03:26.836Z [INFO] server-f892f9eb-43c7-836f-62f2-750af82387a0: snapshot complete up to: index=12 -2025-10-27T04:03:26.837Z [INFO] server-fd5665b8-ec7c-c32f-6f64-5dbc79b7b455: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fd5665b8-ec7c-c32f-6f64-5dbc79b7b455 Address:fd5665b8-ec7c-c32f-6f64-5dbc79b7b455} {Suffrage:Voter ID:server-6ec471c2-0c6b-418e-c296-183d95c4541c Address:6ec471c2-0c6b-418e-c296-183d95c4541c} {Suffrage:Voter ID:server-61e99b97-a6da-c9c8-0749-7c57e604f2ab Address:61e99b97-a6da-c9c8-0749-7c57e604f2ab}]" -2025-10-27T04:03:26.837Z [INFO] server-6ec471c2-0c6b-418e-c296-183d95c4541c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fd5665b8-ec7c-c32f-6f64-5dbc79b7b455 Address:fd5665b8-ec7c-c32f-6f64-5dbc79b7b455} {Suffrage:Voter ID:server-6ec471c2-0c6b-418e-c296-183d95c4541c Address:6ec471c2-0c6b-418e-c296-183d95c4541c} {Suffrage:Voter ID:server-61e99b97-a6da-c9c8-0749-7c57e604f2ab Address:61e99b97-a6da-c9c8-0749-7c57e604f2ab}]" -2025-10-27T04:03:26.837Z [INFO] server-61e99b97-a6da-c9c8-0749-7c57e604f2ab: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fd5665b8-ec7c-c32f-6f64-5dbc79b7b455 Address:fd5665b8-ec7c-c32f-6f64-5dbc79b7b455} {Suffrage:Voter ID:server-6ec471c2-0c6b-418e-c296-183d95c4541c Address:6ec471c2-0c6b-418e-c296-183d95c4541c} {Suffrage:Voter ID:server-61e99b97-a6da-c9c8-0749-7c57e604f2ab Address:61e99b97-a6da-c9c8-0749-7c57e604f2ab}]" -2025-10-27T04:03:26.837Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:26.837Z [INFO] server-fd5665b8-ec7c-c32f-6f64-5dbc79b7b455: entering follower state: follower="Node at fd5665b8-ec7c-c32f-6f64-5dbc79b7b455 [Follower]" leader-address= leader-id= -2025-10-27T04:03:26.837Z [INFO] server-6ec471c2-0c6b-418e-c296-183d95c4541c: entering follower state: follower="Node at 6ec471c2-0c6b-418e-c296-183d95c4541c [Follower]" leader-address= leader-id= -2025-10-27T04:03:26.837Z [INFO] server-61e99b97-a6da-c9c8-0749-7c57e604f2ab: entering follower state: follower="Node at 61e99b97-a6da-c9c8-0749-7c57e604f2ab [Follower]" leader-address= leader-id= -2025-10-27T04:03:27.666Z [WARN] server-6ec471c2-0c6b-418e-c296-183d95c4541c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:27.666Z [INFO] server-6ec471c2-0c6b-418e-c296-183d95c4541c: entering candidate state: node="Node at 6ec471c2-0c6b-418e-c296-183d95c4541c [Candidate]" term=2 -2025-10-27T04:03:27.667Z [INFO] server-6ec471c2-0c6b-418e-c296-183d95c4541c: election won: term=2 tally=2 -2025-10-27T04:03:27.667Z [INFO] server-6ec471c2-0c6b-418e-c296-183d95c4541c: entering leader state: leader="Node at 6ec471c2-0c6b-418e-c296-183d95c4541c [Leader]" -2025-10-27T04:03:27.667Z [INFO] server-6ec471c2-0c6b-418e-c296-183d95c4541c: added peer, starting replication: peer=server-fd5665b8-ec7c-c32f-6f64-5dbc79b7b455 -2025-10-27T04:03:27.667Z [INFO] server-6ec471c2-0c6b-418e-c296-183d95c4541c: added peer, starting replication: peer=server-61e99b97-a6da-c9c8-0749-7c57e604f2ab -2025-10-27T04:03:27.667Z [INFO] server-6ec471c2-0c6b-418e-c296-183d95c4541c: pipelining replication: peer="{Voter server-fd5665b8-ec7c-c32f-6f64-5dbc79b7b455 fd5665b8-ec7c-c32f-6f64-5dbc79b7b455}" -2025-10-27T04:03:27.667Z [INFO] server-6ec471c2-0c6b-418e-c296-183d95c4541c: pipelining replication: peer="{Voter server-61e99b97-a6da-c9c8-0749-7c57e604f2ab 61e99b97-a6da-c9c8-0749-7c57e604f2ab}" -2025-10-27T04:03:28.672Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:27.667033381 +0000 UTC m=+82.661905819 (1 nodes), 829.687709ms from start of poll, 829.88575ms from cluster start. Timeout at 2025-10-27 04:03:28.672038102 +0000 UTC m=+83.666910580, 1.005004761s after stability -2025-10-27T04:03:28.673Z [INFO] creating new snapshot: path=/tmp/raft422104482/snapshots/2-1013-1761537808673.tmp -2025-10-27T04:03:28.674Z [INFO] server-6ec471c2-0c6b-418e-c296-183d95c4541c: copied to local snapshot: bytes=71 -2025-10-27T04:03:28.675Z [INFO] server-6ec471c2-0c6b-418e-c296-183d95c4541c: snapshot restore progress: id=2-1013-1761537808673 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:28.675Z [INFO] server-6ec471c2-0c6b-418e-c296-183d95c4541c: removing all old logs from log store -2025-10-27T04:03:28.675Z [INFO] server-6ec471c2-0c6b-418e-c296-183d95c4541c: compacting logs: from=1 to=2 -2025-10-27T04:03:28.675Z [INFO] server-6ec471c2-0c6b-418e-c296-183d95c4541c: restored user snapshot: index=1013 -2025-10-27T04:03:28.675Z [ERROR] server-6ec471c2-0c6b-418e-c296-183d95c4541c: failed to get log: index=2 error="log not found" -2025-10-27T04:03:28.675Z [INFO] server-6ec471c2-0c6b-418e-c296-183d95c4541c: aborting pipeline replication: peer="{Voter server-61e99b97-a6da-c9c8-0749-7c57e604f2ab 61e99b97-a6da-c9c8-0749-7c57e604f2ab}" -2025-10-27T04:03:28.675Z [ERROR] server-6ec471c2-0c6b-418e-c296-183d95c4541c: failed to get log: index=2 error="log not found" -2025-10-27T04:03:28.675Z [INFO] server-6ec471c2-0c6b-418e-c296-183d95c4541c: aborting pipeline replication: peer="{Voter server-fd5665b8-ec7c-c32f-6f64-5dbc79b7b455 fd5665b8-ec7c-c32f-6f64-5dbc79b7b455}" -2025-10-27T04:03:28.680Z [ERROR] server-6ec471c2-0c6b-418e-c296-183d95c4541c: failed to get log: index=2 error="log not found" -2025-10-27T04:03:28.680Z [INFO] creating new snapshot: path=/tmp/raft1810173711/snapshots/2-1013-1761537808680.tmp -2025-10-27T04:03:28.681Z [INFO] server-61e99b97-a6da-c9c8-0749-7c57e604f2ab: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:28.681Z [INFO] server-61e99b97-a6da-c9c8-0749-7c57e604f2ab: copied to local snapshot: bytes=71 -2025-10-27T04:03:28.681Z [INFO] server-61e99b97-a6da-c9c8-0749-7c57e604f2ab: snapshot restore progress: id=2-1013-1761537808680 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:28.681Z [INFO] server-61e99b97-a6da-c9c8-0749-7c57e604f2ab: removing all old logs from log store -2025-10-27T04:03:28.681Z [INFO] server-61e99b97-a6da-c9c8-0749-7c57e604f2ab: compacting logs: from=1 to=2 -2025-10-27T04:03:28.681Z [INFO] server-61e99b97-a6da-c9c8-0749-7c57e604f2ab: Installed remote snapshot -2025-10-27T04:03:28.681Z [INFO] server-6ec471c2-0c6b-418e-c296-183d95c4541c: pipelining replication: peer="{Voter server-61e99b97-a6da-c9c8-0749-7c57e604f2ab 61e99b97-a6da-c9c8-0749-7c57e604f2ab}" -2025-10-27T04:03:28.681Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:03:28.684Z [ERROR] server-6ec471c2-0c6b-418e-c296-183d95c4541c: failed to get log: index=2 error="log not found" -2025-10-27T04:03:28.684Z [INFO] creating new snapshot: path=/tmp/raft3686653422/snapshots/2-1013-1761537808684.tmp -2025-10-27T04:03:28.684Z [INFO] server-fd5665b8-ec7c-c32f-6f64-5dbc79b7b455: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:28.684Z [INFO] server-fd5665b8-ec7c-c32f-6f64-5dbc79b7b455: copied to local snapshot: bytes=71 -2025-10-27T04:03:28.685Z [INFO] server-fd5665b8-ec7c-c32f-6f64-5dbc79b7b455: snapshot restore progress: id=2-1013-1761537808684 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:28.685Z [INFO] server-fd5665b8-ec7c-c32f-6f64-5dbc79b7b455: removing all old logs from log store -2025-10-27T04:03:28.685Z [INFO] server-fd5665b8-ec7c-c32f-6f64-5dbc79b7b455: compacting logs: from=1 to=2 -2025-10-27T04:03:28.685Z [INFO] server-fd5665b8-ec7c-c32f-6f64-5dbc79b7b455: Installed remote snapshot -2025-10-27T04:03:28.685Z [INFO] server-6ec471c2-0c6b-418e-c296-183d95c4541c: pipelining replication: peer="{Voter server-fd5665b8-ec7c-c32f-6f64-5dbc79b7b455 fd5665b8-ec7c-c32f-6f64-5dbc79b7b455}" -2025-10-27T04:03:29.686Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:03:28.681552273 +0000 UTC m=+83.676424711 (2 nodes), 600ns from start of poll, 1.844404642s from cluster start. Timeout at 2025-10-27 04:03:29.686565193 +0000 UTC m=+84.681437631, 1.00501292s after stability -2025-10-27T04:03:29.697Z [INFO] server-f892f9eb-43c7-836f-62f2-750af82387a0: aborting pipeline replication: peer="{Voter server-5d975827-46b8-d55f-6d05-79ee3deceb6d 5d975827-46b8-d55f-6d05-79ee3deceb6d}" -2025-10-27T04:03:29.698Z [INFO] server-f892f9eb-43c7-836f-62f2-750af82387a0: aborting pipeline replication: peer="{Voter server-a3bf5301-9f81-5083-9f79-14bf60db99d4 a3bf5301-9f81-5083-9f79-14bf60db99d4}" +2026-11-29T10:29:27.458Z [INFO] server-b903c005-3501-9fb6-c2e8-7921fd802ed3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b903c005-3501-9fb6-c2e8-7921fd802ed3 Address:b903c005-3501-9fb6-c2e8-7921fd802ed3} {Suffrage:Voter ID:server-4ea85434-17d3-d4ec-bc88-26f376953ca3 Address:4ea85434-17d3-d4ec-bc88-26f376953ca3} {Suffrage:Voter ID:server-7c8c9983-0e64-18ec-04d3-305342a3ecd7 Address:7c8c9983-0e64-18ec-04d3-305342a3ecd7}]" +2026-11-29T10:29:27.459Z [INFO] server-4ea85434-17d3-d4ec-bc88-26f376953ca3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b903c005-3501-9fb6-c2e8-7921fd802ed3 Address:b903c005-3501-9fb6-c2e8-7921fd802ed3} {Suffrage:Voter ID:server-4ea85434-17d3-d4ec-bc88-26f376953ca3 Address:4ea85434-17d3-d4ec-bc88-26f376953ca3} {Suffrage:Voter ID:server-7c8c9983-0e64-18ec-04d3-305342a3ecd7 Address:7c8c9983-0e64-18ec-04d3-305342a3ecd7}]" +2026-11-29T10:29:27.459Z [INFO] server-4ea85434-17d3-d4ec-bc88-26f376953ca3: entering follower state: follower="Node at 4ea85434-17d3-d4ec-bc88-26f376953ca3 [Follower]" leader-address= leader-id= +2026-11-29T10:29:27.459Z [INFO] server-7c8c9983-0e64-18ec-04d3-305342a3ecd7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b903c005-3501-9fb6-c2e8-7921fd802ed3 Address:b903c005-3501-9fb6-c2e8-7921fd802ed3} {Suffrage:Voter ID:server-4ea85434-17d3-d4ec-bc88-26f376953ca3 Address:4ea85434-17d3-d4ec-bc88-26f376953ca3} {Suffrage:Voter ID:server-7c8c9983-0e64-18ec-04d3-305342a3ecd7 Address:7c8c9983-0e64-18ec-04d3-305342a3ecd7}]" +2026-11-29T10:29:27.459Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:27.459Z [INFO] server-b903c005-3501-9fb6-c2e8-7921fd802ed3: entering follower state: follower="Node at b903c005-3501-9fb6-c2e8-7921fd802ed3 [Follower]" leader-address= leader-id= +2026-11-29T10:29:27.459Z [INFO] server-7c8c9983-0e64-18ec-04d3-305342a3ecd7: entering follower state: follower="Node at 7c8c9983-0e64-18ec-04d3-305342a3ecd7 [Follower]" leader-address= leader-id= +2026-11-29T10:29:28.113Z [WARN] server-b903c005-3501-9fb6-c2e8-7921fd802ed3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:28.113Z [INFO] server-b903c005-3501-9fb6-c2e8-7921fd802ed3: entering candidate state: node="Node at b903c005-3501-9fb6-c2e8-7921fd802ed3 [Candidate]" term=2 +2026-11-29T10:29:28.113Z [INFO] server-b903c005-3501-9fb6-c2e8-7921fd802ed3: election won: term=2 tally=2 +2026-11-29T10:29:28.113Z [INFO] server-b903c005-3501-9fb6-c2e8-7921fd802ed3: entering leader state: leader="Node at b903c005-3501-9fb6-c2e8-7921fd802ed3 [Leader]" +2026-11-29T10:29:28.113Z [INFO] server-b903c005-3501-9fb6-c2e8-7921fd802ed3: added peer, starting replication: peer=server-4ea85434-17d3-d4ec-bc88-26f376953ca3 +2026-11-29T10:29:28.113Z [INFO] server-b903c005-3501-9fb6-c2e8-7921fd802ed3: added peer, starting replication: peer=server-7c8c9983-0e64-18ec-04d3-305342a3ecd7 +2026-11-29T10:29:28.113Z [INFO] server-b903c005-3501-9fb6-c2e8-7921fd802ed3: pipelining replication: peer="{Voter server-7c8c9983-0e64-18ec-04d3-305342a3ecd7 7c8c9983-0e64-18ec-04d3-305342a3ecd7}" +2026-11-29T10:29:28.113Z [INFO] server-b903c005-3501-9fb6-c2e8-7921fd802ed3: pipelining replication: peer="{Voter server-4ea85434-17d3-d4ec-bc88-26f376953ca3 4ea85434-17d3-d4ec-bc88-26f376953ca3}" +2026-11-29T10:29:29.118Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:28.113462139 +0000 UTC m=+78.848516201 (1 nodes), 654.373ms from start of poll, 654.58548ms from cluster start. Timeout at 2026-11-29 10:29:29.118464699 +0000 UTC m=+79.853518801, 1.0050026s after stability +2026-11-29T10:29:29.119Z [INFO] server-b903c005-3501-9fb6-c2e8-7921fd802ed3: starting snapshot up to: index=12 +2026-11-29T10:29:29.119Z [INFO] creating new snapshot: path=/tmp/raft1028005788/snapshots/2-12-1795948169119.tmp +2026-11-29T10:29:29.119Z [INFO] server-b903c005-3501-9fb6-c2e8-7921fd802ed3: snapshot complete up to: index=12 +2026-11-29T10:29:29.120Z [INFO] server-a409bcf7-c4a1-1ed7-062b-8a9c59839207: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a409bcf7-c4a1-1ed7-062b-8a9c59839207 Address:a409bcf7-c4a1-1ed7-062b-8a9c59839207} {Suffrage:Voter ID:server-761c9bb1-62c0-0896-9696-2ec8a5b1146e Address:761c9bb1-62c0-0896-9696-2ec8a5b1146e} {Suffrage:Voter ID:server-b423b9b9-cac5-c126-6fdd-bdef53a58962 Address:b423b9b9-cac5-c126-6fdd-bdef53a58962}]" +2026-11-29T10:29:29.120Z [INFO] server-a409bcf7-c4a1-1ed7-062b-8a9c59839207: entering follower state: follower="Node at a409bcf7-c4a1-1ed7-062b-8a9c59839207 [Follower]" leader-address= leader-id= +2026-11-29T10:29:29.120Z [INFO] server-761c9bb1-62c0-0896-9696-2ec8a5b1146e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a409bcf7-c4a1-1ed7-062b-8a9c59839207 Address:a409bcf7-c4a1-1ed7-062b-8a9c59839207} {Suffrage:Voter ID:server-761c9bb1-62c0-0896-9696-2ec8a5b1146e Address:761c9bb1-62c0-0896-9696-2ec8a5b1146e} {Suffrage:Voter ID:server-b423b9b9-cac5-c126-6fdd-bdef53a58962 Address:b423b9b9-cac5-c126-6fdd-bdef53a58962}]" +2026-11-29T10:29:29.120Z [INFO] server-761c9bb1-62c0-0896-9696-2ec8a5b1146e: entering follower state: follower="Node at 761c9bb1-62c0-0896-9696-2ec8a5b1146e [Follower]" leader-address= leader-id= +2026-11-29T10:29:29.120Z [INFO] server-b423b9b9-cac5-c126-6fdd-bdef53a58962: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a409bcf7-c4a1-1ed7-062b-8a9c59839207 Address:a409bcf7-c4a1-1ed7-062b-8a9c59839207} {Suffrage:Voter ID:server-761c9bb1-62c0-0896-9696-2ec8a5b1146e Address:761c9bb1-62c0-0896-9696-2ec8a5b1146e} {Suffrage:Voter ID:server-b423b9b9-cac5-c126-6fdd-bdef53a58962 Address:b423b9b9-cac5-c126-6fdd-bdef53a58962}]" +2026-11-29T10:29:29.120Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:29.121Z [INFO] server-b423b9b9-cac5-c126-6fdd-bdef53a58962: entering follower state: follower="Node at b423b9b9-cac5-c126-6fdd-bdef53a58962 [Follower]" leader-address= leader-id= +2026-11-29T10:29:29.816Z [WARN] server-b423b9b9-cac5-c126-6fdd-bdef53a58962: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:29.816Z [INFO] server-b423b9b9-cac5-c126-6fdd-bdef53a58962: entering candidate state: node="Node at b423b9b9-cac5-c126-6fdd-bdef53a58962 [Candidate]" term=2 +2026-11-29T10:29:29.816Z [INFO] server-b423b9b9-cac5-c126-6fdd-bdef53a58962: election won: term=2 tally=2 +2026-11-29T10:29:29.816Z [INFO] server-b423b9b9-cac5-c126-6fdd-bdef53a58962: entering leader state: leader="Node at b423b9b9-cac5-c126-6fdd-bdef53a58962 [Leader]" +2026-11-29T10:29:29.816Z [INFO] server-b423b9b9-cac5-c126-6fdd-bdef53a58962: added peer, starting replication: peer=server-a409bcf7-c4a1-1ed7-062b-8a9c59839207 +2026-11-29T10:29:29.816Z [INFO] server-b423b9b9-cac5-c126-6fdd-bdef53a58962: added peer, starting replication: peer=server-761c9bb1-62c0-0896-9696-2ec8a5b1146e +2026-11-29T10:29:29.817Z [INFO] server-b423b9b9-cac5-c126-6fdd-bdef53a58962: pipelining replication: peer="{Voter server-761c9bb1-62c0-0896-9696-2ec8a5b1146e 761c9bb1-62c0-0896-9696-2ec8a5b1146e}" +2026-11-29T10:29:29.817Z [INFO] server-b423b9b9-cac5-c126-6fdd-bdef53a58962: pipelining replication: peer="{Voter server-a409bcf7-c4a1-1ed7-062b-8a9c59839207 a409bcf7-c4a1-1ed7-062b-8a9c59839207}" +2026-11-29T10:29:30.822Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:29.816836819 +0000 UTC m=+80.551891001 (1 nodes), 695.8498ms from start of poll, 696.2836ms from cluster start. Timeout at 2026-11-29 10:29:30.821838139 +0000 UTC m=+81.556892241, 1.00500124s after stability +2026-11-29T10:29:30.822Z [INFO] creating new snapshot: path=/tmp/raft1931081588/snapshots/2-1013-1795948170822.tmp +2026-11-29T10:29:30.823Z [INFO] server-b423b9b9-cac5-c126-6fdd-bdef53a58962: copied to local snapshot: bytes=71 +2026-11-29T10:29:30.823Z [INFO] server-b423b9b9-cac5-c126-6fdd-bdef53a58962: snapshot restore progress: id=2-1013-1795948170822 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:30.823Z [INFO] server-b423b9b9-cac5-c126-6fdd-bdef53a58962: removing all old logs from log store +2026-11-29T10:29:30.823Z [INFO] server-b423b9b9-cac5-c126-6fdd-bdef53a58962: compacting logs: from=1 to=2 +2026-11-29T10:29:30.823Z [INFO] server-b423b9b9-cac5-c126-6fdd-bdef53a58962: restored user snapshot: index=1013 +2026-11-29T10:29:30.823Z [ERROR] server-b423b9b9-cac5-c126-6fdd-bdef53a58962: failed to get log: index=2 error="log not found" +2026-11-29T10:29:30.823Z [INFO] server-b423b9b9-cac5-c126-6fdd-bdef53a58962: aborting pipeline replication: peer="{Voter server-761c9bb1-62c0-0896-9696-2ec8a5b1146e 761c9bb1-62c0-0896-9696-2ec8a5b1146e}" +2026-11-29T10:29:30.823Z [ERROR] server-b423b9b9-cac5-c126-6fdd-bdef53a58962: failed to get log: index=2 error="log not found" +2026-11-29T10:29:30.823Z [INFO] server-b423b9b9-cac5-c126-6fdd-bdef53a58962: aborting pipeline replication: peer="{Voter server-a409bcf7-c4a1-1ed7-062b-8a9c59839207 a409bcf7-c4a1-1ed7-062b-8a9c59839207}" +2026-11-29T10:29:30.830Z [ERROR] server-b423b9b9-cac5-c126-6fdd-bdef53a58962: failed to get log: index=2 error="log not found" +2026-11-29T10:29:30.831Z [INFO] creating new snapshot: path=/tmp/raft1508521460/snapshots/2-1013-1795948170831.tmp +2026-11-29T10:29:30.831Z [INFO] server-761c9bb1-62c0-0896-9696-2ec8a5b1146e: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:30.831Z [INFO] server-761c9bb1-62c0-0896-9696-2ec8a5b1146e: copied to local snapshot: bytes=71 +2026-11-29T10:29:30.831Z [INFO] server-761c9bb1-62c0-0896-9696-2ec8a5b1146e: snapshot restore progress: id=2-1013-1795948170831 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:30.831Z [INFO] server-761c9bb1-62c0-0896-9696-2ec8a5b1146e: removing all old logs from log store +2026-11-29T10:29:30.831Z [INFO] server-761c9bb1-62c0-0896-9696-2ec8a5b1146e: compacting logs: from=1 to=2 +2026-11-29T10:29:30.831Z [INFO] server-761c9bb1-62c0-0896-9696-2ec8a5b1146e: Installed remote snapshot +2026-11-29T10:29:30.831Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:30.831Z [INFO] server-b423b9b9-cac5-c126-6fdd-bdef53a58962: pipelining replication: peer="{Voter server-761c9bb1-62c0-0896-9696-2ec8a5b1146e 761c9bb1-62c0-0896-9696-2ec8a5b1146e}" +2026-11-29T10:29:30.834Z [ERROR] server-b423b9b9-cac5-c126-6fdd-bdef53a58962: failed to get log: index=2 error="log not found" +2026-11-29T10:29:30.834Z [INFO] creating new snapshot: path=/tmp/raft4216311710/snapshots/2-1013-1795948170834.tmp +2026-11-29T10:29:30.834Z [INFO] server-a409bcf7-c4a1-1ed7-062b-8a9c59839207: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:30.835Z [INFO] server-a409bcf7-c4a1-1ed7-062b-8a9c59839207: copied to local snapshot: bytes=71 +2026-11-29T10:29:30.835Z [INFO] server-a409bcf7-c4a1-1ed7-062b-8a9c59839207: snapshot restore progress: id=2-1013-1795948170834 last-index=1013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:30.835Z [INFO] server-a409bcf7-c4a1-1ed7-062b-8a9c59839207: removing all old logs from log store +2026-11-29T10:29:30.835Z [INFO] server-a409bcf7-c4a1-1ed7-062b-8a9c59839207: compacting logs: from=1 to=2 +2026-11-29T10:29:30.835Z [INFO] server-a409bcf7-c4a1-1ed7-062b-8a9c59839207: Installed remote snapshot +2026-11-29T10:29:30.835Z [INFO] server-b423b9b9-cac5-c126-6fdd-bdef53a58962: pipelining replication: peer="{Voter server-a409bcf7-c4a1-1ed7-062b-8a9c59839207 a409bcf7-c4a1-1ed7-062b-8a9c59839207}" +2026-11-29T10:29:31.837Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:30.831896579 +0000 UTC m=+81.566950681 (2 nodes), 960ns from start of poll, 1.71134328s from cluster start. Timeout at 2026-11-29 10:29:31.836922539 +0000 UTC m=+82.571976601, 1.00502592s after stability +2026-11-29T10:29:31.850Z [INFO] server-b903c005-3501-9fb6-c2e8-7921fd802ed3: aborting pipeline replication: peer="{Voter server-7c8c9983-0e64-18ec-04d3-305342a3ecd7 7c8c9983-0e64-18ec-04d3-305342a3ecd7}" +2026-11-29T10:29:31.850Z [INFO] server-b903c005-3501-9fb6-c2e8-7921fd802ed3: aborting pipeline replication: peer="{Voter server-4ea85434-17d3-d4ec-bc88-26f376953ca3 4ea85434-17d3-d4ec-bc88-26f376953ca3}" === RUN TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_false -2025-10-27T04:03:29.698Z [INFO] server-a6605762-a073-6de8-5e5f-792f2974edcc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a6605762-a073-6de8-5e5f-792f2974edcc Address:a6605762-a073-6de8-5e5f-792f2974edcc} {Suffrage:Voter ID:server-49b776ff-0f68-11ad-d244-fb807c079668 Address:49b776ff-0f68-11ad-d244-fb807c079668} {Suffrage:Voter ID:server-739235d5-a50a-280d-4fe2-ad2a22bbc109 Address:739235d5-a50a-280d-4fe2-ad2a22bbc109}]" -2025-10-27T04:03:29.699Z [INFO] server-49b776ff-0f68-11ad-d244-fb807c079668: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a6605762-a073-6de8-5e5f-792f2974edcc Address:a6605762-a073-6de8-5e5f-792f2974edcc} {Suffrage:Voter ID:server-49b776ff-0f68-11ad-d244-fb807c079668 Address:49b776ff-0f68-11ad-d244-fb807c079668} {Suffrage:Voter ID:server-739235d5-a50a-280d-4fe2-ad2a22bbc109 Address:739235d5-a50a-280d-4fe2-ad2a22bbc109}]" -2025-10-27T04:03:29.699Z [INFO] server-739235d5-a50a-280d-4fe2-ad2a22bbc109: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a6605762-a073-6de8-5e5f-792f2974edcc Address:a6605762-a073-6de8-5e5f-792f2974edcc} {Suffrage:Voter ID:server-49b776ff-0f68-11ad-d244-fb807c079668 Address:49b776ff-0f68-11ad-d244-fb807c079668} {Suffrage:Voter ID:server-739235d5-a50a-280d-4fe2-ad2a22bbc109 Address:739235d5-a50a-280d-4fe2-ad2a22bbc109}]" -2025-10-27T04:03:29.699Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:29.699Z [INFO] server-a6605762-a073-6de8-5e5f-792f2974edcc: entering follower state: follower="Node at a6605762-a073-6de8-5e5f-792f2974edcc [Follower]" leader-address= leader-id= -2025-10-27T04:03:29.699Z [INFO] server-49b776ff-0f68-11ad-d244-fb807c079668: entering follower state: follower="Node at 49b776ff-0f68-11ad-d244-fb807c079668 [Follower]" leader-address= leader-id= -2025-10-27T04:03:29.699Z [INFO] server-739235d5-a50a-280d-4fe2-ad2a22bbc109: entering follower state: follower="Node at 739235d5-a50a-280d-4fe2-ad2a22bbc109 [Follower]" leader-address= leader-id= -2025-10-27T04:03:30.210Z [WARN] server-49b776ff-0f68-11ad-d244-fb807c079668: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:30.210Z [INFO] server-49b776ff-0f68-11ad-d244-fb807c079668: entering candidate state: node="Node at 49b776ff-0f68-11ad-d244-fb807c079668 [Candidate]" term=2 -2025-10-27T04:03:30.211Z [INFO] server-49b776ff-0f68-11ad-d244-fb807c079668: election won: term=2 tally=2 -2025-10-27T04:03:30.211Z [INFO] server-49b776ff-0f68-11ad-d244-fb807c079668: entering leader state: leader="Node at 49b776ff-0f68-11ad-d244-fb807c079668 [Leader]" -2025-10-27T04:03:30.211Z [INFO] server-49b776ff-0f68-11ad-d244-fb807c079668: added peer, starting replication: peer=server-a6605762-a073-6de8-5e5f-792f2974edcc -2025-10-27T04:03:30.211Z [INFO] server-49b776ff-0f68-11ad-d244-fb807c079668: added peer, starting replication: peer=server-739235d5-a50a-280d-4fe2-ad2a22bbc109 -2025-10-27T04:03:30.211Z [INFO] server-49b776ff-0f68-11ad-d244-fb807c079668: pipelining replication: peer="{Voter server-a6605762-a073-6de8-5e5f-792f2974edcc a6605762-a073-6de8-5e5f-792f2974edcc}" -2025-10-27T04:03:30.211Z [INFO] server-49b776ff-0f68-11ad-d244-fb807c079668: pipelining replication: peer="{Voter server-739235d5-a50a-280d-4fe2-ad2a22bbc109 739235d5-a50a-280d-4fe2-ad2a22bbc109}" -2025-10-27T04:03:31.216Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:30.211045243 +0000 UTC m=+85.205917681 (1 nodes), 511.900418ms from start of poll, 512.288821ms from cluster start. Timeout at 2025-10-27 04:03:31.216056003 +0000 UTC m=+86.210928521, 1.00501084s after stability -2025-10-27T04:03:31.217Z [INFO] server-49b776ff-0f68-11ad-d244-fb807c079668: starting snapshot up to: index=12 -2025-10-27T04:03:31.217Z [INFO] creating new snapshot: path=/tmp/raft1742012095/snapshots/2-12-1761537811217.tmp -2025-10-27T04:03:31.217Z [INFO] server-49b776ff-0f68-11ad-d244-fb807c079668: snapshot complete up to: index=12 -2025-10-27T04:03:31.218Z [INFO] creating new snapshot: path=/tmp/raft1742012095/snapshots/2-10013-1761537811218.tmp -2025-10-27T04:03:31.218Z [INFO] server-49b776ff-0f68-11ad-d244-fb807c079668: copied to local snapshot: bytes=71 -2025-10-27T04:03:31.219Z [INFO] server-49b776ff-0f68-11ad-d244-fb807c079668: snapshot restore progress: id=2-10013-1761537811218 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:31.219Z [INFO] server-49b776ff-0f68-11ad-d244-fb807c079668: restored user snapshot: index=10013 -2025-10-27T04:03:31.219Z [ERROR] server-49b776ff-0f68-11ad-d244-fb807c079668: failed to get log: index=23 error="log not found" -2025-10-27T04:03:31.219Z [INFO] server-49b776ff-0f68-11ad-d244-fb807c079668: aborting pipeline replication: peer="{Voter server-a6605762-a073-6de8-5e5f-792f2974edcc a6605762-a073-6de8-5e5f-792f2974edcc}" -2025-10-27T04:03:31.219Z [ERROR] server-49b776ff-0f68-11ad-d244-fb807c079668: failed to get log: index=23 error="log not found" -2025-10-27T04:03:31.219Z [INFO] server-49b776ff-0f68-11ad-d244-fb807c079668: aborting pipeline replication: peer="{Voter server-739235d5-a50a-280d-4fe2-ad2a22bbc109 739235d5-a50a-280d-4fe2-ad2a22bbc109}" -2025-10-27T04:03:31.227Z [ERROR] server-49b776ff-0f68-11ad-d244-fb807c079668: failed to get log: index=23 error="log not found" -2025-10-27T04:03:31.227Z [INFO] creating new snapshot: path=/tmp/raft3578200554/snapshots/2-10013-1761537811227.tmp -2025-10-27T04:03:31.228Z [INFO] server-739235d5-a50a-280d-4fe2-ad2a22bbc109: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:31.228Z [INFO] server-739235d5-a50a-280d-4fe2-ad2a22bbc109: copied to local snapshot: bytes=71 -2025-10-27T04:03:31.228Z [ERROR] server-49b776ff-0f68-11ad-d244-fb807c079668: failed to get log: index=23 error="log not found" -2025-10-27T04:03:31.228Z [INFO] server-739235d5-a50a-280d-4fe2-ad2a22bbc109: snapshot restore progress: id=2-10013-1761537811227 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:31.228Z [INFO] server-739235d5-a50a-280d-4fe2-ad2a22bbc109: Installed remote snapshot -2025-10-27T04:03:31.228Z [WARN] server-739235d5-a50a-280d-4fe2-ad2a22bbc109: skipping application of old log: index=22 -2025-10-27T04:03:31.228Z [INFO] server-49b776ff-0f68-11ad-d244-fb807c079668: pipelining replication: peer="{Voter server-739235d5-a50a-280d-4fe2-ad2a22bbc109 739235d5-a50a-280d-4fe2-ad2a22bbc109}" -2025-10-27T04:03:31.228Z [INFO] creating new snapshot: path=/tmp/raft1061050591/snapshots/2-10013-1761537811228.tmp -2025-10-27T04:03:31.228Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:03:31.228Z [INFO] server-a6605762-a073-6de8-5e5f-792f2974edcc: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:31.229Z [INFO] server-a6605762-a073-6de8-5e5f-792f2974edcc: copied to local snapshot: bytes=71 -2025-10-27T04:03:31.229Z [INFO] server-a6605762-a073-6de8-5e5f-792f2974edcc: snapshot restore progress: id=2-10013-1761537811228 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:31.229Z [INFO] server-a6605762-a073-6de8-5e5f-792f2974edcc: Installed remote snapshot -2025-10-27T04:03:31.229Z [INFO] server-49b776ff-0f68-11ad-d244-fb807c079668: pipelining replication: peer="{Voter server-a6605762-a073-6de8-5e5f-792f2974edcc a6605762-a073-6de8-5e5f-792f2974edcc}" -2025-10-27T04:03:32.233Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:03:31.228687715 +0000 UTC m=+86.223560193 (2 nodes), 880ns from start of poll, 1.529931333s from cluster start. Timeout at 2025-10-27 04:03:32.233709395 +0000 UTC m=+87.228581873, 1.00502168s after stability -2025-10-27T04:03:32.246Z [INFO] server-49b776ff-0f68-11ad-d244-fb807c079668: aborting pipeline replication: peer="{Voter server-739235d5-a50a-280d-4fe2-ad2a22bbc109 739235d5-a50a-280d-4fe2-ad2a22bbc109}" -2025-10-27T04:03:32.246Z [INFO] server-49b776ff-0f68-11ad-d244-fb807c079668: aborting pipeline replication: peer="{Voter server-a6605762-a073-6de8-5e5f-792f2974edcc a6605762-a073-6de8-5e5f-792f2974edcc}" +2026-11-29T10:29:31.851Z [INFO] server-83f8a567-8196-a1f0-0254-117f05475ea0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-83f8a567-8196-a1f0-0254-117f05475ea0 Address:83f8a567-8196-a1f0-0254-117f05475ea0} {Suffrage:Voter ID:server-83d2b0aa-af06-6d71-6ad2-0c0579882f86 Address:83d2b0aa-af06-6d71-6ad2-0c0579882f86} {Suffrage:Voter ID:server-59011c36-423d-e122-5895-cd2e48af1185 Address:59011c36-423d-e122-5895-cd2e48af1185}]" +2026-11-29T10:29:31.851Z [INFO] server-83d2b0aa-af06-6d71-6ad2-0c0579882f86: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-83f8a567-8196-a1f0-0254-117f05475ea0 Address:83f8a567-8196-a1f0-0254-117f05475ea0} {Suffrage:Voter ID:server-83d2b0aa-af06-6d71-6ad2-0c0579882f86 Address:83d2b0aa-af06-6d71-6ad2-0c0579882f86} {Suffrage:Voter ID:server-59011c36-423d-e122-5895-cd2e48af1185 Address:59011c36-423d-e122-5895-cd2e48af1185}]" +2026-11-29T10:29:31.851Z [INFO] server-83f8a567-8196-a1f0-0254-117f05475ea0: entering follower state: follower="Node at 83f8a567-8196-a1f0-0254-117f05475ea0 [Follower]" leader-address= leader-id= +2026-11-29T10:29:31.851Z [INFO] server-59011c36-423d-e122-5895-cd2e48af1185: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-83f8a567-8196-a1f0-0254-117f05475ea0 Address:83f8a567-8196-a1f0-0254-117f05475ea0} {Suffrage:Voter ID:server-83d2b0aa-af06-6d71-6ad2-0c0579882f86 Address:83d2b0aa-af06-6d71-6ad2-0c0579882f86} {Suffrage:Voter ID:server-59011c36-423d-e122-5895-cd2e48af1185 Address:59011c36-423d-e122-5895-cd2e48af1185}]" +2026-11-29T10:29:31.851Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:31.851Z [INFO] server-59011c36-423d-e122-5895-cd2e48af1185: entering follower state: follower="Node at 59011c36-423d-e122-5895-cd2e48af1185 [Follower]" leader-address= leader-id= +2026-11-29T10:29:31.851Z [INFO] server-83d2b0aa-af06-6d71-6ad2-0c0579882f86: entering follower state: follower="Node at 83d2b0aa-af06-6d71-6ad2-0c0579882f86 [Follower]" leader-address= leader-id= +2026-11-29T10:29:32.591Z [WARN] server-83d2b0aa-af06-6d71-6ad2-0c0579882f86: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:32.591Z [INFO] server-83d2b0aa-af06-6d71-6ad2-0c0579882f86: entering candidate state: node="Node at 83d2b0aa-af06-6d71-6ad2-0c0579882f86 [Candidate]" term=2 +2026-11-29T10:29:32.591Z [INFO] server-83d2b0aa-af06-6d71-6ad2-0c0579882f86: election won: term=2 tally=2 +2026-11-29T10:29:32.591Z [INFO] server-83d2b0aa-af06-6d71-6ad2-0c0579882f86: entering leader state: leader="Node at 83d2b0aa-af06-6d71-6ad2-0c0579882f86 [Leader]" +2026-11-29T10:29:32.591Z [INFO] server-83d2b0aa-af06-6d71-6ad2-0c0579882f86: added peer, starting replication: peer=server-83f8a567-8196-a1f0-0254-117f05475ea0 +2026-11-29T10:29:32.591Z [INFO] server-83d2b0aa-af06-6d71-6ad2-0c0579882f86: added peer, starting replication: peer=server-59011c36-423d-e122-5895-cd2e48af1185 +2026-11-29T10:29:32.591Z [INFO] server-83d2b0aa-af06-6d71-6ad2-0c0579882f86: pipelining replication: peer="{Voter server-59011c36-423d-e122-5895-cd2e48af1185 59011c36-423d-e122-5895-cd2e48af1185}" +2026-11-29T10:29:32.592Z [INFO] server-83d2b0aa-af06-6d71-6ad2-0c0579882f86: pipelining replication: peer="{Voter server-83f8a567-8196-a1f0-0254-117f05475ea0 83f8a567-8196-a1f0-0254-117f05475ea0}" +2026-11-29T10:29:33.597Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:32.591745499 +0000 UTC m=+83.326799601 (1 nodes), 740.37376ms from start of poll, 740.55208ms from cluster start. Timeout at 2026-11-29 10:29:33.596747179 +0000 UTC m=+84.331801281, 1.00500168s after stability +2026-11-29T10:29:33.597Z [INFO] server-83d2b0aa-af06-6d71-6ad2-0c0579882f86: starting snapshot up to: index=12 +2026-11-29T10:29:33.597Z [INFO] creating new snapshot: path=/tmp/raft227784918/snapshots/2-12-1795948173597.tmp +2026-11-29T10:29:33.598Z [INFO] server-83d2b0aa-af06-6d71-6ad2-0c0579882f86: snapshot complete up to: index=12 +2026-11-29T10:29:33.598Z [INFO] creating new snapshot: path=/tmp/raft227784918/snapshots/2-10013-1795948173598.tmp +2026-11-29T10:29:33.598Z [INFO] server-83d2b0aa-af06-6d71-6ad2-0c0579882f86: copied to local snapshot: bytes=71 +2026-11-29T10:29:33.598Z [INFO] server-83d2b0aa-af06-6d71-6ad2-0c0579882f86: snapshot restore progress: id=2-10013-1795948173598 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:33.598Z [INFO] server-83d2b0aa-af06-6d71-6ad2-0c0579882f86: restored user snapshot: index=10013 +2026-11-29T10:29:33.598Z [ERROR] server-83d2b0aa-af06-6d71-6ad2-0c0579882f86: failed to get log: index=23 error="log not found" +2026-11-29T10:29:33.598Z [INFO] server-83d2b0aa-af06-6d71-6ad2-0c0579882f86: aborting pipeline replication: peer="{Voter server-59011c36-423d-e122-5895-cd2e48af1185 59011c36-423d-e122-5895-cd2e48af1185}" +2026-11-29T10:29:33.598Z [ERROR] server-83d2b0aa-af06-6d71-6ad2-0c0579882f86: failed to get log: index=23 error="log not found" +2026-11-29T10:29:33.598Z [INFO] server-83d2b0aa-af06-6d71-6ad2-0c0579882f86: aborting pipeline replication: peer="{Voter server-83f8a567-8196-a1f0-0254-117f05475ea0 83f8a567-8196-a1f0-0254-117f05475ea0}" +2026-11-29T10:29:33.598Z [ERROR] server-83d2b0aa-af06-6d71-6ad2-0c0579882f86: failed to get log: index=23 error="log not found" +2026-11-29T10:29:33.599Z [INFO] creating new snapshot: path=/tmp/raft1133623385/snapshots/2-10013-1795948173599.tmp +2026-11-29T10:29:33.599Z [INFO] server-83f8a567-8196-a1f0-0254-117f05475ea0: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:33.599Z [INFO] server-83f8a567-8196-a1f0-0254-117f05475ea0: copied to local snapshot: bytes=71 +2026-11-29T10:29:33.599Z [INFO] server-83f8a567-8196-a1f0-0254-117f05475ea0: snapshot restore progress: id=2-10013-1795948173599 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:33.599Z [INFO] server-83f8a567-8196-a1f0-0254-117f05475ea0: Installed remote snapshot +2026-11-29T10:29:33.599Z [WARN] server-83f8a567-8196-a1f0-0254-117f05475ea0: skipping application of old log: index=22 +2026-11-29T10:29:33.599Z [INFO] server-83d2b0aa-af06-6d71-6ad2-0c0579882f86: pipelining replication: peer="{Voter server-83f8a567-8196-a1f0-0254-117f05475ea0 83f8a567-8196-a1f0-0254-117f05475ea0}" +2026-11-29T10:29:33.599Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:33.605Z [ERROR] server-83d2b0aa-af06-6d71-6ad2-0c0579882f86: failed to get log: index=23 error="log not found" +2026-11-29T10:29:33.605Z [INFO] creating new snapshot: path=/tmp/raft3981018582/snapshots/2-10013-1795948173605.tmp +2026-11-29T10:29:33.606Z [INFO] server-59011c36-423d-e122-5895-cd2e48af1185: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:33.606Z [INFO] server-59011c36-423d-e122-5895-cd2e48af1185: copied to local snapshot: bytes=71 +2026-11-29T10:29:33.606Z [INFO] server-59011c36-423d-e122-5895-cd2e48af1185: snapshot restore progress: id=2-10013-1795948173605 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:33.606Z [INFO] server-59011c36-423d-e122-5895-cd2e48af1185: Installed remote snapshot +2026-11-29T10:29:33.606Z [INFO] server-83d2b0aa-af06-6d71-6ad2-0c0579882f86: pipelining replication: peer="{Voter server-59011c36-423d-e122-5895-cd2e48af1185 59011c36-423d-e122-5895-cd2e48af1185}" +2026-11-29T10:29:34.605Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:33.599610459 +0000 UTC m=+84.334664561 (2 nodes), 440ns from start of poll, 1.74841704s from cluster start. Timeout at 2026-11-29 10:29:34.604623499 +0000 UTC m=+85.339677601, 1.00501304s after stability +2026-11-29T10:29:34.617Z [INFO] server-83d2b0aa-af06-6d71-6ad2-0c0579882f86: aborting pipeline replication: peer="{Voter server-59011c36-423d-e122-5895-cd2e48af1185 59011c36-423d-e122-5895-cd2e48af1185}" +2026-11-29T10:29:34.617Z [INFO] server-83d2b0aa-af06-6d71-6ad2-0c0579882f86: aborting pipeline replication: peer="{Voter server-83f8a567-8196-a1f0-0254-117f05475ea0 83f8a567-8196-a1f0-0254-117f05475ea0}" === RUN TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_false -2025-10-27T04:03:32.247Z [INFO] server-48525403-d334-982a-200e-1f80cfe36831: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-48525403-d334-982a-200e-1f80cfe36831 Address:48525403-d334-982a-200e-1f80cfe36831} {Suffrage:Voter ID:server-f0dc6e3b-5c20-e886-f58d-92aaad34936e Address:f0dc6e3b-5c20-e886-f58d-92aaad34936e} {Suffrage:Voter ID:server-d4504473-5e12-4f55-16d5-c07dbbf57496 Address:d4504473-5e12-4f55-16d5-c07dbbf57496}]" -2025-10-27T04:03:32.247Z [INFO] server-48525403-d334-982a-200e-1f80cfe36831: entering follower state: follower="Node at 48525403-d334-982a-200e-1f80cfe36831 [Follower]" leader-address= leader-id= -2025-10-27T04:03:32.247Z [INFO] server-f0dc6e3b-5c20-e886-f58d-92aaad34936e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-48525403-d334-982a-200e-1f80cfe36831 Address:48525403-d334-982a-200e-1f80cfe36831} {Suffrage:Voter ID:server-f0dc6e3b-5c20-e886-f58d-92aaad34936e Address:f0dc6e3b-5c20-e886-f58d-92aaad34936e} {Suffrage:Voter ID:server-d4504473-5e12-4f55-16d5-c07dbbf57496 Address:d4504473-5e12-4f55-16d5-c07dbbf57496}]" -2025-10-27T04:03:32.247Z [INFO] server-f0dc6e3b-5c20-e886-f58d-92aaad34936e: entering follower state: follower="Node at f0dc6e3b-5c20-e886-f58d-92aaad34936e [Follower]" leader-address= leader-id= -2025-10-27T04:03:32.247Z [INFO] server-d4504473-5e12-4f55-16d5-c07dbbf57496: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-48525403-d334-982a-200e-1f80cfe36831 Address:48525403-d334-982a-200e-1f80cfe36831} {Suffrage:Voter ID:server-f0dc6e3b-5c20-e886-f58d-92aaad34936e Address:f0dc6e3b-5c20-e886-f58d-92aaad34936e} {Suffrage:Voter ID:server-d4504473-5e12-4f55-16d5-c07dbbf57496 Address:d4504473-5e12-4f55-16d5-c07dbbf57496}]" -2025-10-27T04:03:32.247Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:32.248Z [INFO] server-d4504473-5e12-4f55-16d5-c07dbbf57496: entering follower state: follower="Node at d4504473-5e12-4f55-16d5-c07dbbf57496 [Follower]" leader-address= leader-id= -2025-10-27T04:03:32.781Z [WARN] server-48525403-d334-982a-200e-1f80cfe36831: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:32.781Z [INFO] server-48525403-d334-982a-200e-1f80cfe36831: entering candidate state: node="Node at 48525403-d334-982a-200e-1f80cfe36831 [Candidate]" term=2 -2025-10-27T04:03:32.781Z [INFO] server-48525403-d334-982a-200e-1f80cfe36831: election won: term=2 tally=2 -2025-10-27T04:03:32.781Z [INFO] server-48525403-d334-982a-200e-1f80cfe36831: entering leader state: leader="Node at 48525403-d334-982a-200e-1f80cfe36831 [Leader]" -2025-10-27T04:03:32.781Z [INFO] server-48525403-d334-982a-200e-1f80cfe36831: added peer, starting replication: peer=server-f0dc6e3b-5c20-e886-f58d-92aaad34936e -2025-10-27T04:03:32.781Z [INFO] server-48525403-d334-982a-200e-1f80cfe36831: added peer, starting replication: peer=server-d4504473-5e12-4f55-16d5-c07dbbf57496 -2025-10-27T04:03:32.781Z [INFO] server-48525403-d334-982a-200e-1f80cfe36831: pipelining replication: peer="{Voter server-d4504473-5e12-4f55-16d5-c07dbbf57496 d4504473-5e12-4f55-16d5-c07dbbf57496}" -2025-10-27T04:03:32.781Z [INFO] server-48525403-d334-982a-200e-1f80cfe36831: pipelining replication: peer="{Voter server-f0dc6e3b-5c20-e886-f58d-92aaad34936e f0dc6e3b-5c20-e886-f58d-92aaad34936e}" -2025-10-27T04:03:33.787Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:32.781560045 +0000 UTC m=+87.776432523 (1 nodes), 533.568646ms from start of poll, 534.045089ms from cluster start. Timeout at 2025-10-27 04:03:33.786563885 +0000 UTC m=+88.781436323, 1.0050038s after stability -2025-10-27T04:03:33.787Z [INFO] server-48525403-d334-982a-200e-1f80cfe36831: starting snapshot up to: index=12 -2025-10-27T04:03:33.787Z [INFO] creating new snapshot: path=/tmp/raft708185190/snapshots/2-12-1761537813787.tmp -2025-10-27T04:03:33.788Z [INFO] server-48525403-d334-982a-200e-1f80cfe36831: snapshot complete up to: index=12 -2025-10-27T04:03:33.789Z [INFO] creating new snapshot: path=/tmp/raft708185190/snapshots/2-10013-1761537813789.tmp -2025-10-27T04:03:33.790Z [INFO] server-48525403-d334-982a-200e-1f80cfe36831: copied to local snapshot: bytes=71 -2025-10-27T04:03:33.790Z [INFO] server-48525403-d334-982a-200e-1f80cfe36831: snapshot restore progress: id=2-10013-1761537813789 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:33.790Z [INFO] server-48525403-d334-982a-200e-1f80cfe36831: removing all old logs from log store -2025-10-27T04:03:33.790Z [INFO] server-48525403-d334-982a-200e-1f80cfe36831: compacting logs: from=1 to=22 -2025-10-27T04:03:33.790Z [INFO] server-48525403-d334-982a-200e-1f80cfe36831: restored user snapshot: index=10013 -2025-10-27T04:03:33.790Z [ERROR] server-48525403-d334-982a-200e-1f80cfe36831: failed to get log: index=22 error="log not found" -2025-10-27T04:03:33.790Z [INFO] server-48525403-d334-982a-200e-1f80cfe36831: aborting pipeline replication: peer="{Voter server-f0dc6e3b-5c20-e886-f58d-92aaad34936e f0dc6e3b-5c20-e886-f58d-92aaad34936e}" -2025-10-27T04:03:33.790Z [ERROR] server-48525403-d334-982a-200e-1f80cfe36831: failed to get log: index=22 error="log not found" -2025-10-27T04:03:33.790Z [INFO] server-48525403-d334-982a-200e-1f80cfe36831: aborting pipeline replication: peer="{Voter server-d4504473-5e12-4f55-16d5-c07dbbf57496 d4504473-5e12-4f55-16d5-c07dbbf57496}" -2025-10-27T04:03:33.796Z [ERROR] server-48525403-d334-982a-200e-1f80cfe36831: failed to get log: index=22 error="log not found" -2025-10-27T04:03:33.796Z [INFO] creating new snapshot: path=/tmp/raft3172200541/snapshots/2-10013-1761537813796.tmp -2025-10-27T04:03:33.797Z [INFO] server-d4504473-5e12-4f55-16d5-c07dbbf57496: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:33.797Z [INFO] server-d4504473-5e12-4f55-16d5-c07dbbf57496: copied to local snapshot: bytes=71 -2025-10-27T04:03:33.797Z [INFO] server-d4504473-5e12-4f55-16d5-c07dbbf57496: snapshot restore progress: id=2-10013-1761537813796 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:33.797Z [INFO] server-d4504473-5e12-4f55-16d5-c07dbbf57496: removing all old logs from log store -2025-10-27T04:03:33.797Z [INFO] server-d4504473-5e12-4f55-16d5-c07dbbf57496: compacting logs: from=1 to=22 -2025-10-27T04:03:33.797Z [INFO] server-d4504473-5e12-4f55-16d5-c07dbbf57496: Installed remote snapshot -2025-10-27T04:03:33.797Z [INFO] server-48525403-d334-982a-200e-1f80cfe36831: pipelining replication: peer="{Voter server-d4504473-5e12-4f55-16d5-c07dbbf57496 d4504473-5e12-4f55-16d5-c07dbbf57496}" -2025-10-27T04:03:33.797Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:03:33.799Z [ERROR] server-48525403-d334-982a-200e-1f80cfe36831: failed to get log: index=22 error="log not found" -2025-10-27T04:03:33.800Z [INFO] creating new snapshot: path=/tmp/raft3943045069/snapshots/2-10013-1761537813800.tmp -2025-10-27T04:03:33.800Z [INFO] server-f0dc6e3b-5c20-e886-f58d-92aaad34936e: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:33.800Z [INFO] server-f0dc6e3b-5c20-e886-f58d-92aaad34936e: copied to local snapshot: bytes=71 -2025-10-27T04:03:33.800Z [INFO] server-f0dc6e3b-5c20-e886-f58d-92aaad34936e: snapshot restore progress: id=2-10013-1761537813800 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:33.800Z [INFO] server-f0dc6e3b-5c20-e886-f58d-92aaad34936e: removing all old logs from log store -2025-10-27T04:03:33.800Z [INFO] server-f0dc6e3b-5c20-e886-f58d-92aaad34936e: compacting logs: from=1 to=22 -2025-10-27T04:03:33.800Z [INFO] server-f0dc6e3b-5c20-e886-f58d-92aaad34936e: Installed remote snapshot -2025-10-27T04:03:33.800Z [INFO] server-48525403-d334-982a-200e-1f80cfe36831: pipelining replication: peer="{Voter server-f0dc6e3b-5c20-e886-f58d-92aaad34936e f0dc6e3b-5c20-e886-f58d-92aaad34936e}" -2025-10-27T04:03:34.802Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:03:33.797661507 +0000 UTC m=+88.792533945 (2 nodes), 800ns from start of poll, 1.550146511s from cluster start. Timeout at 2025-10-27 04:03:34.802678387 +0000 UTC m=+89.797550825, 1.00501688s after stability -2025-10-27T04:03:34.825Z [INFO] server-48525403-d334-982a-200e-1f80cfe36831: aborting pipeline replication: peer="{Voter server-d4504473-5e12-4f55-16d5-c07dbbf57496 d4504473-5e12-4f55-16d5-c07dbbf57496}" -2025-10-27T04:03:34.825Z [INFO] server-48525403-d334-982a-200e-1f80cfe36831: aborting pipeline replication: peer="{Voter server-f0dc6e3b-5c20-e886-f58d-92aaad34936e f0dc6e3b-5c20-e886-f58d-92aaad34936e}" +2026-11-29T10:29:34.618Z [INFO] server-7d81bcce-5579-103c-75c2-2e5d728ff231: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7d81bcce-5579-103c-75c2-2e5d728ff231 Address:7d81bcce-5579-103c-75c2-2e5d728ff231} {Suffrage:Voter ID:server-09aac037-2a8c-86e2-efa3-5516d34aad7a Address:09aac037-2a8c-86e2-efa3-5516d34aad7a} {Suffrage:Voter ID:server-ab4c3019-f651-8d99-067d-75559baf69ba Address:ab4c3019-f651-8d99-067d-75559baf69ba}]" +2026-11-29T10:29:34.618Z [INFO] server-09aac037-2a8c-86e2-efa3-5516d34aad7a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7d81bcce-5579-103c-75c2-2e5d728ff231 Address:7d81bcce-5579-103c-75c2-2e5d728ff231} {Suffrage:Voter ID:server-09aac037-2a8c-86e2-efa3-5516d34aad7a Address:09aac037-2a8c-86e2-efa3-5516d34aad7a} {Suffrage:Voter ID:server-ab4c3019-f651-8d99-067d-75559baf69ba Address:ab4c3019-f651-8d99-067d-75559baf69ba}]" +2026-11-29T10:29:34.618Z [INFO] server-ab4c3019-f651-8d99-067d-75559baf69ba: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7d81bcce-5579-103c-75c2-2e5d728ff231 Address:7d81bcce-5579-103c-75c2-2e5d728ff231} {Suffrage:Voter ID:server-09aac037-2a8c-86e2-efa3-5516d34aad7a Address:09aac037-2a8c-86e2-efa3-5516d34aad7a} {Suffrage:Voter ID:server-ab4c3019-f651-8d99-067d-75559baf69ba Address:ab4c3019-f651-8d99-067d-75559baf69ba}]" +2026-11-29T10:29:34.618Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:34.618Z [INFO] server-7d81bcce-5579-103c-75c2-2e5d728ff231: entering follower state: follower="Node at 7d81bcce-5579-103c-75c2-2e5d728ff231 [Follower]" leader-address= leader-id= +2026-11-29T10:29:34.618Z [INFO] server-09aac037-2a8c-86e2-efa3-5516d34aad7a: entering follower state: follower="Node at 09aac037-2a8c-86e2-efa3-5516d34aad7a [Follower]" leader-address= leader-id= +2026-11-29T10:29:34.618Z [INFO] server-ab4c3019-f651-8d99-067d-75559baf69ba: entering follower state: follower="Node at ab4c3019-f651-8d99-067d-75559baf69ba [Follower]" leader-address= leader-id= +2026-11-29T10:29:35.131Z [WARN] server-7d81bcce-5579-103c-75c2-2e5d728ff231: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:35.131Z [INFO] server-7d81bcce-5579-103c-75c2-2e5d728ff231: entering candidate state: node="Node at 7d81bcce-5579-103c-75c2-2e5d728ff231 [Candidate]" term=2 +2026-11-29T10:29:35.131Z [INFO] server-7d81bcce-5579-103c-75c2-2e5d728ff231: election won: term=2 tally=2 +2026-11-29T10:29:35.131Z [INFO] server-7d81bcce-5579-103c-75c2-2e5d728ff231: entering leader state: leader="Node at 7d81bcce-5579-103c-75c2-2e5d728ff231 [Leader]" +2026-11-29T10:29:35.131Z [INFO] server-7d81bcce-5579-103c-75c2-2e5d728ff231: added peer, starting replication: peer=server-09aac037-2a8c-86e2-efa3-5516d34aad7a +2026-11-29T10:29:35.131Z [INFO] server-7d81bcce-5579-103c-75c2-2e5d728ff231: added peer, starting replication: peer=server-ab4c3019-f651-8d99-067d-75559baf69ba +2026-11-29T10:29:35.131Z [INFO] server-7d81bcce-5579-103c-75c2-2e5d728ff231: pipelining replication: peer="{Voter server-ab4c3019-f651-8d99-067d-75559baf69ba ab4c3019-f651-8d99-067d-75559baf69ba}" +2026-11-29T10:29:35.131Z [INFO] server-7d81bcce-5579-103c-75c2-2e5d728ff231: pipelining replication: peer="{Voter server-09aac037-2a8c-86e2-efa3-5516d34aad7a 09aac037-2a8c-86e2-efa3-5516d34aad7a}" +2026-11-29T10:29:36.138Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:35.131565899 +0000 UTC m=+85.866620001 (1 nodes), 513.1594ms from start of poll, 513.34032ms from cluster start. Timeout at 2026-11-29 10:29:36.136570019 +0000 UTC m=+86.871624201, 1.0050042s after stability +2026-11-29T10:29:36.138Z [INFO] server-7d81bcce-5579-103c-75c2-2e5d728ff231: starting snapshot up to: index=12 +2026-11-29T10:29:36.138Z [INFO] creating new snapshot: path=/tmp/raft892105742/snapshots/2-12-1795948176138.tmp +2026-11-29T10:29:36.138Z [INFO] server-7d81bcce-5579-103c-75c2-2e5d728ff231: snapshot complete up to: index=12 +2026-11-29T10:29:36.139Z [INFO] creating new snapshot: path=/tmp/raft892105742/snapshots/2-10013-1795948176139.tmp +2026-11-29T10:29:36.139Z [INFO] server-7d81bcce-5579-103c-75c2-2e5d728ff231: copied to local snapshot: bytes=71 +2026-11-29T10:29:36.139Z [INFO] server-7d81bcce-5579-103c-75c2-2e5d728ff231: snapshot restore progress: id=2-10013-1795948176139 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:36.139Z [INFO] server-7d81bcce-5579-103c-75c2-2e5d728ff231: removing all old logs from log store +2026-11-29T10:29:36.139Z [INFO] server-7d81bcce-5579-103c-75c2-2e5d728ff231: compacting logs: from=1 to=22 +2026-11-29T10:29:36.139Z [INFO] server-7d81bcce-5579-103c-75c2-2e5d728ff231: restored user snapshot: index=10013 +2026-11-29T10:29:36.139Z [ERROR] server-7d81bcce-5579-103c-75c2-2e5d728ff231: failed to get log: index=22 error="log not found" +2026-11-29T10:29:36.139Z [INFO] server-7d81bcce-5579-103c-75c2-2e5d728ff231: aborting pipeline replication: peer="{Voter server-ab4c3019-f651-8d99-067d-75559baf69ba ab4c3019-f651-8d99-067d-75559baf69ba}" +2026-11-29T10:29:36.139Z [ERROR] server-7d81bcce-5579-103c-75c2-2e5d728ff231: failed to get log: index=22 error="log not found" +2026-11-29T10:29:36.139Z [INFO] server-7d81bcce-5579-103c-75c2-2e5d728ff231: aborting pipeline replication: peer="{Voter server-09aac037-2a8c-86e2-efa3-5516d34aad7a 09aac037-2a8c-86e2-efa3-5516d34aad7a}" +2026-11-29T10:29:36.146Z [ERROR] server-7d81bcce-5579-103c-75c2-2e5d728ff231: failed to get log: index=22 error="log not found" +2026-11-29T10:29:36.147Z [INFO] creating new snapshot: path=/tmp/raft3086312379/snapshots/2-10013-1795948176147.tmp +2026-11-29T10:29:36.147Z [INFO] server-09aac037-2a8c-86e2-efa3-5516d34aad7a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:36.147Z [INFO] server-09aac037-2a8c-86e2-efa3-5516d34aad7a: copied to local snapshot: bytes=71 +2026-11-29T10:29:36.147Z [INFO] server-09aac037-2a8c-86e2-efa3-5516d34aad7a: snapshot restore progress: id=2-10013-1795948176147 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:36.147Z [INFO] server-09aac037-2a8c-86e2-efa3-5516d34aad7a: removing all old logs from log store +2026-11-29T10:29:36.147Z [INFO] server-09aac037-2a8c-86e2-efa3-5516d34aad7a: compacting logs: from=1 to=22 +2026-11-29T10:29:36.147Z [INFO] server-09aac037-2a8c-86e2-efa3-5516d34aad7a: Installed remote snapshot +2026-11-29T10:29:36.147Z [WARN] server-09aac037-2a8c-86e2-efa3-5516d34aad7a: skipping application of old log: index=22 +2026-11-29T10:29:36.147Z [INFO] server-7d81bcce-5579-103c-75c2-2e5d728ff231: pipelining replication: peer="{Voter server-09aac037-2a8c-86e2-efa3-5516d34aad7a 09aac037-2a8c-86e2-efa3-5516d34aad7a}" +2026-11-29T10:29:36.147Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:36.147Z [ERROR] server-7d81bcce-5579-103c-75c2-2e5d728ff231: failed to get log: index=22 error="log not found" +2026-11-29T10:29:36.147Z [INFO] creating new snapshot: path=/tmp/raft795813430/snapshots/2-10013-1795948176147.tmp +2026-11-29T10:29:36.148Z [INFO] server-ab4c3019-f651-8d99-067d-75559baf69ba: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:36.148Z [INFO] server-ab4c3019-f651-8d99-067d-75559baf69ba: copied to local snapshot: bytes=71 +2026-11-29T10:29:36.148Z [INFO] server-ab4c3019-f651-8d99-067d-75559baf69ba: snapshot restore progress: id=2-10013-1795948176147 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:36.148Z [INFO] server-ab4c3019-f651-8d99-067d-75559baf69ba: removing all old logs from log store +2026-11-29T10:29:36.148Z [INFO] server-ab4c3019-f651-8d99-067d-75559baf69ba: compacting logs: from=1 to=22 +2026-11-29T10:29:36.148Z [INFO] server-ab4c3019-f651-8d99-067d-75559baf69ba: Installed remote snapshot +2026-11-29T10:29:36.148Z [INFO] server-7d81bcce-5579-103c-75c2-2e5d728ff231: pipelining replication: peer="{Voter server-ab4c3019-f651-8d99-067d-75559baf69ba ab4c3019-f651-8d99-067d-75559baf69ba}" +2026-11-29T10:29:37.152Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:36.147743619 +0000 UTC m=+86.882797721 (2 nodes), 480ns from start of poll, 1.52951804s from cluster start. Timeout at 2026-11-29 10:29:37.152775939 +0000 UTC m=+87.887830041, 1.00503232s after stability +2026-11-29T10:29:37.164Z [INFO] server-7d81bcce-5579-103c-75c2-2e5d728ff231: aborting pipeline replication: peer="{Voter server-ab4c3019-f651-8d99-067d-75559baf69ba ab4c3019-f651-8d99-067d-75559baf69ba}" +2026-11-29T10:29:37.164Z [INFO] server-7d81bcce-5579-103c-75c2-2e5d728ff231: aborting pipeline replication: peer="{Voter server-09aac037-2a8c-86e2-efa3-5516d34aad7a 09aac037-2a8c-86e2-efa3-5516d34aad7a}" === RUN TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_true -2025-10-27T04:03:34.825Z [INFO] server-17376f89-1602-df04-2a54-a3561510b029: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-17376f89-1602-df04-2a54-a3561510b029 Address:17376f89-1602-df04-2a54-a3561510b029} {Suffrage:Voter ID:server-96263d0c-8a48-3791-99df-abbbd25dcac4 Address:96263d0c-8a48-3791-99df-abbbd25dcac4} {Suffrage:Voter ID:server-1f295cb1-69ae-0b1b-e0d9-e928ada322ca Address:1f295cb1-69ae-0b1b-e0d9-e928ada322ca}]" -2025-10-27T04:03:34.825Z [INFO] server-96263d0c-8a48-3791-99df-abbbd25dcac4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-17376f89-1602-df04-2a54-a3561510b029 Address:17376f89-1602-df04-2a54-a3561510b029} {Suffrage:Voter ID:server-96263d0c-8a48-3791-99df-abbbd25dcac4 Address:96263d0c-8a48-3791-99df-abbbd25dcac4} {Suffrage:Voter ID:server-1f295cb1-69ae-0b1b-e0d9-e928ada322ca Address:1f295cb1-69ae-0b1b-e0d9-e928ada322ca}]" -2025-10-27T04:03:34.825Z [INFO] server-17376f89-1602-df04-2a54-a3561510b029: entering follower state: follower="Node at 17376f89-1602-df04-2a54-a3561510b029 [Follower]" leader-address= leader-id= -2025-10-27T04:03:34.826Z [INFO] server-96263d0c-8a48-3791-99df-abbbd25dcac4: entering follower state: follower="Node at 96263d0c-8a48-3791-99df-abbbd25dcac4 [Follower]" leader-address= leader-id= -2025-10-27T04:03:34.826Z [INFO] server-1f295cb1-69ae-0b1b-e0d9-e928ada322ca: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-17376f89-1602-df04-2a54-a3561510b029 Address:17376f89-1602-df04-2a54-a3561510b029} {Suffrage:Voter ID:server-96263d0c-8a48-3791-99df-abbbd25dcac4 Address:96263d0c-8a48-3791-99df-abbbd25dcac4} {Suffrage:Voter ID:server-1f295cb1-69ae-0b1b-e0d9-e928ada322ca Address:1f295cb1-69ae-0b1b-e0d9-e928ada322ca}]" -2025-10-27T04:03:34.826Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:34.826Z [INFO] server-1f295cb1-69ae-0b1b-e0d9-e928ada322ca: entering follower state: follower="Node at 1f295cb1-69ae-0b1b-e0d9-e928ada322ca [Follower]" leader-address= leader-id= -2025-10-27T04:03:35.649Z [WARN] server-17376f89-1602-df04-2a54-a3561510b029: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:35.649Z [INFO] server-17376f89-1602-df04-2a54-a3561510b029: entering candidate state: node="Node at 17376f89-1602-df04-2a54-a3561510b029 [Candidate]" term=2 -2025-10-27T04:03:35.649Z [INFO] server-17376f89-1602-df04-2a54-a3561510b029: election won: term=2 tally=2 -2025-10-27T04:03:35.649Z [INFO] server-17376f89-1602-df04-2a54-a3561510b029: entering leader state: leader="Node at 17376f89-1602-df04-2a54-a3561510b029 [Leader]" -2025-10-27T04:03:35.649Z [INFO] server-17376f89-1602-df04-2a54-a3561510b029: added peer, starting replication: peer=server-96263d0c-8a48-3791-99df-abbbd25dcac4 -2025-10-27T04:03:35.649Z [INFO] server-17376f89-1602-df04-2a54-a3561510b029: added peer, starting replication: peer=server-1f295cb1-69ae-0b1b-e0d9-e928ada322ca -2025-10-27T04:03:35.649Z [INFO] server-17376f89-1602-df04-2a54-a3561510b029: pipelining replication: peer="{Voter server-1f295cb1-69ae-0b1b-e0d9-e928ada322ca 1f295cb1-69ae-0b1b-e0d9-e928ada322ca}" -2025-10-27T04:03:35.649Z [INFO] server-17376f89-1602-df04-2a54-a3561510b029: pipelining replication: peer="{Voter server-96263d0c-8a48-3791-99df-abbbd25dcac4 96263d0c-8a48-3791-99df-abbbd25dcac4}" -2025-10-27T04:03:36.655Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:35.649851281 +0000 UTC m=+90.644723719 (1 nodes), 823.781908ms from start of poll, 823.999909ms from cluster start. Timeout at 2025-10-27 04:03:36.654864081 +0000 UTC m=+91.649736479, 1.00501276s after stability -2025-10-27T04:03:36.655Z [INFO] server-17376f89-1602-df04-2a54-a3561510b029: starting snapshot up to: index=12 -2025-10-27T04:03:36.655Z [INFO] creating new snapshot: path=/tmp/raft1484719253/snapshots/2-12-1761537816655.tmp -2025-10-27T04:03:36.657Z [INFO] server-17376f89-1602-df04-2a54-a3561510b029: snapshot complete up to: index=12 -2025-10-27T04:03:36.658Z [INFO] server-e4e492f1-8d77-3433-eadb-830d58474112: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e4e492f1-8d77-3433-eadb-830d58474112 Address:e4e492f1-8d77-3433-eadb-830d58474112} {Suffrage:Voter ID:server-a1338678-1b16-28f6-2942-f73b97d17a4d Address:a1338678-1b16-28f6-2942-f73b97d17a4d} {Suffrage:Voter ID:server-def35d83-cd5d-9408-48f3-2161fddf6527 Address:def35d83-cd5d-9408-48f3-2161fddf6527}]" -2025-10-27T04:03:36.658Z [INFO] server-e4e492f1-8d77-3433-eadb-830d58474112: entering follower state: follower="Node at e4e492f1-8d77-3433-eadb-830d58474112 [Follower]" leader-address= leader-id= -2025-10-27T04:03:36.658Z [INFO] server-a1338678-1b16-28f6-2942-f73b97d17a4d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e4e492f1-8d77-3433-eadb-830d58474112 Address:e4e492f1-8d77-3433-eadb-830d58474112} {Suffrage:Voter ID:server-a1338678-1b16-28f6-2942-f73b97d17a4d Address:a1338678-1b16-28f6-2942-f73b97d17a4d} {Suffrage:Voter ID:server-def35d83-cd5d-9408-48f3-2161fddf6527 Address:def35d83-cd5d-9408-48f3-2161fddf6527}]" -2025-10-27T04:03:36.658Z [INFO] server-a1338678-1b16-28f6-2942-f73b97d17a4d: entering follower state: follower="Node at a1338678-1b16-28f6-2942-f73b97d17a4d [Follower]" leader-address= leader-id= -2025-10-27T04:03:36.658Z [INFO] server-def35d83-cd5d-9408-48f3-2161fddf6527: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e4e492f1-8d77-3433-eadb-830d58474112 Address:e4e492f1-8d77-3433-eadb-830d58474112} {Suffrage:Voter ID:server-a1338678-1b16-28f6-2942-f73b97d17a4d Address:a1338678-1b16-28f6-2942-f73b97d17a4d} {Suffrage:Voter ID:server-def35d83-cd5d-9408-48f3-2161fddf6527 Address:def35d83-cd5d-9408-48f3-2161fddf6527}]" -2025-10-27T04:03:36.658Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:36.658Z [INFO] server-def35d83-cd5d-9408-48f3-2161fddf6527: entering follower state: follower="Node at def35d83-cd5d-9408-48f3-2161fddf6527 [Follower]" leader-address= leader-id= -2025-10-27T04:03:37.180Z [WARN] server-e4e492f1-8d77-3433-eadb-830d58474112: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:37.180Z [INFO] server-e4e492f1-8d77-3433-eadb-830d58474112: entering candidate state: node="Node at e4e492f1-8d77-3433-eadb-830d58474112 [Candidate]" term=2 -2025-10-27T04:03:37.180Z [INFO] server-e4e492f1-8d77-3433-eadb-830d58474112: election won: term=2 tally=2 -2025-10-27T04:03:37.180Z [INFO] server-e4e492f1-8d77-3433-eadb-830d58474112: entering leader state: leader="Node at e4e492f1-8d77-3433-eadb-830d58474112 [Leader]" -2025-10-27T04:03:37.180Z [INFO] server-e4e492f1-8d77-3433-eadb-830d58474112: added peer, starting replication: peer=server-a1338678-1b16-28f6-2942-f73b97d17a4d -2025-10-27T04:03:37.180Z [INFO] server-e4e492f1-8d77-3433-eadb-830d58474112: added peer, starting replication: peer=server-def35d83-cd5d-9408-48f3-2161fddf6527 -2025-10-27T04:03:37.181Z [INFO] server-e4e492f1-8d77-3433-eadb-830d58474112: pipelining replication: peer="{Voter server-def35d83-cd5d-9408-48f3-2161fddf6527 def35d83-cd5d-9408-48f3-2161fddf6527}" -2025-10-27T04:03:37.181Z [INFO] server-e4e492f1-8d77-3433-eadb-830d58474112: pipelining replication: peer="{Voter server-a1338678-1b16-28f6-2942-f73b97d17a4d a1338678-1b16-28f6-2942-f73b97d17a4d}" -2025-10-27T04:03:38.186Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:37.181028423 +0000 UTC m=+92.175900821 (1 nodes), 522.296889ms from start of poll, 522.671131ms from cluster start. Timeout at 2025-10-27 04:03:38.186029943 +0000 UTC m=+93.180902381, 1.00500156s after stability -2025-10-27T04:03:38.186Z [INFO] creating new snapshot: path=/tmp/raft6579578/snapshots/2-10013-1761537818186.tmp -2025-10-27T04:03:38.186Z [INFO] server-e4e492f1-8d77-3433-eadb-830d58474112: copied to local snapshot: bytes=71 -2025-10-27T04:03:38.186Z [INFO] server-e4e492f1-8d77-3433-eadb-830d58474112: snapshot restore progress: id=2-10013-1761537818186 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:38.186Z [INFO] server-e4e492f1-8d77-3433-eadb-830d58474112: restored user snapshot: index=10013 -2025-10-27T04:03:38.186Z [ERROR] server-e4e492f1-8d77-3433-eadb-830d58474112: failed to get log: index=3 error="log not found" -2025-10-27T04:03:38.186Z [ERROR] server-e4e492f1-8d77-3433-eadb-830d58474112: failed to get log: index=3 error="log not found" -2025-10-27T04:03:38.186Z [INFO] server-e4e492f1-8d77-3433-eadb-830d58474112: aborting pipeline replication: peer="{Voter server-a1338678-1b16-28f6-2942-f73b97d17a4d a1338678-1b16-28f6-2942-f73b97d17a4d}" -2025-10-27T04:03:38.186Z [INFO] server-e4e492f1-8d77-3433-eadb-830d58474112: aborting pipeline replication: peer="{Voter server-def35d83-cd5d-9408-48f3-2161fddf6527 def35d83-cd5d-9408-48f3-2161fddf6527}" -2025-10-27T04:03:38.193Z [ERROR] server-e4e492f1-8d77-3433-eadb-830d58474112: failed to get log: index=3 error="log not found" -2025-10-27T04:03:38.193Z [ERROR] server-e4e492f1-8d77-3433-eadb-830d58474112: failed to get log: index=3 error="log not found" -2025-10-27T04:03:38.193Z [INFO] creating new snapshot: path=/tmp/raft633442481/snapshots/2-10013-1761537818193.tmp -2025-10-27T04:03:38.193Z [INFO] creating new snapshot: path=/tmp/raft1010195175/snapshots/2-10013-1761537818193.tmp -2025-10-27T04:03:38.193Z [INFO] server-a1338678-1b16-28f6-2942-f73b97d17a4d: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:38.193Z [INFO] server-def35d83-cd5d-9408-48f3-2161fddf6527: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:38.193Z [INFO] server-a1338678-1b16-28f6-2942-f73b97d17a4d: copied to local snapshot: bytes=71 -2025-10-27T04:03:38.193Z [INFO] server-def35d83-cd5d-9408-48f3-2161fddf6527: copied to local snapshot: bytes=71 -2025-10-27T04:03:38.194Z [INFO] server-a1338678-1b16-28f6-2942-f73b97d17a4d: snapshot restore progress: id=2-10013-1761537818193 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:38.194Z [INFO] server-a1338678-1b16-28f6-2942-f73b97d17a4d: Installed remote snapshot -2025-10-27T04:03:38.194Z [INFO] server-def35d83-cd5d-9408-48f3-2161fddf6527: snapshot restore progress: id=2-10013-1761537818193 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:38.194Z [INFO] server-def35d83-cd5d-9408-48f3-2161fddf6527: Installed remote snapshot -2025-10-27T04:03:38.194Z [INFO] server-e4e492f1-8d77-3433-eadb-830d58474112: pipelining replication: peer="{Voter server-a1338678-1b16-28f6-2942-f73b97d17a4d a1338678-1b16-28f6-2942-f73b97d17a4d}" -2025-10-27T04:03:38.194Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:03:38.194Z [INFO] server-e4e492f1-8d77-3433-eadb-830d58474112: pipelining replication: peer="{Voter server-def35d83-cd5d-9408-48f3-2161fddf6527 def35d83-cd5d-9408-48f3-2161fddf6527}" -2025-10-27T04:03:39.199Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:03:38.194146664 +0000 UTC m=+93.189019062 (2 nodes), 480ns from start of poll, 1.535789372s from cluster start. Timeout at 2025-10-27 04:03:39.199165225 +0000 UTC m=+94.194037663, 1.005018601s after stability -2025-10-27T04:03:39.212Z [INFO] server-17376f89-1602-df04-2a54-a3561510b029: aborting pipeline replication: peer="{Voter server-1f295cb1-69ae-0b1b-e0d9-e928ada322ca 1f295cb1-69ae-0b1b-e0d9-e928ada322ca}" -2025-10-27T04:03:39.212Z [INFO] server-17376f89-1602-df04-2a54-a3561510b029: aborting pipeline replication: peer="{Voter server-96263d0c-8a48-3791-99df-abbbd25dcac4 96263d0c-8a48-3791-99df-abbbd25dcac4}" +2026-11-29T10:29:37.165Z [INFO] server-eb1d78e7-88a2-4072-10da-14ed710e0aa0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eb1d78e7-88a2-4072-10da-14ed710e0aa0 Address:eb1d78e7-88a2-4072-10da-14ed710e0aa0} {Suffrage:Voter ID:server-ee133ae3-fd63-3462-25a3-c2117e5eb922 Address:ee133ae3-fd63-3462-25a3-c2117e5eb922} {Suffrage:Voter ID:server-1b89377e-b460-c292-80fc-7bd6fa4ad775 Address:1b89377e-b460-c292-80fc-7bd6fa4ad775}]" +2026-11-29T10:29:37.165Z [INFO] server-eb1d78e7-88a2-4072-10da-14ed710e0aa0: entering follower state: follower="Node at eb1d78e7-88a2-4072-10da-14ed710e0aa0 [Follower]" leader-address= leader-id= +2026-11-29T10:29:37.165Z [INFO] server-ee133ae3-fd63-3462-25a3-c2117e5eb922: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eb1d78e7-88a2-4072-10da-14ed710e0aa0 Address:eb1d78e7-88a2-4072-10da-14ed710e0aa0} {Suffrage:Voter ID:server-ee133ae3-fd63-3462-25a3-c2117e5eb922 Address:ee133ae3-fd63-3462-25a3-c2117e5eb922} {Suffrage:Voter ID:server-1b89377e-b460-c292-80fc-7bd6fa4ad775 Address:1b89377e-b460-c292-80fc-7bd6fa4ad775}]" +2026-11-29T10:29:37.165Z [INFO] server-ee133ae3-fd63-3462-25a3-c2117e5eb922: entering follower state: follower="Node at ee133ae3-fd63-3462-25a3-c2117e5eb922 [Follower]" leader-address= leader-id= +2026-11-29T10:29:37.166Z [INFO] server-1b89377e-b460-c292-80fc-7bd6fa4ad775: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eb1d78e7-88a2-4072-10da-14ed710e0aa0 Address:eb1d78e7-88a2-4072-10da-14ed710e0aa0} {Suffrage:Voter ID:server-ee133ae3-fd63-3462-25a3-c2117e5eb922 Address:ee133ae3-fd63-3462-25a3-c2117e5eb922} {Suffrage:Voter ID:server-1b89377e-b460-c292-80fc-7bd6fa4ad775 Address:1b89377e-b460-c292-80fc-7bd6fa4ad775}]" +2026-11-29T10:29:37.166Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:37.166Z [INFO] server-1b89377e-b460-c292-80fc-7bd6fa4ad775: entering follower state: follower="Node at 1b89377e-b460-c292-80fc-7bd6fa4ad775 [Follower]" leader-address= leader-id= +2026-11-29T10:29:37.952Z [WARN] server-ee133ae3-fd63-3462-25a3-c2117e5eb922: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:37.952Z [INFO] server-ee133ae3-fd63-3462-25a3-c2117e5eb922: entering candidate state: node="Node at ee133ae3-fd63-3462-25a3-c2117e5eb922 [Candidate]" term=2 +2026-11-29T10:29:37.953Z [INFO] server-ee133ae3-fd63-3462-25a3-c2117e5eb922: election won: term=2 tally=2 +2026-11-29T10:29:37.953Z [INFO] server-ee133ae3-fd63-3462-25a3-c2117e5eb922: entering leader state: leader="Node at ee133ae3-fd63-3462-25a3-c2117e5eb922 [Leader]" +2026-11-29T10:29:37.953Z [INFO] server-ee133ae3-fd63-3462-25a3-c2117e5eb922: added peer, starting replication: peer=server-eb1d78e7-88a2-4072-10da-14ed710e0aa0 +2026-11-29T10:29:37.953Z [INFO] server-ee133ae3-fd63-3462-25a3-c2117e5eb922: added peer, starting replication: peer=server-1b89377e-b460-c292-80fc-7bd6fa4ad775 +2026-11-29T10:29:37.953Z [INFO] server-ee133ae3-fd63-3462-25a3-c2117e5eb922: pipelining replication: peer="{Voter server-1b89377e-b460-c292-80fc-7bd6fa4ad775 1b89377e-b460-c292-80fc-7bd6fa4ad775}" +2026-11-29T10:29:37.953Z [INFO] server-ee133ae3-fd63-3462-25a3-c2117e5eb922: pipelining replication: peer="{Voter server-eb1d78e7-88a2-4072-10da-14ed710e0aa0 eb1d78e7-88a2-4072-10da-14ed710e0aa0}" +2026-11-29T10:29:38.959Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:37.953201219 +0000 UTC m=+88.688255321 (1 nodes), 787.1286ms from start of poll, 787.73152ms from cluster start. Timeout at 2026-11-29 10:29:38.958201699 +0000 UTC m=+89.693255841, 1.00500052s after stability +2026-11-29T10:29:38.959Z [INFO] server-ee133ae3-fd63-3462-25a3-c2117e5eb922: starting snapshot up to: index=12 +2026-11-29T10:29:38.959Z [INFO] creating new snapshot: path=/tmp/raft3215524510/snapshots/2-12-1795948178959.tmp +2026-11-29T10:29:38.959Z [INFO] server-ee133ae3-fd63-3462-25a3-c2117e5eb922: snapshot complete up to: index=12 +2026-11-29T10:29:38.960Z [INFO] server-ae5ccd21-c172-236d-accd-e1b9e0515b0e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ae5ccd21-c172-236d-accd-e1b9e0515b0e Address:ae5ccd21-c172-236d-accd-e1b9e0515b0e} {Suffrage:Voter ID:server-570810f9-dcd7-1ab1-f22c-23adb1ba0363 Address:570810f9-dcd7-1ab1-f22c-23adb1ba0363} {Suffrage:Voter ID:server-031d70a6-7b7e-13c8-0385-e277ad32dbb9 Address:031d70a6-7b7e-13c8-0385-e277ad32dbb9}]" +2026-11-29T10:29:38.960Z [INFO] server-570810f9-dcd7-1ab1-f22c-23adb1ba0363: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ae5ccd21-c172-236d-accd-e1b9e0515b0e Address:ae5ccd21-c172-236d-accd-e1b9e0515b0e} {Suffrage:Voter ID:server-570810f9-dcd7-1ab1-f22c-23adb1ba0363 Address:570810f9-dcd7-1ab1-f22c-23adb1ba0363} {Suffrage:Voter ID:server-031d70a6-7b7e-13c8-0385-e277ad32dbb9 Address:031d70a6-7b7e-13c8-0385-e277ad32dbb9}]" +2026-11-29T10:29:38.960Z [INFO] server-031d70a6-7b7e-13c8-0385-e277ad32dbb9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ae5ccd21-c172-236d-accd-e1b9e0515b0e Address:ae5ccd21-c172-236d-accd-e1b9e0515b0e} {Suffrage:Voter ID:server-570810f9-dcd7-1ab1-f22c-23adb1ba0363 Address:570810f9-dcd7-1ab1-f22c-23adb1ba0363} {Suffrage:Voter ID:server-031d70a6-7b7e-13c8-0385-e277ad32dbb9 Address:031d70a6-7b7e-13c8-0385-e277ad32dbb9}]" +2026-11-29T10:29:38.960Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:38.960Z [INFO] server-ae5ccd21-c172-236d-accd-e1b9e0515b0e: entering follower state: follower="Node at ae5ccd21-c172-236d-accd-e1b9e0515b0e [Follower]" leader-address= leader-id= +2026-11-29T10:29:38.960Z [INFO] server-570810f9-dcd7-1ab1-f22c-23adb1ba0363: entering follower state: follower="Node at 570810f9-dcd7-1ab1-f22c-23adb1ba0363 [Follower]" leader-address= leader-id= +2026-11-29T10:29:38.960Z [INFO] server-031d70a6-7b7e-13c8-0385-e277ad32dbb9: entering follower state: follower="Node at 031d70a6-7b7e-13c8-0385-e277ad32dbb9 [Follower]" leader-address= leader-id= +2026-11-29T10:29:39.897Z [WARN] server-ae5ccd21-c172-236d-accd-e1b9e0515b0e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:39.897Z [INFO] server-ae5ccd21-c172-236d-accd-e1b9e0515b0e: entering candidate state: node="Node at ae5ccd21-c172-236d-accd-e1b9e0515b0e [Candidate]" term=2 +2026-11-29T10:29:39.897Z [INFO] server-ae5ccd21-c172-236d-accd-e1b9e0515b0e: election won: term=2 tally=2 +2026-11-29T10:29:39.897Z [INFO] server-ae5ccd21-c172-236d-accd-e1b9e0515b0e: entering leader state: leader="Node at ae5ccd21-c172-236d-accd-e1b9e0515b0e [Leader]" +2026-11-29T10:29:39.897Z [INFO] server-ae5ccd21-c172-236d-accd-e1b9e0515b0e: added peer, starting replication: peer=server-570810f9-dcd7-1ab1-f22c-23adb1ba0363 +2026-11-29T10:29:39.897Z [INFO] server-ae5ccd21-c172-236d-accd-e1b9e0515b0e: added peer, starting replication: peer=server-031d70a6-7b7e-13c8-0385-e277ad32dbb9 +2026-11-29T10:29:39.897Z [INFO] server-ae5ccd21-c172-236d-accd-e1b9e0515b0e: pipelining replication: peer="{Voter server-570810f9-dcd7-1ab1-f22c-23adb1ba0363 570810f9-dcd7-1ab1-f22c-23adb1ba0363}" +2026-11-29T10:29:39.897Z [INFO] server-ae5ccd21-c172-236d-accd-e1b9e0515b0e: pipelining replication: peer="{Voter server-031d70a6-7b7e-13c8-0385-e277ad32dbb9 031d70a6-7b7e-13c8-0385-e277ad32dbb9}" +2026-11-29T10:29:40.902Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:39.897805179 +0000 UTC m=+90.632859281 (1 nodes), 937.23056ms from start of poll, 937.44364ms from cluster start. Timeout at 2026-11-29 10:29:40.902808219 +0000 UTC m=+91.637862321, 1.00500304s after stability +2026-11-29T10:29:40.903Z [INFO] creating new snapshot: path=/tmp/raft2912393557/snapshots/2-10013-1795948180903.tmp +2026-11-29T10:29:40.903Z [INFO] server-ae5ccd21-c172-236d-accd-e1b9e0515b0e: copied to local snapshot: bytes=71 +2026-11-29T10:29:40.903Z [INFO] server-ae5ccd21-c172-236d-accd-e1b9e0515b0e: snapshot restore progress: id=2-10013-1795948180903 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:40.903Z [INFO] server-ae5ccd21-c172-236d-accd-e1b9e0515b0e: restored user snapshot: index=10013 +2026-11-29T10:29:40.903Z [ERROR] server-ae5ccd21-c172-236d-accd-e1b9e0515b0e: failed to get log: index=3 error="log not found" +2026-11-29T10:29:40.903Z [INFO] server-ae5ccd21-c172-236d-accd-e1b9e0515b0e: aborting pipeline replication: peer="{Voter server-031d70a6-7b7e-13c8-0385-e277ad32dbb9 031d70a6-7b7e-13c8-0385-e277ad32dbb9}" +2026-11-29T10:29:40.903Z [ERROR] server-ae5ccd21-c172-236d-accd-e1b9e0515b0e: failed to get log: index=3 error="log not found" +2026-11-29T10:29:40.903Z [INFO] server-ae5ccd21-c172-236d-accd-e1b9e0515b0e: aborting pipeline replication: peer="{Voter server-570810f9-dcd7-1ab1-f22c-23adb1ba0363 570810f9-dcd7-1ab1-f22c-23adb1ba0363}" +2026-11-29T10:29:40.912Z [ERROR] server-ae5ccd21-c172-236d-accd-e1b9e0515b0e: failed to get log: index=3 error="log not found" +2026-11-29T10:29:40.912Z [ERROR] server-ae5ccd21-c172-236d-accd-e1b9e0515b0e: failed to get log: index=3 error="log not found" +2026-11-29T10:29:40.912Z [INFO] creating new snapshot: path=/tmp/raft3303166273/snapshots/2-10013-1795948180912.tmp +2026-11-29T10:29:40.912Z [INFO] creating new snapshot: path=/tmp/raft943323753/snapshots/2-10013-1795948180912.tmp +2026-11-29T10:29:40.912Z [INFO] server-031d70a6-7b7e-13c8-0385-e277ad32dbb9: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:40.912Z [INFO] server-031d70a6-7b7e-13c8-0385-e277ad32dbb9: copied to local snapshot: bytes=71 +2026-11-29T10:29:40.912Z [INFO] server-031d70a6-7b7e-13c8-0385-e277ad32dbb9: snapshot restore progress: id=2-10013-1795948180912 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:40.912Z [INFO] server-031d70a6-7b7e-13c8-0385-e277ad32dbb9: Installed remote snapshot +2026-11-29T10:29:40.912Z [INFO] server-ae5ccd21-c172-236d-accd-e1b9e0515b0e: pipelining replication: peer="{Voter server-031d70a6-7b7e-13c8-0385-e277ad32dbb9 031d70a6-7b7e-13c8-0385-e277ad32dbb9}" +2026-11-29T10:29:40.912Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:40.913Z [INFO] server-570810f9-dcd7-1ab1-f22c-23adb1ba0363: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:40.913Z [INFO] server-570810f9-dcd7-1ab1-f22c-23adb1ba0363: copied to local snapshot: bytes=71 +2026-11-29T10:29:40.913Z [INFO] server-570810f9-dcd7-1ab1-f22c-23adb1ba0363: snapshot restore progress: id=2-10013-1795948180912 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:40.913Z [INFO] server-570810f9-dcd7-1ab1-f22c-23adb1ba0363: Installed remote snapshot +2026-11-29T10:29:40.913Z [INFO] server-ae5ccd21-c172-236d-accd-e1b9e0515b0e: pipelining replication: peer="{Voter server-570810f9-dcd7-1ab1-f22c-23adb1ba0363 570810f9-dcd7-1ab1-f22c-23adb1ba0363}" +2026-11-29T10:29:41.918Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:40.913003299 +0000 UTC m=+91.648057401 (2 nodes), 560ns from start of poll, 1.95264176s from cluster start. Timeout at 2026-11-29 10:29:41.918026819 +0000 UTC m=+92.653080921, 1.00502352s after stability +2026-11-29T10:29:41.934Z [INFO] server-ee133ae3-fd63-3462-25a3-c2117e5eb922: aborting pipeline replication: peer="{Voter server-1b89377e-b460-c292-80fc-7bd6fa4ad775 1b89377e-b460-c292-80fc-7bd6fa4ad775}" +2026-11-29T10:29:41.934Z [INFO] server-ee133ae3-fd63-3462-25a3-c2117e5eb922: aborting pipeline replication: peer="{Voter server-eb1d78e7-88a2-4072-10da-14ed710e0aa0 eb1d78e7-88a2-4072-10da-14ed710e0aa0}" === RUN TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_true -2025-10-27T04:03:39.213Z [INFO] server-a0af204c-e56f-b5c1-20a2-aeda81403c7b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a0af204c-e56f-b5c1-20a2-aeda81403c7b Address:a0af204c-e56f-b5c1-20a2-aeda81403c7b} {Suffrage:Voter ID:server-1a746462-7aa2-ee77-0e11-618aa41b8bd4 Address:1a746462-7aa2-ee77-0e11-618aa41b8bd4} {Suffrage:Voter ID:server-39fdbd81-0f3a-ddd5-8ab3-a5f5a58a60ea Address:39fdbd81-0f3a-ddd5-8ab3-a5f5a58a60ea}]" -2025-10-27T04:03:39.213Z [INFO] server-a0af204c-e56f-b5c1-20a2-aeda81403c7b: entering follower state: follower="Node at a0af204c-e56f-b5c1-20a2-aeda81403c7b [Follower]" leader-address= leader-id= -2025-10-27T04:03:39.213Z [INFO] server-1a746462-7aa2-ee77-0e11-618aa41b8bd4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a0af204c-e56f-b5c1-20a2-aeda81403c7b Address:a0af204c-e56f-b5c1-20a2-aeda81403c7b} {Suffrage:Voter ID:server-1a746462-7aa2-ee77-0e11-618aa41b8bd4 Address:1a746462-7aa2-ee77-0e11-618aa41b8bd4} {Suffrage:Voter ID:server-39fdbd81-0f3a-ddd5-8ab3-a5f5a58a60ea Address:39fdbd81-0f3a-ddd5-8ab3-a5f5a58a60ea}]" -2025-10-27T04:03:39.213Z [INFO] server-1a746462-7aa2-ee77-0e11-618aa41b8bd4: entering follower state: follower="Node at 1a746462-7aa2-ee77-0e11-618aa41b8bd4 [Follower]" leader-address= leader-id= -2025-10-27T04:03:39.213Z [INFO] server-39fdbd81-0f3a-ddd5-8ab3-a5f5a58a60ea: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a0af204c-e56f-b5c1-20a2-aeda81403c7b Address:a0af204c-e56f-b5c1-20a2-aeda81403c7b} {Suffrage:Voter ID:server-1a746462-7aa2-ee77-0e11-618aa41b8bd4 Address:1a746462-7aa2-ee77-0e11-618aa41b8bd4} {Suffrage:Voter ID:server-39fdbd81-0f3a-ddd5-8ab3-a5f5a58a60ea Address:39fdbd81-0f3a-ddd5-8ab3-a5f5a58a60ea}]" -2025-10-27T04:03:39.213Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:39.213Z [INFO] server-39fdbd81-0f3a-ddd5-8ab3-a5f5a58a60ea: entering follower state: follower="Node at 39fdbd81-0f3a-ddd5-8ab3-a5f5a58a60ea [Follower]" leader-address= leader-id= -2025-10-27T04:03:39.786Z [WARN] server-1a746462-7aa2-ee77-0e11-618aa41b8bd4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:39.786Z [INFO] server-1a746462-7aa2-ee77-0e11-618aa41b8bd4: entering candidate state: node="Node at 1a746462-7aa2-ee77-0e11-618aa41b8bd4 [Candidate]" term=2 -2025-10-27T04:03:39.786Z [INFO] server-1a746462-7aa2-ee77-0e11-618aa41b8bd4: election won: term=2 tally=2 -2025-10-27T04:03:39.786Z [INFO] server-1a746462-7aa2-ee77-0e11-618aa41b8bd4: entering leader state: leader="Node at 1a746462-7aa2-ee77-0e11-618aa41b8bd4 [Leader]" -2025-10-27T04:03:39.786Z [INFO] server-1a746462-7aa2-ee77-0e11-618aa41b8bd4: added peer, starting replication: peer=server-a0af204c-e56f-b5c1-20a2-aeda81403c7b -2025-10-27T04:03:39.786Z [INFO] server-1a746462-7aa2-ee77-0e11-618aa41b8bd4: added peer, starting replication: peer=server-39fdbd81-0f3a-ddd5-8ab3-a5f5a58a60ea -2025-10-27T04:03:39.786Z [INFO] server-1a746462-7aa2-ee77-0e11-618aa41b8bd4: pipelining replication: peer="{Voter server-39fdbd81-0f3a-ddd5-8ab3-a5f5a58a60ea 39fdbd81-0f3a-ddd5-8ab3-a5f5a58a60ea}" -2025-10-27T04:03:39.786Z [INFO] server-1a746462-7aa2-ee77-0e11-618aa41b8bd4: pipelining replication: peer="{Voter server-a0af204c-e56f-b5c1-20a2-aeda81403c7b a0af204c-e56f-b5c1-20a2-aeda81403c7b}" -2025-10-27T04:03:40.792Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:39.786735625 +0000 UTC m=+94.781608103 (1 nodes), 573.233317ms from start of poll, 573.7244ms from cluster start. Timeout at 2025-10-27 04:03:40.791740225 +0000 UTC m=+95.786612703, 1.0050046s after stability -2025-10-27T04:03:40.792Z [INFO] server-1a746462-7aa2-ee77-0e11-618aa41b8bd4: starting snapshot up to: index=12 -2025-10-27T04:03:40.792Z [INFO] creating new snapshot: path=/tmp/raft1545456691/snapshots/2-12-1761537820792.tmp -2025-10-27T04:03:40.793Z [INFO] server-1a746462-7aa2-ee77-0e11-618aa41b8bd4: snapshot complete up to: index=12 -2025-10-27T04:03:40.793Z [INFO] server-8bb1c1e7-ca9a-6253-4b8b-0fdf4301229a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8bb1c1e7-ca9a-6253-4b8b-0fdf4301229a Address:8bb1c1e7-ca9a-6253-4b8b-0fdf4301229a} {Suffrage:Voter ID:server-4e515b3b-c1ae-34a0-8b20-2e0af5202f64 Address:4e515b3b-c1ae-34a0-8b20-2e0af5202f64} {Suffrage:Voter ID:server-b736051d-ea30-76b8-a947-efee9ee1fc7c Address:b736051d-ea30-76b8-a947-efee9ee1fc7c}]" -2025-10-27T04:03:40.793Z [INFO] server-4e515b3b-c1ae-34a0-8b20-2e0af5202f64: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8bb1c1e7-ca9a-6253-4b8b-0fdf4301229a Address:8bb1c1e7-ca9a-6253-4b8b-0fdf4301229a} {Suffrage:Voter ID:server-4e515b3b-c1ae-34a0-8b20-2e0af5202f64 Address:4e515b3b-c1ae-34a0-8b20-2e0af5202f64} {Suffrage:Voter ID:server-b736051d-ea30-76b8-a947-efee9ee1fc7c Address:b736051d-ea30-76b8-a947-efee9ee1fc7c}]" -2025-10-27T04:03:40.793Z [INFO] server-b736051d-ea30-76b8-a947-efee9ee1fc7c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8bb1c1e7-ca9a-6253-4b8b-0fdf4301229a Address:8bb1c1e7-ca9a-6253-4b8b-0fdf4301229a} {Suffrage:Voter ID:server-4e515b3b-c1ae-34a0-8b20-2e0af5202f64 Address:4e515b3b-c1ae-34a0-8b20-2e0af5202f64} {Suffrage:Voter ID:server-b736051d-ea30-76b8-a947-efee9ee1fc7c Address:b736051d-ea30-76b8-a947-efee9ee1fc7c}]" -2025-10-27T04:03:40.793Z [INFO] server-8bb1c1e7-ca9a-6253-4b8b-0fdf4301229a: entering follower state: follower="Node at 8bb1c1e7-ca9a-6253-4b8b-0fdf4301229a [Follower]" leader-address= leader-id= -2025-10-27T04:03:40.793Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:40.793Z [INFO] server-4e515b3b-c1ae-34a0-8b20-2e0af5202f64: entering follower state: follower="Node at 4e515b3b-c1ae-34a0-8b20-2e0af5202f64 [Follower]" leader-address= leader-id= -2025-10-27T04:03:40.793Z [INFO] server-b736051d-ea30-76b8-a947-efee9ee1fc7c: entering follower state: follower="Node at b736051d-ea30-76b8-a947-efee9ee1fc7c [Follower]" leader-address= leader-id= -2025-10-27T04:03:41.295Z [WARN] server-4e515b3b-c1ae-34a0-8b20-2e0af5202f64: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:41.295Z [INFO] server-4e515b3b-c1ae-34a0-8b20-2e0af5202f64: entering candidate state: node="Node at 4e515b3b-c1ae-34a0-8b20-2e0af5202f64 [Candidate]" term=2 -2025-10-27T04:03:41.295Z [INFO] server-4e515b3b-c1ae-34a0-8b20-2e0af5202f64: election won: term=2 tally=2 -2025-10-27T04:03:41.295Z [INFO] server-4e515b3b-c1ae-34a0-8b20-2e0af5202f64: entering leader state: leader="Node at 4e515b3b-c1ae-34a0-8b20-2e0af5202f64 [Leader]" -2025-10-27T04:03:41.295Z [INFO] server-4e515b3b-c1ae-34a0-8b20-2e0af5202f64: added peer, starting replication: peer=server-8bb1c1e7-ca9a-6253-4b8b-0fdf4301229a -2025-10-27T04:03:41.295Z [INFO] server-4e515b3b-c1ae-34a0-8b20-2e0af5202f64: added peer, starting replication: peer=server-b736051d-ea30-76b8-a947-efee9ee1fc7c -2025-10-27T04:03:41.295Z [INFO] server-4e515b3b-c1ae-34a0-8b20-2e0af5202f64: pipelining replication: peer="{Voter server-b736051d-ea30-76b8-a947-efee9ee1fc7c b736051d-ea30-76b8-a947-efee9ee1fc7c}" -2025-10-27T04:03:41.295Z [INFO] server-4e515b3b-c1ae-34a0-8b20-2e0af5202f64: pipelining replication: peer="{Voter server-8bb1c1e7-ca9a-6253-4b8b-0fdf4301229a 8bb1c1e7-ca9a-6253-4b8b-0fdf4301229a}" -2025-10-27T04:03:42.301Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:41.295607853 +0000 UTC m=+96.290480291 (1 nodes), 501.738348ms from start of poll, 501.977629ms from cluster start. Timeout at 2025-10-27 04:03:42.300612573 +0000 UTC m=+97.295485051, 1.00500476s after stability -2025-10-27T04:03:42.301Z [INFO] creating new snapshot: path=/tmp/raft3323179323/snapshots/2-10013-1761537822301.tmp -2025-10-27T04:03:42.301Z [INFO] server-4e515b3b-c1ae-34a0-8b20-2e0af5202f64: copied to local snapshot: bytes=71 -2025-10-27T04:03:42.302Z [INFO] server-4e515b3b-c1ae-34a0-8b20-2e0af5202f64: snapshot restore progress: id=2-10013-1761537822301 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:42.302Z [INFO] server-4e515b3b-c1ae-34a0-8b20-2e0af5202f64: removing all old logs from log store -2025-10-27T04:03:42.302Z [INFO] server-4e515b3b-c1ae-34a0-8b20-2e0af5202f64: compacting logs: from=1 to=2 -2025-10-27T04:03:42.302Z [INFO] server-4e515b3b-c1ae-34a0-8b20-2e0af5202f64: restored user snapshot: index=10013 -2025-10-27T04:03:42.302Z [ERROR] server-4e515b3b-c1ae-34a0-8b20-2e0af5202f64: failed to get log: index=2 error="log not found" -2025-10-27T04:03:42.302Z [INFO] server-4e515b3b-c1ae-34a0-8b20-2e0af5202f64: aborting pipeline replication: peer="{Voter server-b736051d-ea30-76b8-a947-efee9ee1fc7c b736051d-ea30-76b8-a947-efee9ee1fc7c}" -2025-10-27T04:03:42.302Z [ERROR] server-4e515b3b-c1ae-34a0-8b20-2e0af5202f64: failed to get log: index=2 error="log not found" -2025-10-27T04:03:42.302Z [INFO] server-4e515b3b-c1ae-34a0-8b20-2e0af5202f64: aborting pipeline replication: peer="{Voter server-8bb1c1e7-ca9a-6253-4b8b-0fdf4301229a 8bb1c1e7-ca9a-6253-4b8b-0fdf4301229a}" -2025-10-27T04:03:42.308Z [ERROR] server-4e515b3b-c1ae-34a0-8b20-2e0af5202f64: failed to get log: index=2 error="log not found" -2025-10-27T04:03:42.308Z [INFO] creating new snapshot: path=/tmp/raft3443536488/snapshots/2-10013-1761537822308.tmp -2025-10-27T04:03:42.308Z [INFO] server-b736051d-ea30-76b8-a947-efee9ee1fc7c: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:42.308Z [INFO] server-b736051d-ea30-76b8-a947-efee9ee1fc7c: copied to local snapshot: bytes=71 -2025-10-27T04:03:42.308Z [INFO] server-b736051d-ea30-76b8-a947-efee9ee1fc7c: snapshot restore progress: id=2-10013-1761537822308 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:42.308Z [INFO] server-b736051d-ea30-76b8-a947-efee9ee1fc7c: removing all old logs from log store -2025-10-27T04:03:42.308Z [INFO] server-b736051d-ea30-76b8-a947-efee9ee1fc7c: compacting logs: from=1 to=2 -2025-10-27T04:03:42.308Z [INFO] server-b736051d-ea30-76b8-a947-efee9ee1fc7c: Installed remote snapshot -2025-10-27T04:03:42.308Z [INFO] server-4e515b3b-c1ae-34a0-8b20-2e0af5202f64: pipelining replication: peer="{Voter server-b736051d-ea30-76b8-a947-efee9ee1fc7c b736051d-ea30-76b8-a947-efee9ee1fc7c}" -2025-10-27T04:03:42.308Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:03:42.310Z [ERROR] server-4e515b3b-c1ae-34a0-8b20-2e0af5202f64: failed to get log: index=2 error="log not found" -2025-10-27T04:03:42.310Z [INFO] creating new snapshot: path=/tmp/raft1627453452/snapshots/2-10013-1761537822310.tmp -2025-10-27T04:03:42.311Z [INFO] server-8bb1c1e7-ca9a-6253-4b8b-0fdf4301229a: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:42.311Z [INFO] server-8bb1c1e7-ca9a-6253-4b8b-0fdf4301229a: copied to local snapshot: bytes=71 -2025-10-27T04:03:42.311Z [INFO] server-8bb1c1e7-ca9a-6253-4b8b-0fdf4301229a: snapshot restore progress: id=2-10013-1761537822310 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% -2025-10-27T04:03:42.311Z [INFO] server-8bb1c1e7-ca9a-6253-4b8b-0fdf4301229a: removing all old logs from log store -2025-10-27T04:03:42.311Z [INFO] server-8bb1c1e7-ca9a-6253-4b8b-0fdf4301229a: compacting logs: from=1 to=2 -2025-10-27T04:03:42.311Z [INFO] server-8bb1c1e7-ca9a-6253-4b8b-0fdf4301229a: Installed remote snapshot -2025-10-27T04:03:42.312Z [INFO] server-4e515b3b-c1ae-34a0-8b20-2e0af5202f64: pipelining replication: peer="{Voter server-8bb1c1e7-ca9a-6253-4b8b-0fdf4301229a 8bb1c1e7-ca9a-6253-4b8b-0fdf4301229a}" -2025-10-27T04:03:43.314Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:03:42.308863736 +0000 UTC m=+97.303736174 (2 nodes), 760ns from start of poll, 1.515233512s from cluster start. Timeout at 2025-10-27 04:03:43.313879936 +0000 UTC m=+98.308752374, 1.0050162s after stability -2025-10-27T04:03:43.327Z [INFO] server-1a746462-7aa2-ee77-0e11-618aa41b8bd4: aborting pipeline replication: peer="{Voter server-39fdbd81-0f3a-ddd5-8ab3-a5f5a58a60ea 39fdbd81-0f3a-ddd5-8ab3-a5f5a58a60ea}" -2025-10-27T04:03:43.327Z [INFO] server-1a746462-7aa2-ee77-0e11-618aa41b8bd4: aborting pipeline replication: peer="{Voter server-a0af204c-e56f-b5c1-20a2-aeda81403c7b a0af204c-e56f-b5c1-20a2-aeda81403c7b}" ---- PASS: TestRaft_UserRestore (84.23s) - --- PASS: TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_false (2.57s) - --- PASS: TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_false (2.79s) - --- PASS: TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_true (4.11s) - --- PASS: TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_true (4.18s) - --- PASS: TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_false (2.66s) - --- PASS: TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_false (2.69s) - --- PASS: TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_true (4.13s) - --- PASS: TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_true (4.18s) - --- PASS: TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_false (2.60s) - --- PASS: TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_false (2.65s) - --- PASS: TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_true (4.84s) - --- PASS: TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_true (4.46s) - --- PASS: TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_false (2.83s) - --- PASS: TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_false (2.62s) - --- PASS: TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_true (4.67s) - --- PASS: TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_true (4.57s) - --- PASS: TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_false (2.56s) - --- PASS: TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_false (2.73s) - --- PASS: TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_true (4.36s) - --- PASS: TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_true (4.40s) - --- PASS: TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_false (2.55s) - --- PASS: TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_false (2.58s) - --- PASS: TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_true (4.39s) - --- PASS: TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_true (4.11s) +2026-11-29T10:29:41.934Z [INFO] server-b6f85d8b-9f3b-e391-a093-64d880edad87: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b6f85d8b-9f3b-e391-a093-64d880edad87 Address:b6f85d8b-9f3b-e391-a093-64d880edad87} {Suffrage:Voter ID:server-d17be42c-9df6-979e-9a1c-193ac2a4b3d5 Address:d17be42c-9df6-979e-9a1c-193ac2a4b3d5} {Suffrage:Voter ID:server-2b6471c0-5f18-c838-cc7f-461315abd2a7 Address:2b6471c0-5f18-c838-cc7f-461315abd2a7}]" +2026-11-29T10:29:41.935Z [INFO] server-d17be42c-9df6-979e-9a1c-193ac2a4b3d5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b6f85d8b-9f3b-e391-a093-64d880edad87 Address:b6f85d8b-9f3b-e391-a093-64d880edad87} {Suffrage:Voter ID:server-d17be42c-9df6-979e-9a1c-193ac2a4b3d5 Address:d17be42c-9df6-979e-9a1c-193ac2a4b3d5} {Suffrage:Voter ID:server-2b6471c0-5f18-c838-cc7f-461315abd2a7 Address:2b6471c0-5f18-c838-cc7f-461315abd2a7}]" +2026-11-29T10:29:41.935Z [INFO] server-2b6471c0-5f18-c838-cc7f-461315abd2a7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b6f85d8b-9f3b-e391-a093-64d880edad87 Address:b6f85d8b-9f3b-e391-a093-64d880edad87} {Suffrage:Voter ID:server-d17be42c-9df6-979e-9a1c-193ac2a4b3d5 Address:d17be42c-9df6-979e-9a1c-193ac2a4b3d5} {Suffrage:Voter ID:server-2b6471c0-5f18-c838-cc7f-461315abd2a7 Address:2b6471c0-5f18-c838-cc7f-461315abd2a7}]" +2026-11-29T10:29:41.935Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:41.935Z [INFO] server-d17be42c-9df6-979e-9a1c-193ac2a4b3d5: entering follower state: follower="Node at d17be42c-9df6-979e-9a1c-193ac2a4b3d5 [Follower]" leader-address= leader-id= +2026-11-29T10:29:41.935Z [INFO] server-2b6471c0-5f18-c838-cc7f-461315abd2a7: entering follower state: follower="Node at 2b6471c0-5f18-c838-cc7f-461315abd2a7 [Follower]" leader-address= leader-id= +2026-11-29T10:29:41.935Z [INFO] server-b6f85d8b-9f3b-e391-a093-64d880edad87: entering follower state: follower="Node at b6f85d8b-9f3b-e391-a093-64d880edad87 [Follower]" leader-address= leader-id= +2026-11-29T10:29:42.699Z [WARN] server-d17be42c-9df6-979e-9a1c-193ac2a4b3d5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:42.699Z [INFO] server-d17be42c-9df6-979e-9a1c-193ac2a4b3d5: entering candidate state: node="Node at d17be42c-9df6-979e-9a1c-193ac2a4b3d5 [Candidate]" term=2 +2026-11-29T10:29:42.699Z [INFO] server-d17be42c-9df6-979e-9a1c-193ac2a4b3d5: election won: term=2 tally=2 +2026-11-29T10:29:42.699Z [INFO] server-d17be42c-9df6-979e-9a1c-193ac2a4b3d5: entering leader state: leader="Node at d17be42c-9df6-979e-9a1c-193ac2a4b3d5 [Leader]" +2026-11-29T10:29:42.699Z [INFO] server-d17be42c-9df6-979e-9a1c-193ac2a4b3d5: added peer, starting replication: peer=server-b6f85d8b-9f3b-e391-a093-64d880edad87 +2026-11-29T10:29:42.699Z [INFO] server-d17be42c-9df6-979e-9a1c-193ac2a4b3d5: added peer, starting replication: peer=server-2b6471c0-5f18-c838-cc7f-461315abd2a7 +2026-11-29T10:29:42.699Z [INFO] server-d17be42c-9df6-979e-9a1c-193ac2a4b3d5: pipelining replication: peer="{Voter server-2b6471c0-5f18-c838-cc7f-461315abd2a7 2b6471c0-5f18-c838-cc7f-461315abd2a7}" +2026-11-29T10:29:42.699Z [INFO] server-d17be42c-9df6-979e-9a1c-193ac2a4b3d5: pipelining replication: peer="{Voter server-b6f85d8b-9f3b-e391-a093-64d880edad87 b6f85d8b-9f3b-e391-a093-64d880edad87}" +2026-11-29T10:29:43.704Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:42.699448899 +0000 UTC m=+93.434503041 (1 nodes), 764.10956ms from start of poll, 764.59128ms from cluster start. Timeout at 2026-11-29 10:29:43.704452099 +0000 UTC m=+94.439506241, 1.0050032s after stability +2026-11-29T10:29:43.705Z [INFO] server-d17be42c-9df6-979e-9a1c-193ac2a4b3d5: starting snapshot up to: index=12 +2026-11-29T10:29:43.705Z [INFO] creating new snapshot: path=/tmp/raft3797497067/snapshots/2-12-1795948183705.tmp +2026-11-29T10:29:43.705Z [INFO] server-d17be42c-9df6-979e-9a1c-193ac2a4b3d5: snapshot complete up to: index=12 +2026-11-29T10:29:43.706Z [INFO] server-35f8dab0-3f9d-b157-377b-b8c1556aa2ee: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-35f8dab0-3f9d-b157-377b-b8c1556aa2ee Address:35f8dab0-3f9d-b157-377b-b8c1556aa2ee} {Suffrage:Voter ID:server-8135eb93-4b25-6ca1-b484-f6a5e568a0c6 Address:8135eb93-4b25-6ca1-b484-f6a5e568a0c6} {Suffrage:Voter ID:server-ea6bc646-ad1f-c8c5-915b-7923010af6af Address:ea6bc646-ad1f-c8c5-915b-7923010af6af}]" +2026-11-29T10:29:43.706Z [INFO] server-8135eb93-4b25-6ca1-b484-f6a5e568a0c6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-35f8dab0-3f9d-b157-377b-b8c1556aa2ee Address:35f8dab0-3f9d-b157-377b-b8c1556aa2ee} {Suffrage:Voter ID:server-8135eb93-4b25-6ca1-b484-f6a5e568a0c6 Address:8135eb93-4b25-6ca1-b484-f6a5e568a0c6} {Suffrage:Voter ID:server-ea6bc646-ad1f-c8c5-915b-7923010af6af Address:ea6bc646-ad1f-c8c5-915b-7923010af6af}]" +2026-11-29T10:29:43.706Z [INFO] server-ea6bc646-ad1f-c8c5-915b-7923010af6af: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-35f8dab0-3f9d-b157-377b-b8c1556aa2ee Address:35f8dab0-3f9d-b157-377b-b8c1556aa2ee} {Suffrage:Voter ID:server-8135eb93-4b25-6ca1-b484-f6a5e568a0c6 Address:8135eb93-4b25-6ca1-b484-f6a5e568a0c6} {Suffrage:Voter ID:server-ea6bc646-ad1f-c8c5-915b-7923010af6af Address:ea6bc646-ad1f-c8c5-915b-7923010af6af}]" +2026-11-29T10:29:43.706Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:43.706Z [INFO] server-35f8dab0-3f9d-b157-377b-b8c1556aa2ee: entering follower state: follower="Node at 35f8dab0-3f9d-b157-377b-b8c1556aa2ee [Follower]" leader-address= leader-id= +2026-11-29T10:29:43.706Z [INFO] server-ea6bc646-ad1f-c8c5-915b-7923010af6af: entering follower state: follower="Node at ea6bc646-ad1f-c8c5-915b-7923010af6af [Follower]" leader-address= leader-id= +2026-11-29T10:29:43.706Z [INFO] server-8135eb93-4b25-6ca1-b484-f6a5e568a0c6: entering follower state: follower="Node at 8135eb93-4b25-6ca1-b484-f6a5e568a0c6 [Follower]" leader-address= leader-id= +2026-11-29T10:29:44.214Z [WARN] server-35f8dab0-3f9d-b157-377b-b8c1556aa2ee: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:44.214Z [INFO] server-35f8dab0-3f9d-b157-377b-b8c1556aa2ee: entering candidate state: node="Node at 35f8dab0-3f9d-b157-377b-b8c1556aa2ee [Candidate]" term=2 +2026-11-29T10:29:44.214Z [INFO] server-35f8dab0-3f9d-b157-377b-b8c1556aa2ee: election won: term=2 tally=2 +2026-11-29T10:29:44.214Z [INFO] server-35f8dab0-3f9d-b157-377b-b8c1556aa2ee: entering leader state: leader="Node at 35f8dab0-3f9d-b157-377b-b8c1556aa2ee [Leader]" +2026-11-29T10:29:44.214Z [INFO] server-35f8dab0-3f9d-b157-377b-b8c1556aa2ee: added peer, starting replication: peer=server-8135eb93-4b25-6ca1-b484-f6a5e568a0c6 +2026-11-29T10:29:44.214Z [INFO] server-35f8dab0-3f9d-b157-377b-b8c1556aa2ee: added peer, starting replication: peer=server-ea6bc646-ad1f-c8c5-915b-7923010af6af +2026-11-29T10:29:44.214Z [INFO] server-35f8dab0-3f9d-b157-377b-b8c1556aa2ee: pipelining replication: peer="{Voter server-8135eb93-4b25-6ca1-b484-f6a5e568a0c6 8135eb93-4b25-6ca1-b484-f6a5e568a0c6}" +2026-11-29T10:29:44.214Z [INFO] server-35f8dab0-3f9d-b157-377b-b8c1556aa2ee: pipelining replication: peer="{Voter server-ea6bc646-ad1f-c8c5-915b-7923010af6af ea6bc646-ad1f-c8c5-915b-7923010af6af}" +2026-11-29T10:29:45.220Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:44.214878899 +0000 UTC m=+94.949933001 (1 nodes), 508.62908ms from start of poll, 508.93832ms from cluster start. Timeout at 2026-11-29 10:29:45.219884619 +0000 UTC m=+95.954938721, 1.00500572s after stability +2026-11-29T10:29:45.220Z [INFO] creating new snapshot: path=/tmp/raft2521881021/snapshots/2-10013-1795948185220.tmp +2026-11-29T10:29:45.220Z [INFO] server-35f8dab0-3f9d-b157-377b-b8c1556aa2ee: copied to local snapshot: bytes=71 +2026-11-29T10:29:45.221Z [INFO] server-35f8dab0-3f9d-b157-377b-b8c1556aa2ee: snapshot restore progress: id=2-10013-1795948185220 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:45.221Z [INFO] server-35f8dab0-3f9d-b157-377b-b8c1556aa2ee: removing all old logs from log store +2026-11-29T10:29:45.221Z [INFO] server-35f8dab0-3f9d-b157-377b-b8c1556aa2ee: compacting logs: from=1 to=2 +2026-11-29T10:29:45.221Z [INFO] server-35f8dab0-3f9d-b157-377b-b8c1556aa2ee: restored user snapshot: index=10013 +2026-11-29T10:29:45.221Z [ERROR] server-35f8dab0-3f9d-b157-377b-b8c1556aa2ee: failed to get log: index=2 error="log not found" +2026-11-29T10:29:45.221Z [INFO] server-35f8dab0-3f9d-b157-377b-b8c1556aa2ee: aborting pipeline replication: peer="{Voter server-ea6bc646-ad1f-c8c5-915b-7923010af6af ea6bc646-ad1f-c8c5-915b-7923010af6af}" +2026-11-29T10:29:45.221Z [ERROR] server-35f8dab0-3f9d-b157-377b-b8c1556aa2ee: failed to get log: index=2 error="log not found" +2026-11-29T10:29:45.221Z [INFO] server-35f8dab0-3f9d-b157-377b-b8c1556aa2ee: aborting pipeline replication: peer="{Voter server-8135eb93-4b25-6ca1-b484-f6a5e568a0c6 8135eb93-4b25-6ca1-b484-f6a5e568a0c6}" +2026-11-29T10:29:45.227Z [ERROR] server-35f8dab0-3f9d-b157-377b-b8c1556aa2ee: failed to get log: index=2 error="log not found" +2026-11-29T10:29:45.227Z [INFO] creating new snapshot: path=/tmp/raft2703170826/snapshots/2-10013-1795948185227.tmp +2026-11-29T10:29:45.227Z [INFO] server-8135eb93-4b25-6ca1-b484-f6a5e568a0c6: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:45.228Z [ERROR] server-35f8dab0-3f9d-b157-377b-b8c1556aa2ee: failed to get log: index=2 error="log not found" +2026-11-29T10:29:45.228Z [INFO] server-8135eb93-4b25-6ca1-b484-f6a5e568a0c6: copied to local snapshot: bytes=71 +2026-11-29T10:29:45.228Z [INFO] server-8135eb93-4b25-6ca1-b484-f6a5e568a0c6: snapshot restore progress: id=2-10013-1795948185227 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:45.228Z [INFO] server-8135eb93-4b25-6ca1-b484-f6a5e568a0c6: removing all old logs from log store +2026-11-29T10:29:45.228Z [INFO] server-8135eb93-4b25-6ca1-b484-f6a5e568a0c6: compacting logs: from=1 to=2 +2026-11-29T10:29:45.228Z [INFO] server-8135eb93-4b25-6ca1-b484-f6a5e568a0c6: Installed remote snapshot +2026-11-29T10:29:45.228Z [INFO] creating new snapshot: path=/tmp/raft2579516425/snapshots/2-10013-1795948185228.tmp +2026-11-29T10:29:45.228Z [INFO] server-35f8dab0-3f9d-b157-377b-b8c1556aa2ee: pipelining replication: peer="{Voter server-8135eb93-4b25-6ca1-b484-f6a5e568a0c6 8135eb93-4b25-6ca1-b484-f6a5e568a0c6}" +2026-11-29T10:29:45.228Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:45.228Z [INFO] server-ea6bc646-ad1f-c8c5-915b-7923010af6af: snapshot network transfer progress: read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:45.228Z [INFO] server-ea6bc646-ad1f-c8c5-915b-7923010af6af: copied to local snapshot: bytes=71 +2026-11-29T10:29:45.228Z [INFO] server-ea6bc646-ad1f-c8c5-915b-7923010af6af: snapshot restore progress: id=2-10013-1795948185228 last-index=10013 last-term=2 size-in-bytes=71 read-bytes=71 percent-complete=100.00% +2026-11-29T10:29:45.228Z [INFO] server-ea6bc646-ad1f-c8c5-915b-7923010af6af: removing all old logs from log store +2026-11-29T10:29:45.228Z [INFO] server-ea6bc646-ad1f-c8c5-915b-7923010af6af: compacting logs: from=1 to=2 +2026-11-29T10:29:45.228Z [INFO] server-ea6bc646-ad1f-c8c5-915b-7923010af6af: Installed remote snapshot +2026-11-29T10:29:45.228Z [INFO] server-35f8dab0-3f9d-b157-377b-b8c1556aa2ee: pipelining replication: peer="{Voter server-ea6bc646-ad1f-c8c5-915b-7923010af6af ea6bc646-ad1f-c8c5-915b-7923010af6af}" +2026-11-29T10:29:46.233Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:45.228339499 +0000 UTC m=+95.963393601 (2 nodes), 600ns from start of poll, 1.52239892s from cluster start. Timeout at 2026-11-29 10:29:46.233359539 +0000 UTC m=+96.968413681, 1.00502008s after stability +2026-11-29T10:29:46.244Z [INFO] server-d17be42c-9df6-979e-9a1c-193ac2a4b3d5: aborting pipeline replication: peer="{Voter server-2b6471c0-5f18-c838-cc7f-461315abd2a7 2b6471c0-5f18-c838-cc7f-461315abd2a7}" +2026-11-29T10:29:46.244Z [INFO] server-d17be42c-9df6-979e-9a1c-193ac2a4b3d5: aborting pipeline replication: peer="{Voter server-b6f85d8b-9f3b-e391-a093-64d880edad87 b6f85d8b-9f3b-e391-a093-64d880edad87}" +--- PASS: TestRaft_UserRestore (84.03s) + --- PASS: TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_false (2.70s) + --- PASS: TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_false (2.63s) + --- PASS: TestRaft_UserRestore/case_0_|_restored_to_new_cluster:_true (4.23s) + --- PASS: TestRaft_UserRestore/monotonic_case_0_|_restored_to_new_cluster:_true (4.30s) + --- PASS: TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_false (2.63s) + --- PASS: TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_false (2.77s) + --- PASS: TestRaft_UserRestore/case_1_|_restored_to_new_cluster:_true (4.20s) + --- PASS: TestRaft_UserRestore/monotonic_case_1_|_restored_to_new_cluster:_true (4.53s) + --- PASS: TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_false (2.54s) + --- PASS: TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_false (2.72s) + --- PASS: TestRaft_UserRestore/case_2_|_restored_to_new_cluster:_true (4.16s) + --- PASS: TestRaft_UserRestore/monotonic_case_2_|_restored_to_new_cluster:_true (4.16s) + --- PASS: TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_false (2.80s) + --- PASS: TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_false (3.04s) + --- PASS: TestRaft_UserRestore/case_100_|_restored_to_new_cluster:_true (4.16s) + --- PASS: TestRaft_UserRestore/monotonic_case_100_|_restored_to_new_cluster:_true (4.25s) + --- PASS: TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_false (2.54s) + --- PASS: TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_false (2.61s) + --- PASS: TestRaft_UserRestore/case_1000_|_restored_to_new_cluster:_true (4.28s) + --- PASS: TestRaft_UserRestore/monotonic_case_1000_|_restored_to_new_cluster:_true (4.39s) + --- PASS: TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_false (2.77s) + --- PASS: TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_false (2.55s) + --- PASS: TestRaft_UserRestore/case_10000_|_restored_to_new_cluster:_true (4.77s) + --- PASS: TestRaft_UserRestore/monotonic_case_10000_|_restored_to_new_cluster:_true (4.31s) === RUN TestRaft_SendSnapshotFollower -2025-10-27T04:03:43.327Z [INFO] server-cdfeea54-a6b3-9992-df12-13c6d1b6c419: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cdfeea54-a6b3-9992-df12-13c6d1b6c419 Address:cdfeea54-a6b3-9992-df12-13c6d1b6c419} {Suffrage:Voter ID:server-8a95685e-5192-ce55-868e-2900974dd2aa Address:8a95685e-5192-ce55-868e-2900974dd2aa} {Suffrage:Voter ID:server-f79ae124-4251-0308-4d6b-2a44610bc0cf Address:f79ae124-4251-0308-4d6b-2a44610bc0cf}]" -2025-10-27T04:03:43.327Z [INFO] server-8a95685e-5192-ce55-868e-2900974dd2aa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cdfeea54-a6b3-9992-df12-13c6d1b6c419 Address:cdfeea54-a6b3-9992-df12-13c6d1b6c419} {Suffrage:Voter ID:server-8a95685e-5192-ce55-868e-2900974dd2aa Address:8a95685e-5192-ce55-868e-2900974dd2aa} {Suffrage:Voter ID:server-f79ae124-4251-0308-4d6b-2a44610bc0cf Address:f79ae124-4251-0308-4d6b-2a44610bc0cf}]" -2025-10-27T04:03:43.327Z [INFO] server-cdfeea54-a6b3-9992-df12-13c6d1b6c419: entering follower state: follower="Node at cdfeea54-a6b3-9992-df12-13c6d1b6c419 [Follower]" leader-address= leader-id= -2025-10-27T04:03:43.327Z [INFO] server-f79ae124-4251-0308-4d6b-2a44610bc0cf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cdfeea54-a6b3-9992-df12-13c6d1b6c419 Address:cdfeea54-a6b3-9992-df12-13c6d1b6c419} {Suffrage:Voter ID:server-8a95685e-5192-ce55-868e-2900974dd2aa Address:8a95685e-5192-ce55-868e-2900974dd2aa} {Suffrage:Voter ID:server-f79ae124-4251-0308-4d6b-2a44610bc0cf Address:f79ae124-4251-0308-4d6b-2a44610bc0cf}]" -2025-10-27T04:03:43.327Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:03:43.327Z [INFO] server-8a95685e-5192-ce55-868e-2900974dd2aa: entering follower state: follower="Node at 8a95685e-5192-ce55-868e-2900974dd2aa [Follower]" leader-address= leader-id= -2025-10-27T04:03:43.327Z [INFO] server-f79ae124-4251-0308-4d6b-2a44610bc0cf: entering follower state: follower="Node at f79ae124-4251-0308-4d6b-2a44610bc0cf [Follower]" leader-address= leader-id= -2025-10-27T04:03:43.396Z [WARN] server-f79ae124-4251-0308-4d6b-2a44610bc0cf: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:43.396Z [INFO] server-f79ae124-4251-0308-4d6b-2a44610bc0cf: entering candidate state: node="Node at f79ae124-4251-0308-4d6b-2a44610bc0cf [Candidate]" term=2 -2025-10-27T04:03:43.396Z [INFO] server-f79ae124-4251-0308-4d6b-2a44610bc0cf: election won: term=2 tally=2 -2025-10-27T04:03:43.396Z [INFO] server-f79ae124-4251-0308-4d6b-2a44610bc0cf: entering leader state: leader="Node at f79ae124-4251-0308-4d6b-2a44610bc0cf [Leader]" -2025-10-27T04:03:43.396Z [INFO] server-f79ae124-4251-0308-4d6b-2a44610bc0cf: added peer, starting replication: peer=server-cdfeea54-a6b3-9992-df12-13c6d1b6c419 -2025-10-27T04:03:43.396Z [INFO] server-f79ae124-4251-0308-4d6b-2a44610bc0cf: added peer, starting replication: peer=server-8a95685e-5192-ce55-868e-2900974dd2aa -2025-10-27T04:03:43.396Z [INFO] server-f79ae124-4251-0308-4d6b-2a44610bc0cf: pipelining replication: peer="{Voter server-8a95685e-5192-ce55-868e-2900974dd2aa 8a95685e-5192-ce55-868e-2900974dd2aa}" -2025-10-27T04:03:43.396Z [INFO] server-f79ae124-4251-0308-4d6b-2a44610bc0cf: pipelining replication: peer="{Voter server-cdfeea54-a6b3-9992-df12-13c6d1b6c419 cdfeea54-a6b3-9992-df12-13c6d1b6c419}" -2025-10-27T04:03:43.501Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:03:43.396790688 +0000 UTC m=+98.391663126 (2 nodes), 68.939031ms from start of poll, 69.131552ms from cluster start. Timeout at 2025-10-27 04:03:43.501791768 +0000 UTC m=+98.496664246, 105.00112ms after stability -2025-10-27T04:03:43.501Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:43.606Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:43.501910286 +0000 UTC m=+98.496782764 (1 nodes), 2.08µs from start of poll, 174.25119ms from cluster start. Timeout at 2025-10-27 04:03:43.606949846 +0000 UTC m=+98.601822364, 105.0396ms after stability -2025-10-27T04:03:43.607Z [ERROR] server-f79ae124-4251-0308-4d6b-2a44610bc0cf: failed to pipeline appendEntries: peer="{Voter server-cdfeea54-a6b3-9992-df12-13c6d1b6c419 cdfeea54-a6b3-9992-df12-13c6d1b6c419}" error="append pipeline closed" -2025-10-27T04:03:43.607Z [INFO] server-f79ae124-4251-0308-4d6b-2a44610bc0cf: aborting pipeline replication: peer="{Voter server-cdfeea54-a6b3-9992-df12-13c6d1b6c419 cdfeea54-a6b3-9992-df12-13c6d1b6c419}" -2025-10-27T04:03:43.607Z [ERROR] server-f79ae124-4251-0308-4d6b-2a44610bc0cf: failed to appendEntries to: peer="{Voter server-cdfeea54-a6b3-9992-df12-13c6d1b6c419 cdfeea54-a6b3-9992-df12-13c6d1b6c419}" error="failed to connect to peer: cdfeea54-a6b3-9992-df12-13c6d1b6c419" +2026-11-29T10:29:46.245Z [INFO] server-945782be-5750-29c5-5a67-97778521c947: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-945782be-5750-29c5-5a67-97778521c947 Address:945782be-5750-29c5-5a67-97778521c947} {Suffrage:Voter ID:server-62e5c55a-65eb-0087-8b08-788195409982 Address:62e5c55a-65eb-0087-8b08-788195409982} {Suffrage:Voter ID:server-8e6c31b8-3e58-db51-e2af-129b4c7079cc Address:8e6c31b8-3e58-db51-e2af-129b4c7079cc}]" +2026-11-29T10:29:46.245Z [INFO] server-62e5c55a-65eb-0087-8b08-788195409982: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-945782be-5750-29c5-5a67-97778521c947 Address:945782be-5750-29c5-5a67-97778521c947} {Suffrage:Voter ID:server-62e5c55a-65eb-0087-8b08-788195409982 Address:62e5c55a-65eb-0087-8b08-788195409982} {Suffrage:Voter ID:server-8e6c31b8-3e58-db51-e2af-129b4c7079cc Address:8e6c31b8-3e58-db51-e2af-129b4c7079cc}]" +2026-11-29T10:29:46.245Z [INFO] server-945782be-5750-29c5-5a67-97778521c947: entering follower state: follower="Node at 945782be-5750-29c5-5a67-97778521c947 [Follower]" leader-address= leader-id= +2026-11-29T10:29:46.245Z [INFO] server-62e5c55a-65eb-0087-8b08-788195409982: entering follower state: follower="Node at 62e5c55a-65eb-0087-8b08-788195409982 [Follower]" leader-address= leader-id= +2026-11-29T10:29:46.245Z [INFO] server-8e6c31b8-3e58-db51-e2af-129b4c7079cc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-945782be-5750-29c5-5a67-97778521c947 Address:945782be-5750-29c5-5a67-97778521c947} {Suffrage:Voter ID:server-62e5c55a-65eb-0087-8b08-788195409982 Address:62e5c55a-65eb-0087-8b08-788195409982} {Suffrage:Voter ID:server-8e6c31b8-3e58-db51-e2af-129b4c7079cc Address:8e6c31b8-3e58-db51-e2af-129b4c7079cc}]" +2026-11-29T10:29:46.245Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:46.245Z [INFO] server-8e6c31b8-3e58-db51-e2af-129b4c7079cc: entering follower state: follower="Node at 8e6c31b8-3e58-db51-e2af-129b4c7079cc [Follower]" leader-address= leader-id= +2026-11-29T10:29:46.296Z [WARN] server-8e6c31b8-3e58-db51-e2af-129b4c7079cc: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:46.296Z [INFO] server-8e6c31b8-3e58-db51-e2af-129b4c7079cc: entering candidate state: node="Node at 8e6c31b8-3e58-db51-e2af-129b4c7079cc [Candidate]" term=2 +2026-11-29T10:29:46.296Z [INFO] server-8e6c31b8-3e58-db51-e2af-129b4c7079cc: election won: term=2 tally=2 +2026-11-29T10:29:46.296Z [INFO] server-8e6c31b8-3e58-db51-e2af-129b4c7079cc: entering leader state: leader="Node at 8e6c31b8-3e58-db51-e2af-129b4c7079cc [Leader]" +2026-11-29T10:29:46.296Z [INFO] server-8e6c31b8-3e58-db51-e2af-129b4c7079cc: added peer, starting replication: peer=server-945782be-5750-29c5-5a67-97778521c947 +2026-11-29T10:29:46.297Z [INFO] server-8e6c31b8-3e58-db51-e2af-129b4c7079cc: added peer, starting replication: peer=server-62e5c55a-65eb-0087-8b08-788195409982 +2026-11-29T10:29:46.297Z [INFO] server-8e6c31b8-3e58-db51-e2af-129b4c7079cc: pipelining replication: peer="{Voter server-945782be-5750-29c5-5a67-97778521c947 945782be-5750-29c5-5a67-97778521c947}" +2026-11-29T10:29:46.297Z [INFO] server-8e6c31b8-3e58-db51-e2af-129b4c7079cc: pipelining replication: peer="{Voter server-62e5c55a-65eb-0087-8b08-788195409982 62e5c55a-65eb-0087-8b08-788195409982}" +2026-11-29T10:29:46.402Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:46.297044779 +0000 UTC m=+97.032098881 (2 nodes), 51.68268ms from start of poll, 51.90124ms from cluster start. Timeout at 2026-11-29 10:29:46.402049859 +0000 UTC m=+97.137103961, 105.00508ms after stability +2026-11-29T10:29:46.402Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:46.508Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:46.402515539 +0000 UTC m=+97.137569601 (1 nodes), 1.56µs from start of poll, 157.37196ms from cluster start. Timeout at 2026-11-29 10:29:46.507538619 +0000 UTC m=+97.242592681, 105.02308ms after stability +2026-11-29T10:29:46.509Z [ERROR] server-8e6c31b8-3e58-db51-e2af-129b4c7079cc: failed to pipeline appendEntries: peer="{Voter server-945782be-5750-29c5-5a67-97778521c947 945782be-5750-29c5-5a67-97778521c947}" error="append pipeline closed" +2026-11-29T10:29:46.509Z [INFO] server-8e6c31b8-3e58-db51-e2af-129b4c7079cc: aborting pipeline replication: peer="{Voter server-945782be-5750-29c5-5a67-97778521c947 945782be-5750-29c5-5a67-97778521c947}" +2026-11-29T10:29:46.509Z [ERROR] server-8e6c31b8-3e58-db51-e2af-129b4c7079cc: failed to appendEntries to: peer="{Voter server-945782be-5750-29c5-5a67-97778521c947 945782be-5750-29c5-5a67-97778521c947}" error="failed to connect to peer: 945782be-5750-29c5-5a67-97778521c947" raft_test.go:1627: [INFO] Finished apply without behind follower -2025-10-27T04:03:43.611Z [ERROR] server-cdfeea54-a6b3-9992-df12-13c6d1b6c419: failed to take snapshot: error="nothing new to snapshot" -2025-10-27T04:03:43.611Z [INFO] server-8a95685e-5192-ce55-868e-2900974dd2aa: starting snapshot up to: index=99 -2025-10-27T04:03:43.611Z [INFO] creating new snapshot: path=/tmp/raft38282316/snapshots/2-99-1761537823611.tmp -2025-10-27T04:03:43.611Z [INFO] server-8a95685e-5192-ce55-868e-2900974dd2aa: compacting logs: from=1 to=92 -2025-10-27T04:03:43.612Z [INFO] server-8a95685e-5192-ce55-868e-2900974dd2aa: snapshot complete up to: index=99 -2025-10-27T04:03:43.612Z [INFO] server-f79ae124-4251-0308-4d6b-2a44610bc0cf: starting snapshot up to: index=102 -2025-10-27T04:03:43.612Z [INFO] creating new snapshot: path=/tmp/raft584231792/snapshots/2-102-1761537823612.tmp -2025-10-27T04:03:43.612Z [INFO] server-f79ae124-4251-0308-4d6b-2a44610bc0cf: compacting logs: from=1 to=92 -2025-10-27T04:03:43.612Z [INFO] server-f79ae124-4251-0308-4d6b-2a44610bc0cf: snapshot complete up to: index=102 -2025-10-27T04:03:43.618Z [ERROR] server-f79ae124-4251-0308-4d6b-2a44610bc0cf: failed to get log: index=2 error="log not found" -2025-10-27T04:03:43.618Z [INFO] creating new snapshot: path=/tmp/raft3879727195/snapshots/2-102-1761537823618.tmp -2025-10-27T04:03:43.618Z [INFO] server-cdfeea54-a6b3-9992-df12-13c6d1b6c419: snapshot network transfer progress: read-bytes=693 percent-complete=100.00% -2025-10-27T04:03:43.618Z [INFO] server-cdfeea54-a6b3-9992-df12-13c6d1b6c419: copied to local snapshot: bytes=693 -2025-10-27T04:03:43.618Z [INFO] server-cdfeea54-a6b3-9992-df12-13c6d1b6c419: snapshot restore progress: id=2-102-1761537823618 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% -2025-10-27T04:03:43.618Z [INFO] server-cdfeea54-a6b3-9992-df12-13c6d1b6c419: Installed remote snapshot -2025-10-27T04:03:43.619Z [WARN] server-cdfeea54-a6b3-9992-df12-13c6d1b6c419: skipping application of old log: index=102 -2025-10-27T04:03:43.619Z [INFO] server-f79ae124-4251-0308-4d6b-2a44610bc0cf: pipelining replication: peer="{Voter server-cdfeea54-a6b3-9992-df12-13c6d1b6c419 cdfeea54-a6b3-9992-df12-13c6d1b6c419}" -2025-10-27T04:03:43.624Z [INFO] server-f79ae124-4251-0308-4d6b-2a44610bc0cf: aborting pipeline replication: peer="{Voter server-8a95685e-5192-ce55-868e-2900974dd2aa 8a95685e-5192-ce55-868e-2900974dd2aa}" -2025-10-27T04:03:43.624Z [INFO] server-f79ae124-4251-0308-4d6b-2a44610bc0cf: aborting pipeline replication: peer="{Voter server-cdfeea54-a6b3-9992-df12-13c6d1b6c419 cdfeea54-a6b3-9992-df12-13c6d1b6c419}" ---- PASS: TestRaft_SendSnapshotFollower (0.30s) +2026-11-29T10:29:46.511Z [ERROR] server-945782be-5750-29c5-5a67-97778521c947: failed to take snapshot: error="nothing new to snapshot" +2026-11-29T10:29:46.511Z [INFO] server-62e5c55a-65eb-0087-8b08-788195409982: starting snapshot up to: index=99 +2026-11-29T10:29:46.511Z [INFO] creating new snapshot: path=/tmp/raft3669325136/snapshots/2-99-1795948186511.tmp +2026-11-29T10:29:46.511Z [INFO] server-62e5c55a-65eb-0087-8b08-788195409982: compacting logs: from=1 to=92 +2026-11-29T10:29:46.511Z [INFO] server-62e5c55a-65eb-0087-8b08-788195409982: snapshot complete up to: index=99 +2026-11-29T10:29:46.511Z [INFO] server-8e6c31b8-3e58-db51-e2af-129b4c7079cc: starting snapshot up to: index=102 +2026-11-29T10:29:46.511Z [INFO] creating new snapshot: path=/tmp/raft3939904159/snapshots/2-102-1795948186511.tmp +2026-11-29T10:29:46.512Z [INFO] server-8e6c31b8-3e58-db51-e2af-129b4c7079cc: compacting logs: from=1 to=92 +2026-11-29T10:29:46.512Z [INFO] server-8e6c31b8-3e58-db51-e2af-129b4c7079cc: snapshot complete up to: index=102 +2026-11-29T10:29:46.519Z [ERROR] server-8e6c31b8-3e58-db51-e2af-129b4c7079cc: failed to get log: index=2 error="log not found" +2026-11-29T10:29:46.519Z [INFO] creating new snapshot: path=/tmp/raft2975553241/snapshots/2-102-1795948186519.tmp +2026-11-29T10:29:46.519Z [INFO] server-945782be-5750-29c5-5a67-97778521c947: snapshot network transfer progress: read-bytes=693 percent-complete=100.00% +2026-11-29T10:29:46.519Z [INFO] server-945782be-5750-29c5-5a67-97778521c947: copied to local snapshot: bytes=693 +2026-11-29T10:29:46.520Z [INFO] server-945782be-5750-29c5-5a67-97778521c947: snapshot restore progress: id=2-102-1795948186519 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% +2026-11-29T10:29:46.520Z [INFO] server-945782be-5750-29c5-5a67-97778521c947: Installed remote snapshot +2026-11-29T10:29:46.520Z [WARN] server-945782be-5750-29c5-5a67-97778521c947: skipping application of old log: index=102 +2026-11-29T10:29:46.520Z [INFO] server-8e6c31b8-3e58-db51-e2af-129b4c7079cc: pipelining replication: peer="{Voter server-945782be-5750-29c5-5a67-97778521c947 945782be-5750-29c5-5a67-97778521c947}" +2026-11-29T10:29:46.524Z [INFO] server-8e6c31b8-3e58-db51-e2af-129b4c7079cc: aborting pipeline replication: peer="{Voter server-62e5c55a-65eb-0087-8b08-788195409982 62e5c55a-65eb-0087-8b08-788195409982}" +2026-11-29T10:29:46.524Z [INFO] server-8e6c31b8-3e58-db51-e2af-129b4c7079cc: aborting pipeline replication: peer="{Voter server-945782be-5750-29c5-5a67-97778521c947 945782be-5750-29c5-5a67-97778521c947}" +--- PASS: TestRaft_SendSnapshotFollower (0.28s) === RUN TestRaft_SendSnapshotAndLogsFollower -2025-10-27T04:03:43.624Z [INFO] server-2c6ce847-9c11-3f1f-bb58-aa22691a0b39: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2c6ce847-9c11-3f1f-bb58-aa22691a0b39 Address:2c6ce847-9c11-3f1f-bb58-aa22691a0b39} {Suffrage:Voter ID:server-5a63170f-6a40-0b4a-1f79-cda0039c5562 Address:5a63170f-6a40-0b4a-1f79-cda0039c5562} {Suffrage:Voter ID:server-126d23ae-a15b-40f1-dda7-92e89a60e034 Address:126d23ae-a15b-40f1-dda7-92e89a60e034}]" -2025-10-27T04:03:43.624Z [INFO] server-5a63170f-6a40-0b4a-1f79-cda0039c5562: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2c6ce847-9c11-3f1f-bb58-aa22691a0b39 Address:2c6ce847-9c11-3f1f-bb58-aa22691a0b39} {Suffrage:Voter ID:server-5a63170f-6a40-0b4a-1f79-cda0039c5562 Address:5a63170f-6a40-0b4a-1f79-cda0039c5562} {Suffrage:Voter ID:server-126d23ae-a15b-40f1-dda7-92e89a60e034 Address:126d23ae-a15b-40f1-dda7-92e89a60e034}]" -2025-10-27T04:03:43.624Z [INFO] server-2c6ce847-9c11-3f1f-bb58-aa22691a0b39: entering follower state: follower="Node at 2c6ce847-9c11-3f1f-bb58-aa22691a0b39 [Follower]" leader-address= leader-id= -2025-10-27T04:03:43.624Z [INFO] server-5a63170f-6a40-0b4a-1f79-cda0039c5562: entering follower state: follower="Node at 5a63170f-6a40-0b4a-1f79-cda0039c5562 [Follower]" leader-address= leader-id= -2025-10-27T04:03:43.624Z [INFO] server-126d23ae-a15b-40f1-dda7-92e89a60e034: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2c6ce847-9c11-3f1f-bb58-aa22691a0b39 Address:2c6ce847-9c11-3f1f-bb58-aa22691a0b39} {Suffrage:Voter ID:server-5a63170f-6a40-0b4a-1f79-cda0039c5562 Address:5a63170f-6a40-0b4a-1f79-cda0039c5562} {Suffrage:Voter ID:server-126d23ae-a15b-40f1-dda7-92e89a60e034 Address:126d23ae-a15b-40f1-dda7-92e89a60e034}]" -2025-10-27T04:03:43.624Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:03:43.624Z [INFO] server-126d23ae-a15b-40f1-dda7-92e89a60e034: entering follower state: follower="Node at 126d23ae-a15b-40f1-dda7-92e89a60e034 [Follower]" leader-address= leader-id= -2025-10-27T04:03:43.686Z [WARN] server-126d23ae-a15b-40f1-dda7-92e89a60e034: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:43.686Z [INFO] server-126d23ae-a15b-40f1-dda7-92e89a60e034: entering candidate state: node="Node at 126d23ae-a15b-40f1-dda7-92e89a60e034 [Candidate]" term=2 -2025-10-27T04:03:43.686Z [INFO] server-126d23ae-a15b-40f1-dda7-92e89a60e034: election won: term=2 tally=2 -2025-10-27T04:03:43.686Z [INFO] server-126d23ae-a15b-40f1-dda7-92e89a60e034: entering leader state: leader="Node at 126d23ae-a15b-40f1-dda7-92e89a60e034 [Leader]" -2025-10-27T04:03:43.686Z [INFO] server-126d23ae-a15b-40f1-dda7-92e89a60e034: added peer, starting replication: peer=server-2c6ce847-9c11-3f1f-bb58-aa22691a0b39 -2025-10-27T04:03:43.686Z [INFO] server-126d23ae-a15b-40f1-dda7-92e89a60e034: added peer, starting replication: peer=server-5a63170f-6a40-0b4a-1f79-cda0039c5562 -2025-10-27T04:03:43.686Z [INFO] server-126d23ae-a15b-40f1-dda7-92e89a60e034: pipelining replication: peer="{Voter server-5a63170f-6a40-0b4a-1f79-cda0039c5562 5a63170f-6a40-0b4a-1f79-cda0039c5562}" -2025-10-27T04:03:43.686Z [INFO] server-126d23ae-a15b-40f1-dda7-92e89a60e034: pipelining replication: peer="{Voter server-2c6ce847-9c11-3f1f-bb58-aa22691a0b39 2c6ce847-9c11-3f1f-bb58-aa22691a0b39}" -2025-10-27T04:03:43.791Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:03:43.686754509 +0000 UTC m=+98.681626987 (2 nodes), 61.801942ms from start of poll, 62.055624ms from cluster start. Timeout at 2025-10-27 04:03:43.791757189 +0000 UTC m=+98.786629627, 105.00264ms after stability -2025-10-27T04:03:43.792Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:43.897Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:43.792035108 +0000 UTC m=+98.786907546 (1 nodes), 1.24µs from start of poll, 167.336183ms from cluster start. Timeout at 2025-10-27 04:03:43.897054548 +0000 UTC m=+98.891926946, 105.0194ms after stability -2025-10-27T04:03:43.897Z [ERROR] server-126d23ae-a15b-40f1-dda7-92e89a60e034: failed to pipeline appendEntries: peer="{Voter server-2c6ce847-9c11-3f1f-bb58-aa22691a0b39 2c6ce847-9c11-3f1f-bb58-aa22691a0b39}" error="append pipeline closed" -2025-10-27T04:03:43.897Z [INFO] server-126d23ae-a15b-40f1-dda7-92e89a60e034: aborting pipeline replication: peer="{Voter server-2c6ce847-9c11-3f1f-bb58-aa22691a0b39 2c6ce847-9c11-3f1f-bb58-aa22691a0b39}" -2025-10-27T04:03:43.897Z [ERROR] server-126d23ae-a15b-40f1-dda7-92e89a60e034: failed to appendEntries to: peer="{Voter server-2c6ce847-9c11-3f1f-bb58-aa22691a0b39 2c6ce847-9c11-3f1f-bb58-aa22691a0b39}" error="failed to connect to peer: 2c6ce847-9c11-3f1f-bb58-aa22691a0b39" +2026-11-29T10:29:46.524Z [INFO] server-842d15c8-7700-3ecd-b87d-119f3a22433b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-842d15c8-7700-3ecd-b87d-119f3a22433b Address:842d15c8-7700-3ecd-b87d-119f3a22433b} {Suffrage:Voter ID:server-34952207-ab36-b5c3-cddb-c428865b55ff Address:34952207-ab36-b5c3-cddb-c428865b55ff} {Suffrage:Voter ID:server-c7f94f73-a7fd-88c4-606d-f51907083798 Address:c7f94f73-a7fd-88c4-606d-f51907083798}]" +2026-11-29T10:29:46.524Z [INFO] server-34952207-ab36-b5c3-cddb-c428865b55ff: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-842d15c8-7700-3ecd-b87d-119f3a22433b Address:842d15c8-7700-3ecd-b87d-119f3a22433b} {Suffrage:Voter ID:server-34952207-ab36-b5c3-cddb-c428865b55ff Address:34952207-ab36-b5c3-cddb-c428865b55ff} {Suffrage:Voter ID:server-c7f94f73-a7fd-88c4-606d-f51907083798 Address:c7f94f73-a7fd-88c4-606d-f51907083798}]" +2026-11-29T10:29:46.524Z [INFO] server-c7f94f73-a7fd-88c4-606d-f51907083798: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-842d15c8-7700-3ecd-b87d-119f3a22433b Address:842d15c8-7700-3ecd-b87d-119f3a22433b} {Suffrage:Voter ID:server-34952207-ab36-b5c3-cddb-c428865b55ff Address:34952207-ab36-b5c3-cddb-c428865b55ff} {Suffrage:Voter ID:server-c7f94f73-a7fd-88c4-606d-f51907083798 Address:c7f94f73-a7fd-88c4-606d-f51907083798}]" +2026-11-29T10:29:46.524Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:46.524Z [INFO] server-842d15c8-7700-3ecd-b87d-119f3a22433b: entering follower state: follower="Node at 842d15c8-7700-3ecd-b87d-119f3a22433b [Follower]" leader-address= leader-id= +2026-11-29T10:29:46.524Z [INFO] server-34952207-ab36-b5c3-cddb-c428865b55ff: entering follower state: follower="Node at 34952207-ab36-b5c3-cddb-c428865b55ff [Follower]" leader-address= leader-id= +2026-11-29T10:29:46.524Z [INFO] server-c7f94f73-a7fd-88c4-606d-f51907083798: entering follower state: follower="Node at c7f94f73-a7fd-88c4-606d-f51907083798 [Follower]" leader-address= leader-id= +2026-11-29T10:29:46.576Z [WARN] server-34952207-ab36-b5c3-cddb-c428865b55ff: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:46.576Z [INFO] server-34952207-ab36-b5c3-cddb-c428865b55ff: entering candidate state: node="Node at 34952207-ab36-b5c3-cddb-c428865b55ff [Candidate]" term=2 +2026-11-29T10:29:46.576Z [INFO] server-34952207-ab36-b5c3-cddb-c428865b55ff: election won: term=2 tally=2 +2026-11-29T10:29:46.576Z [INFO] server-34952207-ab36-b5c3-cddb-c428865b55ff: entering leader state: leader="Node at 34952207-ab36-b5c3-cddb-c428865b55ff [Leader]" +2026-11-29T10:29:46.576Z [INFO] server-34952207-ab36-b5c3-cddb-c428865b55ff: added peer, starting replication: peer=server-842d15c8-7700-3ecd-b87d-119f3a22433b +2026-11-29T10:29:46.576Z [INFO] server-34952207-ab36-b5c3-cddb-c428865b55ff: added peer, starting replication: peer=server-c7f94f73-a7fd-88c4-606d-f51907083798 +2026-11-29T10:29:46.576Z [INFO] server-34952207-ab36-b5c3-cddb-c428865b55ff: pipelining replication: peer="{Voter server-c7f94f73-a7fd-88c4-606d-f51907083798 c7f94f73-a7fd-88c4-606d-f51907083798}" +2026-11-29T10:29:46.576Z [INFO] server-34952207-ab36-b5c3-cddb-c428865b55ff: pipelining replication: peer="{Voter server-842d15c8-7700-3ecd-b87d-119f3a22433b 842d15c8-7700-3ecd-b87d-119f3a22433b}" +2026-11-29T10:29:46.681Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:46.576492059 +0000 UTC m=+97.311546121 (2 nodes), 51.75804ms from start of poll, 51.948ms from cluster start. Timeout at 2026-11-29 10:29:46.681500899 +0000 UTC m=+97.416555001, 105.00888ms after stability +2026-11-29T10:29:46.681Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:46.787Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:46.681638979 +0000 UTC m=+97.416693081 (1 nodes), 1.44µs from start of poll, 157.09496ms from cluster start. Timeout at 2026-11-29 10:29:46.786659779 +0000 UTC m=+97.521713881, 105.0208ms after stability +2026-11-29T10:29:46.787Z [ERROR] server-34952207-ab36-b5c3-cddb-c428865b55ff: failed to pipeline appendEntries: peer="{Voter server-842d15c8-7700-3ecd-b87d-119f3a22433b 842d15c8-7700-3ecd-b87d-119f3a22433b}" error="append pipeline closed" +2026-11-29T10:29:46.787Z [INFO] server-34952207-ab36-b5c3-cddb-c428865b55ff: aborting pipeline replication: peer="{Voter server-842d15c8-7700-3ecd-b87d-119f3a22433b 842d15c8-7700-3ecd-b87d-119f3a22433b}" +2026-11-29T10:29:46.787Z [ERROR] server-34952207-ab36-b5c3-cddb-c428865b55ff: failed to appendEntries to: peer="{Voter server-842d15c8-7700-3ecd-b87d-119f3a22433b 842d15c8-7700-3ecd-b87d-119f3a22433b}" error="failed to connect to peer: 842d15c8-7700-3ecd-b87d-119f3a22433b" raft_test.go:1669: [INFO] Finished apply without behind follower -2025-10-27T04:03:43.902Z [ERROR] server-2c6ce847-9c11-3f1f-bb58-aa22691a0b39: failed to take snapshot: error="nothing new to snapshot" -2025-10-27T04:03:43.902Z [INFO] server-5a63170f-6a40-0b4a-1f79-cda0039c5562: starting snapshot up to: index=100 -2025-10-27T04:03:43.902Z [INFO] creating new snapshot: path=/tmp/raft2422501064/snapshots/2-100-1761537823902.tmp -2025-10-27T04:03:43.903Z [INFO] server-5a63170f-6a40-0b4a-1f79-cda0039c5562: compacting logs: from=1 to=92 -2025-10-27T04:03:43.903Z [INFO] server-5a63170f-6a40-0b4a-1f79-cda0039c5562: snapshot complete up to: index=100 -2025-10-27T04:03:43.903Z [INFO] server-126d23ae-a15b-40f1-dda7-92e89a60e034: starting snapshot up to: index=102 -2025-10-27T04:03:43.903Z [INFO] creating new snapshot: path=/tmp/raft1731020482/snapshots/2-102-1761537823903.tmp -2025-10-27T04:03:43.903Z [INFO] server-126d23ae-a15b-40f1-dda7-92e89a60e034: compacting logs: from=1 to=92 -2025-10-27T04:03:43.903Z [INFO] server-126d23ae-a15b-40f1-dda7-92e89a60e034: snapshot complete up to: index=102 -2025-10-27T04:03:43.904Z [ERROR] server-126d23ae-a15b-40f1-dda7-92e89a60e034: failed to heartbeat to: peer=2c6ce847-9c11-3f1f-bb58-aa22691a0b39 backoff time=10ms error="failed to connect to peer: 2c6ce847-9c11-3f1f-bb58-aa22691a0b39" +2026-11-29T10:29:46.790Z [ERROR] server-842d15c8-7700-3ecd-b87d-119f3a22433b: failed to take snapshot: error="nothing new to snapshot" +2026-11-29T10:29:46.790Z [INFO] server-34952207-ab36-b5c3-cddb-c428865b55ff: starting snapshot up to: index=102 +2026-11-29T10:29:46.790Z [INFO] creating new snapshot: path=/tmp/raft994923316/snapshots/2-102-1795948186790.tmp +2026-11-29T10:29:46.790Z [INFO] server-34952207-ab36-b5c3-cddb-c428865b55ff: compacting logs: from=1 to=92 +2026-11-29T10:29:46.790Z [INFO] server-34952207-ab36-b5c3-cddb-c428865b55ff: snapshot complete up to: index=102 +2026-11-29T10:29:46.790Z [INFO] server-c7f94f73-a7fd-88c4-606d-f51907083798: starting snapshot up to: index=98 +2026-11-29T10:29:46.790Z [INFO] creating new snapshot: path=/tmp/raft2929224779/snapshots/2-98-1795948186790.tmp +2026-11-29T10:29:46.790Z [INFO] server-c7f94f73-a7fd-88c4-606d-f51907083798: compacting logs: from=1 to=92 +2026-11-29T10:29:46.790Z [INFO] server-c7f94f73-a7fd-88c4-606d-f51907083798: snapshot complete up to: index=98 +2026-11-29T10:29:46.794Z [ERROR] server-34952207-ab36-b5c3-cddb-c428865b55ff: failed to heartbeat to: peer=842d15c8-7700-3ecd-b87d-119f3a22433b backoff time=10ms error="failed to connect to peer: 842d15c8-7700-3ecd-b87d-119f3a22433b" raft_test.go:1690: [INFO] Finished apply without behind follower -2025-10-27T04:03:43.908Z [ERROR] server-126d23ae-a15b-40f1-dda7-92e89a60e034: failed to get log: index=2 error="log not found" -2025-10-27T04:03:43.908Z [INFO] creating new snapshot: path=/tmp/raft62187159/snapshots/2-102-1761537823908.tmp -2025-10-27T04:03:43.908Z [INFO] server-2c6ce847-9c11-3f1f-bb58-aa22691a0b39: snapshot network transfer progress: read-bytes=693 percent-complete=100.00% -2025-10-27T04:03:43.909Z [INFO] server-2c6ce847-9c11-3f1f-bb58-aa22691a0b39: copied to local snapshot: bytes=693 -2025-10-27T04:03:43.909Z [INFO] server-2c6ce847-9c11-3f1f-bb58-aa22691a0b39: snapshot restore progress: id=2-102-1761537823908 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% -2025-10-27T04:03:43.909Z [INFO] server-2c6ce847-9c11-3f1f-bb58-aa22691a0b39: Installed remote snapshot -2025-10-27T04:03:43.909Z [INFO] server-126d23ae-a15b-40f1-dda7-92e89a60e034: pipelining replication: peer="{Voter server-2c6ce847-9c11-3f1f-bb58-aa22691a0b39 2c6ce847-9c11-3f1f-bb58-aa22691a0b39}" -2025-10-27T04:03:43.917Z [INFO] server-126d23ae-a15b-40f1-dda7-92e89a60e034: aborting pipeline replication: peer="{Voter server-5a63170f-6a40-0b4a-1f79-cda0039c5562 5a63170f-6a40-0b4a-1f79-cda0039c5562}" -2025-10-27T04:03:43.917Z [INFO] server-126d23ae-a15b-40f1-dda7-92e89a60e034: aborting pipeline replication: peer="{Voter server-2c6ce847-9c11-3f1f-bb58-aa22691a0b39 2c6ce847-9c11-3f1f-bb58-aa22691a0b39}" ---- PASS: TestRaft_SendSnapshotAndLogsFollower (0.29s) +2026-11-29T10:29:46.798Z [ERROR] server-34952207-ab36-b5c3-cddb-c428865b55ff: failed to get log: index=2 error="log not found" +2026-11-29T10:29:46.798Z [INFO] creating new snapshot: path=/tmp/raft2588420565/snapshots/2-102-1795948186798.tmp +2026-11-29T10:29:46.798Z [INFO] server-842d15c8-7700-3ecd-b87d-119f3a22433b: snapshot network transfer progress: read-bytes=693 percent-complete=100.00% +2026-11-29T10:29:46.799Z [INFO] server-842d15c8-7700-3ecd-b87d-119f3a22433b: copied to local snapshot: bytes=693 +2026-11-29T10:29:46.799Z [INFO] server-842d15c8-7700-3ecd-b87d-119f3a22433b: snapshot restore progress: id=2-102-1795948186798 last-index=102 last-term=2 size-in-bytes=693 read-bytes=693 percent-complete=100.00% +2026-11-29T10:29:46.799Z [INFO] server-842d15c8-7700-3ecd-b87d-119f3a22433b: Installed remote snapshot +2026-11-29T10:29:46.799Z [INFO] server-34952207-ab36-b5c3-cddb-c428865b55ff: pipelining replication: peer="{Voter server-842d15c8-7700-3ecd-b87d-119f3a22433b 842d15c8-7700-3ecd-b87d-119f3a22433b}" +2026-11-29T10:29:46.806Z [INFO] server-34952207-ab36-b5c3-cddb-c428865b55ff: aborting pipeline replication: peer="{Voter server-c7f94f73-a7fd-88c4-606d-f51907083798 c7f94f73-a7fd-88c4-606d-f51907083798}" +2026-11-29T10:29:46.806Z [INFO] server-34952207-ab36-b5c3-cddb-c428865b55ff: aborting pipeline replication: peer="{Voter server-842d15c8-7700-3ecd-b87d-119f3a22433b 842d15c8-7700-3ecd-b87d-119f3a22433b}" +--- PASS: TestRaft_SendSnapshotAndLogsFollower (0.28s) === RUN TestRaft_ReJoinFollower -2025-10-27T04:03:43.918Z [INFO] server-cd022b19-94ac-ba46-59c8-e7b4304ab069: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd022b19-94ac-ba46-59c8-e7b4304ab069 Address:cd022b19-94ac-ba46-59c8-e7b4304ab069} {Suffrage:Voter ID:server-ac23a7b1-d69d-9fc1-ddcc-3d84f762d299 Address:ac23a7b1-d69d-9fc1-ddcc-3d84f762d299} {Suffrage:Voter ID:server-1d80c12b-e515-e89a-5e7f-4fc6f5705ed1 Address:1d80c12b-e515-e89a-5e7f-4fc6f5705ed1}]" -2025-10-27T04:03:43.918Z [INFO] server-ac23a7b1-d69d-9fc1-ddcc-3d84f762d299: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd022b19-94ac-ba46-59c8-e7b4304ab069 Address:cd022b19-94ac-ba46-59c8-e7b4304ab069} {Suffrage:Voter ID:server-ac23a7b1-d69d-9fc1-ddcc-3d84f762d299 Address:ac23a7b1-d69d-9fc1-ddcc-3d84f762d299} {Suffrage:Voter ID:server-1d80c12b-e515-e89a-5e7f-4fc6f5705ed1 Address:1d80c12b-e515-e89a-5e7f-4fc6f5705ed1}]" -2025-10-27T04:03:43.918Z [INFO] server-cd022b19-94ac-ba46-59c8-e7b4304ab069: entering follower state: follower="Node at cd022b19-94ac-ba46-59c8-e7b4304ab069 [Follower]" leader-address= leader-id= -2025-10-27T04:03:43.918Z [INFO] server-1d80c12b-e515-e89a-5e7f-4fc6f5705ed1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd022b19-94ac-ba46-59c8-e7b4304ab069 Address:cd022b19-94ac-ba46-59c8-e7b4304ab069} {Suffrage:Voter ID:server-ac23a7b1-d69d-9fc1-ddcc-3d84f762d299 Address:ac23a7b1-d69d-9fc1-ddcc-3d84f762d299} {Suffrage:Voter ID:server-1d80c12b-e515-e89a-5e7f-4fc6f5705ed1 Address:1d80c12b-e515-e89a-5e7f-4fc6f5705ed1}]" -2025-10-27T04:03:43.918Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:43.918Z [INFO] server-ac23a7b1-d69d-9fc1-ddcc-3d84f762d299: entering follower state: follower="Node at ac23a7b1-d69d-9fc1-ddcc-3d84f762d299 [Follower]" leader-address= leader-id= -2025-10-27T04:03:43.918Z [INFO] server-1d80c12b-e515-e89a-5e7f-4fc6f5705ed1: entering follower state: follower="Node at 1d80c12b-e515-e89a-5e7f-4fc6f5705ed1 [Follower]" leader-address= leader-id= -2025-10-27T04:03:43.970Z [WARN] server-ac23a7b1-d69d-9fc1-ddcc-3d84f762d299: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:43.970Z [INFO] server-ac23a7b1-d69d-9fc1-ddcc-3d84f762d299: entering candidate state: node="Node at ac23a7b1-d69d-9fc1-ddcc-3d84f762d299 [Candidate]" term=2 -2025-10-27T04:03:43.970Z [INFO] server-ac23a7b1-d69d-9fc1-ddcc-3d84f762d299: election won: term=2 tally=2 -2025-10-27T04:03:43.970Z [INFO] server-ac23a7b1-d69d-9fc1-ddcc-3d84f762d299: entering leader state: leader="Node at ac23a7b1-d69d-9fc1-ddcc-3d84f762d299 [Leader]" -2025-10-27T04:03:43.970Z [INFO] server-ac23a7b1-d69d-9fc1-ddcc-3d84f762d299: added peer, starting replication: peer=server-cd022b19-94ac-ba46-59c8-e7b4304ab069 -2025-10-27T04:03:43.970Z [INFO] server-ac23a7b1-d69d-9fc1-ddcc-3d84f762d299: added peer, starting replication: peer=server-1d80c12b-e515-e89a-5e7f-4fc6f5705ed1 -2025-10-27T04:03:43.970Z [INFO] server-ac23a7b1-d69d-9fc1-ddcc-3d84f762d299: pipelining replication: peer="{Voter server-1d80c12b-e515-e89a-5e7f-4fc6f5705ed1 1d80c12b-e515-e89a-5e7f-4fc6f5705ed1}" -2025-10-27T04:03:43.970Z [INFO] server-ac23a7b1-d69d-9fc1-ddcc-3d84f762d299: pipelining replication: peer="{Voter server-cd022b19-94ac-ba46-59c8-e7b4304ab069 cd022b19-94ac-ba46-59c8-e7b4304ab069}" -2025-10-27T04:03:44.076Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:43.970518208 +0000 UTC m=+98.965390686 (1 nodes), 51.930635ms from start of poll, 52.210677ms from cluster start. Timeout at 2025-10-27 04:03:44.075532368 +0000 UTC m=+99.070404846, 105.01416ms after stability -2025-10-27T04:03:44.082Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:03:44.188Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:03:44.082209451 +0000 UTC m=+99.077081889 (2 nodes), 720ns from start of poll, 163.90188ms from cluster start. Timeout at 2025-10-27 04:03:44.187227251 +0000 UTC m=+99.182099729, 105.01784ms after stability -2025-10-27T04:03:44.188Z [INFO] server-ac23a7b1-d69d-9fc1-ddcc-3d84f762d299: updating configuration: command=RemoveServer server-id=server-cd022b19-94ac-ba46-59c8-e7b4304ab069 server-addr= servers="[{Suffrage:Voter ID:server-ac23a7b1-d69d-9fc1-ddcc-3d84f762d299 Address:ac23a7b1-d69d-9fc1-ddcc-3d84f762d299} {Suffrage:Voter ID:server-1d80c12b-e515-e89a-5e7f-4fc6f5705ed1 Address:1d80c12b-e515-e89a-5e7f-4fc6f5705ed1}]" -2025-10-27T04:03:44.188Z [INFO] server-ac23a7b1-d69d-9fc1-ddcc-3d84f762d299: removed peer, stopping replication: peer=server-cd022b19-94ac-ba46-59c8-e7b4304ab069 last-index=3 -2025-10-27T04:03:44.188Z [INFO] server-ac23a7b1-d69d-9fc1-ddcc-3d84f762d299: aborting pipeline replication: peer="{Voter server-cd022b19-94ac-ba46-59c8-e7b4304ab069 cd022b19-94ac-ba46-59c8-e7b4304ab069}" -2025-10-27T04:03:44.261Z [WARN] server-cd022b19-94ac-ba46-59c8-e7b4304ab069: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election -2025-10-27T04:03:44.294Z [INFO] server-ac23a7b1-d69d-9fc1-ddcc-3d84f762d299: updating configuration: command=AddVoter server-id=server-cd022b19-94ac-ba46-59c8-e7b4304ab069 server-addr=cd022b19-94ac-ba46-59c8-e7b4304ab069 servers="[{Suffrage:Voter ID:server-ac23a7b1-d69d-9fc1-ddcc-3d84f762d299 Address:ac23a7b1-d69d-9fc1-ddcc-3d84f762d299} {Suffrage:Voter ID:server-1d80c12b-e515-e89a-5e7f-4fc6f5705ed1 Address:1d80c12b-e515-e89a-5e7f-4fc6f5705ed1} {Suffrage:Voter ID:server-cd022b19-94ac-ba46-59c8-e7b4304ab069 Address:cd022b19-94ac-ba46-59c8-e7b4304ab069}]" -2025-10-27T04:03:44.294Z [INFO] server-ac23a7b1-d69d-9fc1-ddcc-3d84f762d299: added peer, starting replication: peer=server-cd022b19-94ac-ba46-59c8-e7b4304ab069 -2025-10-27T04:03:44.294Z [WARN] server-cd022b19-94ac-ba46-59c8-e7b4304ab069: failed to get previous log: previous-index=4 last-index=3 error="log not found" -2025-10-27T04:03:44.294Z [WARN] server-ac23a7b1-d69d-9fc1-ddcc-3d84f762d299: appendEntries rejected, sending older logs: peer="{Voter server-cd022b19-94ac-ba46-59c8-e7b4304ab069 cd022b19-94ac-ba46-59c8-e7b4304ab069}" next=4 -2025-10-27T04:03:44.294Z [INFO] server-ac23a7b1-d69d-9fc1-ddcc-3d84f762d299: pipelining replication: peer="{Voter server-cd022b19-94ac-ba46-59c8-e7b4304ab069 cd022b19-94ac-ba46-59c8-e7b4304ab069}" -2025-10-27T04:03:44.294Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:44.400Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:44.294802663 +0000 UTC m=+99.289675141 (1 nodes), 800ns from start of poll, 376.495132ms from cluster start. Timeout at 2025-10-27 04:03:44.399828383 +0000 UTC m=+99.394700861, 105.02572ms after stability -2025-10-27T04:03:44.400Z [INFO] server-ac23a7b1-d69d-9fc1-ddcc-3d84f762d299: aborting pipeline replication: peer="{Voter server-1d80c12b-e515-e89a-5e7f-4fc6f5705ed1 1d80c12b-e515-e89a-5e7f-4fc6f5705ed1}" -2025-10-27T04:03:44.400Z [INFO] server-ac23a7b1-d69d-9fc1-ddcc-3d84f762d299: aborting pipeline replication: peer="{Voter server-cd022b19-94ac-ba46-59c8-e7b4304ab069 cd022b19-94ac-ba46-59c8-e7b4304ab069}" -2025-10-27T04:03:44.901Z [ERROR] server-ac23a7b1-d69d-9fc1-ddcc-3d84f762d299: failed to heartbeat to: peer=cd022b19-94ac-ba46-59c8-e7b4304ab069 backoff time=10ms error="command timed out" ---- PASS: TestRaft_ReJoinFollower (0.98s) +2026-11-29T10:29:46.807Z [INFO] server-cd74985a-928a-b9fc-be90-f59c1f3275d2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd74985a-928a-b9fc-be90-f59c1f3275d2 Address:cd74985a-928a-b9fc-be90-f59c1f3275d2} {Suffrage:Voter ID:server-de1068f8-16b5-4d72-d52f-726e1a73121f Address:de1068f8-16b5-4d72-d52f-726e1a73121f} {Suffrage:Voter ID:server-c6b77ab3-9422-94f2-d4ae-ae6c48ad3f8b Address:c6b77ab3-9422-94f2-d4ae-ae6c48ad3f8b}]" +2026-11-29T10:29:46.807Z [INFO] server-de1068f8-16b5-4d72-d52f-726e1a73121f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd74985a-928a-b9fc-be90-f59c1f3275d2 Address:cd74985a-928a-b9fc-be90-f59c1f3275d2} {Suffrage:Voter ID:server-de1068f8-16b5-4d72-d52f-726e1a73121f Address:de1068f8-16b5-4d72-d52f-726e1a73121f} {Suffrage:Voter ID:server-c6b77ab3-9422-94f2-d4ae-ae6c48ad3f8b Address:c6b77ab3-9422-94f2-d4ae-ae6c48ad3f8b}]" +2026-11-29T10:29:46.807Z [INFO] server-cd74985a-928a-b9fc-be90-f59c1f3275d2: entering follower state: follower="Node at cd74985a-928a-b9fc-be90-f59c1f3275d2 [Follower]" leader-address= leader-id= +2026-11-29T10:29:46.807Z [INFO] server-de1068f8-16b5-4d72-d52f-726e1a73121f: entering follower state: follower="Node at de1068f8-16b5-4d72-d52f-726e1a73121f [Follower]" leader-address= leader-id= +2026-11-29T10:29:46.807Z [INFO] server-c6b77ab3-9422-94f2-d4ae-ae6c48ad3f8b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd74985a-928a-b9fc-be90-f59c1f3275d2 Address:cd74985a-928a-b9fc-be90-f59c1f3275d2} {Suffrage:Voter ID:server-de1068f8-16b5-4d72-d52f-726e1a73121f Address:de1068f8-16b5-4d72-d52f-726e1a73121f} {Suffrage:Voter ID:server-c6b77ab3-9422-94f2-d4ae-ae6c48ad3f8b Address:c6b77ab3-9422-94f2-d4ae-ae6c48ad3f8b}]" +2026-11-29T10:29:46.807Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:46.807Z [INFO] server-c6b77ab3-9422-94f2-d4ae-ae6c48ad3f8b: entering follower state: follower="Node at c6b77ab3-9422-94f2-d4ae-ae6c48ad3f8b [Follower]" leader-address= leader-id= +2026-11-29T10:29:46.872Z [WARN] server-cd74985a-928a-b9fc-be90-f59c1f3275d2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:46.872Z [INFO] server-cd74985a-928a-b9fc-be90-f59c1f3275d2: entering candidate state: node="Node at cd74985a-928a-b9fc-be90-f59c1f3275d2 [Candidate]" term=2 +2026-11-29T10:29:46.872Z [INFO] server-cd74985a-928a-b9fc-be90-f59c1f3275d2: election won: term=2 tally=2 +2026-11-29T10:29:46.872Z [INFO] server-cd74985a-928a-b9fc-be90-f59c1f3275d2: entering leader state: leader="Node at cd74985a-928a-b9fc-be90-f59c1f3275d2 [Leader]" +2026-11-29T10:29:46.872Z [INFO] server-cd74985a-928a-b9fc-be90-f59c1f3275d2: added peer, starting replication: peer=server-de1068f8-16b5-4d72-d52f-726e1a73121f +2026-11-29T10:29:46.872Z [INFO] server-cd74985a-928a-b9fc-be90-f59c1f3275d2: added peer, starting replication: peer=server-c6b77ab3-9422-94f2-d4ae-ae6c48ad3f8b +2026-11-29T10:29:46.872Z [INFO] server-cd74985a-928a-b9fc-be90-f59c1f3275d2: pipelining replication: peer="{Voter server-c6b77ab3-9422-94f2-d4ae-ae6c48ad3f8b c6b77ab3-9422-94f2-d4ae-ae6c48ad3f8b}" +2026-11-29T10:29:46.872Z [INFO] server-cd74985a-928a-b9fc-be90-f59c1f3275d2: pipelining replication: peer="{Voter server-de1068f8-16b5-4d72-d52f-726e1a73121f de1068f8-16b5-4d72-d52f-726e1a73121f}" +2026-11-29T10:29:46.978Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:46.872808699 +0000 UTC m=+97.607862801 (1 nodes), 65.1958ms from start of poll, 65.43308ms from cluster start. Timeout at 2026-11-29 10:29:46.977811019 +0000 UTC m=+97.712865121, 105.00232ms after stability +2026-11-29T10:29:46.983Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:47.089Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:46.983585179 +0000 UTC m=+97.718639361 (2 nodes), 3.56µs from start of poll, 176.20964ms from cluster start. Timeout at 2026-11-29 10:29:47.088616979 +0000 UTC m=+97.823671041, 105.03168ms after stability +2026-11-29T10:29:47.089Z [INFO] server-cd74985a-928a-b9fc-be90-f59c1f3275d2: updating configuration: command=RemoveServer server-id=server-de1068f8-16b5-4d72-d52f-726e1a73121f server-addr= servers="[{Suffrage:Voter ID:server-cd74985a-928a-b9fc-be90-f59c1f3275d2 Address:cd74985a-928a-b9fc-be90-f59c1f3275d2} {Suffrage:Voter ID:server-c6b77ab3-9422-94f2-d4ae-ae6c48ad3f8b Address:c6b77ab3-9422-94f2-d4ae-ae6c48ad3f8b}]" +2026-11-29T10:29:47.089Z [INFO] server-cd74985a-928a-b9fc-be90-f59c1f3275d2: removed peer, stopping replication: peer=server-de1068f8-16b5-4d72-d52f-726e1a73121f last-index=3 +2026-11-29T10:29:47.089Z [INFO] server-cd74985a-928a-b9fc-be90-f59c1f3275d2: aborting pipeline replication: peer="{Voter server-de1068f8-16b5-4d72-d52f-726e1a73121f de1068f8-16b5-4d72-d52f-726e1a73121f}" +2026-11-29T10:29:47.176Z [WARN] server-de1068f8-16b5-4d72-d52f-726e1a73121f: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +2026-11-29T10:29:47.195Z [INFO] server-cd74985a-928a-b9fc-be90-f59c1f3275d2: updating configuration: command=AddVoter server-id=server-de1068f8-16b5-4d72-d52f-726e1a73121f server-addr=de1068f8-16b5-4d72-d52f-726e1a73121f servers="[{Suffrage:Voter ID:server-cd74985a-928a-b9fc-be90-f59c1f3275d2 Address:cd74985a-928a-b9fc-be90-f59c1f3275d2} {Suffrage:Voter ID:server-c6b77ab3-9422-94f2-d4ae-ae6c48ad3f8b Address:c6b77ab3-9422-94f2-d4ae-ae6c48ad3f8b} {Suffrage:Voter ID:server-de1068f8-16b5-4d72-d52f-726e1a73121f Address:de1068f8-16b5-4d72-d52f-726e1a73121f}]" +2026-11-29T10:29:47.195Z [INFO] server-cd74985a-928a-b9fc-be90-f59c1f3275d2: added peer, starting replication: peer=server-de1068f8-16b5-4d72-d52f-726e1a73121f +2026-11-29T10:29:47.195Z [WARN] server-de1068f8-16b5-4d72-d52f-726e1a73121f: failed to get previous log: previous-index=4 last-index=3 error="log not found" +2026-11-29T10:29:47.195Z [WARN] server-cd74985a-928a-b9fc-be90-f59c1f3275d2: appendEntries rejected, sending older logs: peer="{Voter server-de1068f8-16b5-4d72-d52f-726e1a73121f de1068f8-16b5-4d72-d52f-726e1a73121f}" next=4 +2026-11-29T10:29:47.195Z [INFO] server-cd74985a-928a-b9fc-be90-f59c1f3275d2: pipelining replication: peer="{Voter server-de1068f8-16b5-4d72-d52f-726e1a73121f de1068f8-16b5-4d72-d52f-726e1a73121f}" +2026-11-29T10:29:47.195Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:47.300Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:47.195410739 +0000 UTC m=+97.930464841 (1 nodes), 360ns from start of poll, 388.03512ms from cluster start. Timeout at 2026-11-29 10:29:47.300539019 +0000 UTC m=+98.035593121, 105.12828ms after stability +2026-11-29T10:29:47.301Z [INFO] server-cd74985a-928a-b9fc-be90-f59c1f3275d2: aborting pipeline replication: peer="{Voter server-de1068f8-16b5-4d72-d52f-726e1a73121f de1068f8-16b5-4d72-d52f-726e1a73121f}" +2026-11-29T10:29:47.301Z [INFO] server-cd74985a-928a-b9fc-be90-f59c1f3275d2: aborting pipeline replication: peer="{Voter server-c6b77ab3-9422-94f2-d4ae-ae6c48ad3f8b c6b77ab3-9422-94f2-d4ae-ae6c48ad3f8b}" +--- PASS: TestRaft_ReJoinFollower (0.49s) === RUN TestRaft_LeaderLeaseExpire -2025-10-27T04:03:44.902Z [INFO] server-552f2f4e-f089-7b95-edb7-b7f89fff3890: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-552f2f4e-f089-7b95-edb7-b7f89fff3890 Address:552f2f4e-f089-7b95-edb7-b7f89fff3890} {Suffrage:Voter ID:server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f Address:e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f}]" -2025-10-27T04:03:44.902Z [INFO] server-552f2f4e-f089-7b95-edb7-b7f89fff3890: entering follower state: follower="Node at 552f2f4e-f089-7b95-edb7-b7f89fff3890 [Follower]" leader-address= leader-id= -2025-10-27T04:03:44.902Z [INFO] server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-552f2f4e-f089-7b95-edb7-b7f89fff3890 Address:552f2f4e-f089-7b95-edb7-b7f89fff3890} {Suffrage:Voter ID:server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f Address:e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f}]" -2025-10-27T04:03:44.902Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:44.902Z [INFO] server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f: entering follower state: follower="Node at e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f [Follower]" leader-address= leader-id= -2025-10-27T04:03:44.984Z [WARN] server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:44.984Z [INFO] server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f: entering candidate state: node="Node at e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f [Candidate]" term=2 -2025-10-27T04:03:44.984Z [INFO] server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f: election won: term=2 tally=2 -2025-10-27T04:03:44.984Z [INFO] server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f: entering leader state: leader="Node at e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f [Leader]" -2025-10-27T04:03:44.984Z [INFO] server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f: added peer, starting replication: peer=server-552f2f4e-f089-7b95-edb7-b7f89fff3890 -2025-10-27T04:03:44.984Z [INFO] server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f: pipelining replication: peer="{Voter server-552f2f4e-f089-7b95-edb7-b7f89fff3890 552f2f4e-f089-7b95-edb7-b7f89fff3890}" -2025-10-27T04:03:45.089Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:44.984218333 +0000 UTC m=+99.979090811 (1 nodes), 81.399556ms from start of poll, 81.748038ms from cluster start. Timeout at 2025-10-27 04:03:45.089219333 +0000 UTC m=+100.084091811, 105.001ms after stability -2025-10-27T04:03:45.094Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:03:45.199Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:03:45.094396166 +0000 UTC m=+100.089268644 (1 nodes), 2.24µs from start of poll, 191.925871ms from cluster start. Timeout at 2025-10-27 04:03:45.199431126 +0000 UTC m=+100.194303644, 105.035ms after stability - raft_test.go:1799: [INFO] Disconnecting Node at 552f2f4e-f089-7b95-edb7-b7f89fff3890 [Follower] -2025-10-27T04:03:45.203Z [ERROR] server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f: failed to heartbeat to: peer=552f2f4e-f089-7b95-edb7-b7f89fff3890 backoff time=10ms error="failed to connect to peer: 552f2f4e-f089-7b95-edb7-b7f89fff3890" -2025-10-27T04:03:45.203Z [ERROR] server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f: failed to pipeline appendEntries: peer="{Voter server-552f2f4e-f089-7b95-edb7-b7f89fff3890 552f2f4e-f089-7b95-edb7-b7f89fff3890}" error="append pipeline closed" -2025-10-27T04:03:45.203Z [INFO] server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f: aborting pipeline replication: peer="{Voter server-552f2f4e-f089-7b95-edb7-b7f89fff3890 552f2f4e-f089-7b95-edb7-b7f89fff3890}" -2025-10-27T04:03:45.209Z [ERROR] server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f: failed to appendEntries to: peer="{Voter server-552f2f4e-f089-7b95-edb7-b7f89fff3890 552f2f4e-f089-7b95-edb7-b7f89fff3890}" error="failed to connect to peer: 552f2f4e-f089-7b95-edb7-b7f89fff3890" -2025-10-27T04:03:45.223Z [ERROR] server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f: failed to heartbeat to: peer=552f2f4e-f089-7b95-edb7-b7f89fff3890 backoff time=10ms error="failed to connect to peer: 552f2f4e-f089-7b95-edb7-b7f89fff3890" -2025-10-27T04:03:45.225Z [ERROR] server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f: failed to appendEntries to: peer="{Voter server-552f2f4e-f089-7b95-edb7-b7f89fff3890 552f2f4e-f089-7b95-edb7-b7f89fff3890}" error="failed to connect to peer: 552f2f4e-f089-7b95-edb7-b7f89fff3890" -2025-10-27T04:03:45.242Z [ERROR] server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f: failed to appendEntries to: peer="{Voter server-552f2f4e-f089-7b95-edb7-b7f89fff3890 552f2f4e-f089-7b95-edb7-b7f89fff3890}" error="failed to connect to peer: 552f2f4e-f089-7b95-edb7-b7f89fff3890" -2025-10-27T04:03:45.242Z [ERROR] server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f: failed to heartbeat to: peer=552f2f4e-f089-7b95-edb7-b7f89fff3890 backoff time=10ms error="failed to connect to peer: 552f2f4e-f089-7b95-edb7-b7f89fff3890" -2025-10-27T04:03:45.245Z [WARN] server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f: failed to contact: server-id=server-552f2f4e-f089-7b95-edb7-b7f89fff3890 time=51.050549ms -2025-10-27T04:03:45.245Z [WARN] server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f: failed to contact quorum of nodes, stepping down -2025-10-27T04:03:45.246Z [INFO] server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f: entering follower state: follower="Node at e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f [Follower]" leader-address= leader-id= -2025-10-27T04:03:45.246Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:45.288Z [WARN] server-552f2f4e-f089-7b95-edb7-b7f89fff3890: heartbeat timeout reached, starting election: last-leader-addr=e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f last-leader-id=server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f -2025-10-27T04:03:45.288Z [INFO] server-552f2f4e-f089-7b95-edb7-b7f89fff3890: entering candidate state: node="Node at 552f2f4e-f089-7b95-edb7-b7f89fff3890 [Candidate]" term=3 -2025-10-27T04:03:45.288Z [ERROR] server-552f2f4e-f089-7b95-edb7-b7f89fff3890: failed to make requestVote RPC: target="{Voter server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f}" error="failed to connect to peer: e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f" term=3 -2025-10-27T04:03:45.324Z [WARN] server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:45.324Z [INFO] server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f: entering candidate state: node="Node at e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f [Candidate]" term=3 -2025-10-27T04:03:45.324Z [ERROR] server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f: failed to make requestVote RPC: target="{Voter server-552f2f4e-f089-7b95-edb7-b7f89fff3890 552f2f4e-f089-7b95-edb7-b7f89fff3890}" error="failed to connect to peer: 552f2f4e-f089-7b95-edb7-b7f89fff3890" term=3 -2025-10-27T04:03:45.369Z [WARN] server-552f2f4e-f089-7b95-edb7-b7f89fff3890: Election timeout reached, restarting election -2025-10-27T04:03:45.369Z [INFO] server-552f2f4e-f089-7b95-edb7-b7f89fff3890: entering candidate state: node="Node at 552f2f4e-f089-7b95-edb7-b7f89fff3890 [Candidate]" term=4 -2025-10-27T04:03:45.369Z [ERROR] server-552f2f4e-f089-7b95-edb7-b7f89fff3890: failed to make requestVote RPC: target="{Voter server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f}" error="failed to connect to peer: e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f" term=4 -2025-10-27T04:03:45.409Z [WARN] server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f: Election timeout reached, restarting election -2025-10-27T04:03:45.409Z [INFO] server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f: entering candidate state: node="Node at e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f [Candidate]" term=4 -2025-10-27T04:03:45.409Z [ERROR] server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f: failed to make requestVote RPC: target="{Voter server-552f2f4e-f089-7b95-edb7-b7f89fff3890 552f2f4e-f089-7b95-edb7-b7f89fff3890}" error="failed to connect to peer: 552f2f4e-f089-7b95-edb7-b7f89fff3890" term=4 -2025-10-27T04:03:45.430Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:45.324484818 +0000 UTC m=+100.319357336 (0 nodes), 78.434216ms from start of poll, 422.014563ms from cluster start. Timeout at 2025-10-27 04:03:45.429489378 +0000 UTC m=+100.424361856, 105.00452ms after stability -2025-10-27T04:03:45.450Z [WARN] server-552f2f4e-f089-7b95-edb7-b7f89fff3890: Election timeout reached, restarting election -2025-10-27T04:03:45.450Z [INFO] server-552f2f4e-f089-7b95-edb7-b7f89fff3890: entering candidate state: node="Node at 552f2f4e-f089-7b95-edb7-b7f89fff3890 [Candidate]" term=5 -2025-10-27T04:03:45.450Z [ERROR] server-552f2f4e-f089-7b95-edb7-b7f89fff3890: failed to make requestVote RPC: target="{Voter server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f}" error="failed to connect to peer: e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f" term=5 -2025-10-27T04:03:45.487Z [WARN] server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f: Election timeout reached, restarting election -2025-10-27T04:03:45.487Z [INFO] server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f: entering candidate state: node="Node at e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f [Candidate]" term=5 -2025-10-27T04:03:45.487Z [ERROR] server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f: failed to make requestVote RPC: target="{Voter server-552f2f4e-f089-7b95-edb7-b7f89fff3890 552f2f4e-f089-7b95-edb7-b7f89fff3890}" error="failed to connect to peer: 552f2f4e-f089-7b95-edb7-b7f89fff3890" term=5 -2025-10-27T04:03:45.533Z [WARN] server-552f2f4e-f089-7b95-edb7-b7f89fff3890: Election timeout reached, restarting election -2025-10-27T04:03:45.533Z [INFO] server-552f2f4e-f089-7b95-edb7-b7f89fff3890: entering candidate state: node="Node at 552f2f4e-f089-7b95-edb7-b7f89fff3890 [Candidate]" term=6 -2025-10-27T04:03:45.533Z [ERROR] server-552f2f4e-f089-7b95-edb7-b7f89fff3890: failed to make requestVote RPC: target="{Voter server-e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f}" error="failed to connect to peer: e64b2bcf-e1c8-2e7d-4bac-d4177d27ad9f" term=6 ---- PASS: TestRaft_LeaderLeaseExpire (0.63s) +2026-11-29T10:29:47.301Z [INFO] server-bc9725fc-be77-246e-2088-b1eac590f446: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bc9725fc-be77-246e-2088-b1eac590f446 Address:bc9725fc-be77-246e-2088-b1eac590f446} {Suffrage:Voter ID:server-626098c1-2c32-5e1f-4d86-d508c1384c96 Address:626098c1-2c32-5e1f-4d86-d508c1384c96}]" +2026-11-29T10:29:47.301Z [INFO] server-626098c1-2c32-5e1f-4d86-d508c1384c96: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bc9725fc-be77-246e-2088-b1eac590f446 Address:bc9725fc-be77-246e-2088-b1eac590f446} {Suffrage:Voter ID:server-626098c1-2c32-5e1f-4d86-d508c1384c96 Address:626098c1-2c32-5e1f-4d86-d508c1384c96}]" +2026-11-29T10:29:47.301Z [INFO] server-bc9725fc-be77-246e-2088-b1eac590f446: entering follower state: follower="Node at bc9725fc-be77-246e-2088-b1eac590f446 [Follower]" leader-address= leader-id= +2026-11-29T10:29:47.301Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:47.301Z [INFO] server-626098c1-2c32-5e1f-4d86-d508c1384c96: entering follower state: follower="Node at 626098c1-2c32-5e1f-4d86-d508c1384c96 [Follower]" leader-address= leader-id= +2026-11-29T10:29:47.390Z [WARN] server-626098c1-2c32-5e1f-4d86-d508c1384c96: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:47.390Z [INFO] server-626098c1-2c32-5e1f-4d86-d508c1384c96: entering candidate state: node="Node at 626098c1-2c32-5e1f-4d86-d508c1384c96 [Candidate]" term=2 +2026-11-29T10:29:47.390Z [INFO] server-626098c1-2c32-5e1f-4d86-d508c1384c96: election won: term=2 tally=2 +2026-11-29T10:29:47.390Z [INFO] server-626098c1-2c32-5e1f-4d86-d508c1384c96: entering leader state: leader="Node at 626098c1-2c32-5e1f-4d86-d508c1384c96 [Leader]" +2026-11-29T10:29:47.390Z [INFO] server-626098c1-2c32-5e1f-4d86-d508c1384c96: added peer, starting replication: peer=server-bc9725fc-be77-246e-2088-b1eac590f446 +2026-11-29T10:29:47.390Z [INFO] server-626098c1-2c32-5e1f-4d86-d508c1384c96: pipelining replication: peer="{Voter server-bc9725fc-be77-246e-2088-b1eac590f446 bc9725fc-be77-246e-2088-b1eac590f446}" +2026-11-29T10:29:47.495Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:47.390582659 +0000 UTC m=+98.125636761 (1 nodes), 88.9796ms from start of poll, 89.14704ms from cluster start. Timeout at 2026-11-29 10:29:47.495589019 +0000 UTC m=+98.230643121, 105.00636ms after stability +2026-11-29T10:29:47.501Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:47.607Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:47.501581539 +0000 UTC m=+98.236635641 (1 nodes), 2.04µs from start of poll, 200.14592ms from cluster start. Timeout at 2026-11-29 10:29:47.606601859 +0000 UTC m=+98.341655961, 105.02032ms after stability + raft_test.go:1799: [INFO] Disconnecting Node at bc9725fc-be77-246e-2088-b1eac590f446 [Follower] +2026-11-29T10:29:47.614Z [ERROR] server-626098c1-2c32-5e1f-4d86-d508c1384c96: failed to heartbeat to: peer=bc9725fc-be77-246e-2088-b1eac590f446 backoff time=10ms error="failed to connect to peer: bc9725fc-be77-246e-2088-b1eac590f446" +2026-11-29T10:29:47.615Z [ERROR] server-626098c1-2c32-5e1f-4d86-d508c1384c96: failed to pipeline appendEntries: peer="{Voter server-bc9725fc-be77-246e-2088-b1eac590f446 bc9725fc-be77-246e-2088-b1eac590f446}" error="append pipeline closed" +2026-11-29T10:29:47.615Z [INFO] server-626098c1-2c32-5e1f-4d86-d508c1384c96: aborting pipeline replication: peer="{Voter server-bc9725fc-be77-246e-2088-b1eac590f446 bc9725fc-be77-246e-2088-b1eac590f446}" +2026-11-29T10:29:47.624Z [ERROR] server-626098c1-2c32-5e1f-4d86-d508c1384c96: failed to appendEntries to: peer="{Voter server-bc9725fc-be77-246e-2088-b1eac590f446 bc9725fc-be77-246e-2088-b1eac590f446}" error="failed to connect to peer: bc9725fc-be77-246e-2088-b1eac590f446" +2026-11-29T10:29:47.634Z [ERROR] server-626098c1-2c32-5e1f-4d86-d508c1384c96: failed to heartbeat to: peer=bc9725fc-be77-246e-2088-b1eac590f446 backoff time=10ms error="failed to connect to peer: bc9725fc-be77-246e-2088-b1eac590f446" +2026-11-29T10:29:47.644Z [ERROR] server-626098c1-2c32-5e1f-4d86-d508c1384c96: failed to appendEntries to: peer="{Voter server-bc9725fc-be77-246e-2088-b1eac590f446 bc9725fc-be77-246e-2088-b1eac590f446}" error="failed to connect to peer: bc9725fc-be77-246e-2088-b1eac590f446" +2026-11-29T10:29:47.654Z [ERROR] server-626098c1-2c32-5e1f-4d86-d508c1384c96: failed to heartbeat to: peer=bc9725fc-be77-246e-2088-b1eac590f446 backoff time=10ms error="failed to connect to peer: bc9725fc-be77-246e-2088-b1eac590f446" +2026-11-29T10:29:47.657Z [WARN] server-626098c1-2c32-5e1f-4d86-d508c1384c96: failed to contact: server-id=server-bc9725fc-be77-246e-2088-b1eac590f446 time=50.66828ms +2026-11-29T10:29:47.657Z [WARN] server-626098c1-2c32-5e1f-4d86-d508c1384c96: failed to contact quorum of nodes, stepping down +2026-11-29T10:29:47.657Z [INFO] server-626098c1-2c32-5e1f-4d86-d508c1384c96: entering follower state: follower="Node at 626098c1-2c32-5e1f-4d86-d508c1384c96 [Follower]" leader-address= leader-id= +2026-11-29T10:29:47.657Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:47.664Z [ERROR] server-626098c1-2c32-5e1f-4d86-d508c1384c96: failed to appendEntries to: peer="{Voter server-bc9725fc-be77-246e-2088-b1eac590f446 bc9725fc-be77-246e-2088-b1eac590f446}" error="failed to connect to peer: bc9725fc-be77-246e-2088-b1eac590f446" +2026-11-29T10:29:47.686Z [WARN] server-bc9725fc-be77-246e-2088-b1eac590f446: heartbeat timeout reached, starting election: last-leader-addr=626098c1-2c32-5e1f-4d86-d508c1384c96 last-leader-id=server-626098c1-2c32-5e1f-4d86-d508c1384c96 +2026-11-29T10:29:47.686Z [INFO] server-bc9725fc-be77-246e-2088-b1eac590f446: entering candidate state: node="Node at bc9725fc-be77-246e-2088-b1eac590f446 [Candidate]" term=3 +2026-11-29T10:29:47.686Z [ERROR] server-bc9725fc-be77-246e-2088-b1eac590f446: failed to make requestVote RPC: target="{Voter server-626098c1-2c32-5e1f-4d86-d508c1384c96 626098c1-2c32-5e1f-4d86-d508c1384c96}" error="failed to connect to peer: 626098c1-2c32-5e1f-4d86-d508c1384c96" term=3 +2026-11-29T10:29:47.743Z [WARN] server-626098c1-2c32-5e1f-4d86-d508c1384c96: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:47.743Z [INFO] server-626098c1-2c32-5e1f-4d86-d508c1384c96: entering candidate state: node="Node at 626098c1-2c32-5e1f-4d86-d508c1384c96 [Candidate]" term=3 +2026-11-29T10:29:47.743Z [ERROR] server-626098c1-2c32-5e1f-4d86-d508c1384c96: failed to make requestVote RPC: target="{Voter server-bc9725fc-be77-246e-2088-b1eac590f446 bc9725fc-be77-246e-2088-b1eac590f446}" error="failed to connect to peer: bc9725fc-be77-246e-2088-b1eac590f446" term=3 +2026-11-29T10:29:47.781Z [WARN] server-bc9725fc-be77-246e-2088-b1eac590f446: Election timeout reached, restarting election +2026-11-29T10:29:47.781Z [INFO] server-bc9725fc-be77-246e-2088-b1eac590f446: entering candidate state: node="Node at bc9725fc-be77-246e-2088-b1eac590f446 [Candidate]" term=4 +2026-11-29T10:29:47.781Z [ERROR] server-bc9725fc-be77-246e-2088-b1eac590f446: failed to make requestVote RPC: target="{Voter server-626098c1-2c32-5e1f-4d86-d508c1384c96 626098c1-2c32-5e1f-4d86-d508c1384c96}" error="failed to connect to peer: 626098c1-2c32-5e1f-4d86-d508c1384c96" term=4 +2026-11-29T10:29:47.796Z [WARN] server-626098c1-2c32-5e1f-4d86-d508c1384c96: Election timeout reached, restarting election +2026-11-29T10:29:47.796Z [INFO] server-626098c1-2c32-5e1f-4d86-d508c1384c96: entering candidate state: node="Node at 626098c1-2c32-5e1f-4d86-d508c1384c96 [Candidate]" term=4 +2026-11-29T10:29:47.796Z [ERROR] server-626098c1-2c32-5e1f-4d86-d508c1384c96: failed to make requestVote RPC: target="{Voter server-bc9725fc-be77-246e-2088-b1eac590f446 bc9725fc-be77-246e-2088-b1eac590f446}" error="failed to connect to peer: bc9725fc-be77-246e-2088-b1eac590f446" term=4 +2026-11-29T10:29:47.850Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:47.743152979 +0000 UTC m=+98.478207081 (0 nodes), 85.31816ms from start of poll, 441.71736ms from cluster start. Timeout at 2026-11-29 10:29:47.848158299 +0000 UTC m=+98.583212401, 105.00532ms after stability +2026-11-29T10:29:47.863Z [WARN] server-bc9725fc-be77-246e-2088-b1eac590f446: Election timeout reached, restarting election +2026-11-29T10:29:47.863Z [INFO] server-bc9725fc-be77-246e-2088-b1eac590f446: entering candidate state: node="Node at bc9725fc-be77-246e-2088-b1eac590f446 [Candidate]" term=5 +2026-11-29T10:29:47.863Z [ERROR] server-bc9725fc-be77-246e-2088-b1eac590f446: failed to make requestVote RPC: target="{Voter server-626098c1-2c32-5e1f-4d86-d508c1384c96 626098c1-2c32-5e1f-4d86-d508c1384c96}" error="failed to connect to peer: 626098c1-2c32-5e1f-4d86-d508c1384c96" term=5 +2026-11-29T10:29:47.879Z [WARN] server-626098c1-2c32-5e1f-4d86-d508c1384c96: Election timeout reached, restarting election +2026-11-29T10:29:47.879Z [INFO] server-626098c1-2c32-5e1f-4d86-d508c1384c96: entering candidate state: node="Node at 626098c1-2c32-5e1f-4d86-d508c1384c96 [Candidate]" term=5 +2026-11-29T10:29:47.879Z [ERROR] server-626098c1-2c32-5e1f-4d86-d508c1384c96: failed to make requestVote RPC: target="{Voter server-bc9725fc-be77-246e-2088-b1eac590f446 bc9725fc-be77-246e-2088-b1eac590f446}" error="failed to connect to peer: bc9725fc-be77-246e-2088-b1eac590f446" term=5 +2026-11-29T10:29:47.949Z [WARN] server-bc9725fc-be77-246e-2088-b1eac590f446: Election timeout reached, restarting election +2026-11-29T10:29:47.949Z [INFO] server-bc9725fc-be77-246e-2088-b1eac590f446: entering candidate state: node="Node at bc9725fc-be77-246e-2088-b1eac590f446 [Candidate]" term=6 +2026-11-29T10:29:47.949Z [ERROR] server-bc9725fc-be77-246e-2088-b1eac590f446: failed to make requestVote RPC: target="{Voter server-626098c1-2c32-5e1f-4d86-d508c1384c96 626098c1-2c32-5e1f-4d86-d508c1384c96}" error="failed to connect to peer: 626098c1-2c32-5e1f-4d86-d508c1384c96" term=6 +2026-11-29T10:29:47.951Z [WARN] server-626098c1-2c32-5e1f-4d86-d508c1384c96: Election timeout reached, restarting election +2026-11-29T10:29:47.951Z [INFO] server-626098c1-2c32-5e1f-4d86-d508c1384c96: entering candidate state: node="Node at 626098c1-2c32-5e1f-4d86-d508c1384c96 [Candidate]" term=6 +2026-11-29T10:29:47.951Z [ERROR] server-626098c1-2c32-5e1f-4d86-d508c1384c96: failed to make requestVote RPC: target="{Voter server-bc9725fc-be77-246e-2088-b1eac590f446 bc9725fc-be77-246e-2088-b1eac590f446}" error="failed to connect to peer: bc9725fc-be77-246e-2088-b1eac590f446" term=6 +--- PASS: TestRaft_LeaderLeaseExpire (0.65s) === RUN TestRaft_Barrier -2025-10-27T04:03:45.536Z [INFO] server-347ac97e-5d91-8238-8b87-7605d8c2acd8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-347ac97e-5d91-8238-8b87-7605d8c2acd8 Address:347ac97e-5d91-8238-8b87-7605d8c2acd8} {Suffrage:Voter ID:server-2e2c034e-5a4b-8002-0594-ec30c5659105 Address:2e2c034e-5a4b-8002-0594-ec30c5659105} {Suffrage:Voter ID:server-e4c5de17-c627-4523-3057-2b26b2bb6201 Address:e4c5de17-c627-4523-3057-2b26b2bb6201}]" -2025-10-27T04:03:45.536Z [INFO] server-347ac97e-5d91-8238-8b87-7605d8c2acd8: entering follower state: follower="Node at 347ac97e-5d91-8238-8b87-7605d8c2acd8 [Follower]" leader-address= leader-id= -2025-10-27T04:03:45.536Z [INFO] server-2e2c034e-5a4b-8002-0594-ec30c5659105: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-347ac97e-5d91-8238-8b87-7605d8c2acd8 Address:347ac97e-5d91-8238-8b87-7605d8c2acd8} {Suffrage:Voter ID:server-2e2c034e-5a4b-8002-0594-ec30c5659105 Address:2e2c034e-5a4b-8002-0594-ec30c5659105} {Suffrage:Voter ID:server-e4c5de17-c627-4523-3057-2b26b2bb6201 Address:e4c5de17-c627-4523-3057-2b26b2bb6201}]" -2025-10-27T04:03:45.536Z [INFO] server-2e2c034e-5a4b-8002-0594-ec30c5659105: entering follower state: follower="Node at 2e2c034e-5a4b-8002-0594-ec30c5659105 [Follower]" leader-address= leader-id= -2025-10-27T04:03:45.536Z [INFO] server-e4c5de17-c627-4523-3057-2b26b2bb6201: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-347ac97e-5d91-8238-8b87-7605d8c2acd8 Address:347ac97e-5d91-8238-8b87-7605d8c2acd8} {Suffrage:Voter ID:server-2e2c034e-5a4b-8002-0594-ec30c5659105 Address:2e2c034e-5a4b-8002-0594-ec30c5659105} {Suffrage:Voter ID:server-e4c5de17-c627-4523-3057-2b26b2bb6201 Address:e4c5de17-c627-4523-3057-2b26b2bb6201}]" -2025-10-27T04:03:45.536Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:45.536Z [INFO] server-e4c5de17-c627-4523-3057-2b26b2bb6201: entering follower state: follower="Node at e4c5de17-c627-4523-3057-2b26b2bb6201 [Follower]" leader-address= leader-id= -2025-10-27T04:03:45.619Z [WARN] server-347ac97e-5d91-8238-8b87-7605d8c2acd8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:45.619Z [INFO] server-347ac97e-5d91-8238-8b87-7605d8c2acd8: entering candidate state: node="Node at 347ac97e-5d91-8238-8b87-7605d8c2acd8 [Candidate]" term=2 -2025-10-27T04:03:45.619Z [INFO] server-347ac97e-5d91-8238-8b87-7605d8c2acd8: election won: term=2 tally=2 -2025-10-27T04:03:45.619Z [INFO] server-347ac97e-5d91-8238-8b87-7605d8c2acd8: entering leader state: leader="Node at 347ac97e-5d91-8238-8b87-7605d8c2acd8 [Leader]" -2025-10-27T04:03:45.619Z [INFO] server-347ac97e-5d91-8238-8b87-7605d8c2acd8: added peer, starting replication: peer=server-2e2c034e-5a4b-8002-0594-ec30c5659105 -2025-10-27T04:03:45.619Z [INFO] server-347ac97e-5d91-8238-8b87-7605d8c2acd8: added peer, starting replication: peer=server-e4c5de17-c627-4523-3057-2b26b2bb6201 -2025-10-27T04:03:45.619Z [INFO] server-347ac97e-5d91-8238-8b87-7605d8c2acd8: pipelining replication: peer="{Voter server-e4c5de17-c627-4523-3057-2b26b2bb6201 e4c5de17-c627-4523-3057-2b26b2bb6201}" -2025-10-27T04:03:45.619Z [INFO] server-347ac97e-5d91-8238-8b87-7605d8c2acd8: pipelining replication: peer="{Voter server-2e2c034e-5a4b-8002-0594-ec30c5659105 2e2c034e-5a4b-8002-0594-ec30c5659105}" -2025-10-27T04:03:45.725Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:45.619791115 +0000 UTC m=+100.614663633 (1 nodes), 82.916006ms from start of poll, 83.376969ms from cluster start. Timeout at 2025-10-27 04:03:45.724794675 +0000 UTC m=+100.719667113, 105.00348ms after stability -2025-10-27T04:03:45.747Z [INFO] server-347ac97e-5d91-8238-8b87-7605d8c2acd8: aborting pipeline replication: peer="{Voter server-e4c5de17-c627-4523-3057-2b26b2bb6201 e4c5de17-c627-4523-3057-2b26b2bb6201}" -2025-10-27T04:03:45.747Z [INFO] server-347ac97e-5d91-8238-8b87-7605d8c2acd8: aborting pipeline replication: peer="{Voter server-2e2c034e-5a4b-8002-0594-ec30c5659105 2e2c034e-5a4b-8002-0594-ec30c5659105}" ---- PASS: TestRaft_Barrier (0.21s) +2026-11-29T10:29:47.956Z [INFO] server-1587595c-9554-3f9d-6ebe-49761431a695: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1587595c-9554-3f9d-6ebe-49761431a695 Address:1587595c-9554-3f9d-6ebe-49761431a695} {Suffrage:Voter ID:server-c15ed5ea-ced0-b18c-dd9c-daa95e04814f Address:c15ed5ea-ced0-b18c-dd9c-daa95e04814f} {Suffrage:Voter ID:server-a474fc5c-4561-0ade-9e9a-7af9bdff2ff1 Address:a474fc5c-4561-0ade-9e9a-7af9bdff2ff1}]" +2026-11-29T10:29:47.956Z [INFO] server-c15ed5ea-ced0-b18c-dd9c-daa95e04814f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1587595c-9554-3f9d-6ebe-49761431a695 Address:1587595c-9554-3f9d-6ebe-49761431a695} {Suffrage:Voter ID:server-c15ed5ea-ced0-b18c-dd9c-daa95e04814f Address:c15ed5ea-ced0-b18c-dd9c-daa95e04814f} {Suffrage:Voter ID:server-a474fc5c-4561-0ade-9e9a-7af9bdff2ff1 Address:a474fc5c-4561-0ade-9e9a-7af9bdff2ff1}]" +2026-11-29T10:29:47.956Z [INFO] server-a474fc5c-4561-0ade-9e9a-7af9bdff2ff1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1587595c-9554-3f9d-6ebe-49761431a695 Address:1587595c-9554-3f9d-6ebe-49761431a695} {Suffrage:Voter ID:server-c15ed5ea-ced0-b18c-dd9c-daa95e04814f Address:c15ed5ea-ced0-b18c-dd9c-daa95e04814f} {Suffrage:Voter ID:server-a474fc5c-4561-0ade-9e9a-7af9bdff2ff1 Address:a474fc5c-4561-0ade-9e9a-7af9bdff2ff1}]" +2026-11-29T10:29:47.956Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:47.956Z [INFO] server-1587595c-9554-3f9d-6ebe-49761431a695: entering follower state: follower="Node at 1587595c-9554-3f9d-6ebe-49761431a695 [Follower]" leader-address= leader-id= +2026-11-29T10:29:47.956Z [INFO] server-c15ed5ea-ced0-b18c-dd9c-daa95e04814f: entering follower state: follower="Node at c15ed5ea-ced0-b18c-dd9c-daa95e04814f [Follower]" leader-address= leader-id= +2026-11-29T10:29:47.956Z [INFO] server-a474fc5c-4561-0ade-9e9a-7af9bdff2ff1: entering follower state: follower="Node at a474fc5c-4561-0ade-9e9a-7af9bdff2ff1 [Follower]" leader-address= leader-id= +2026-11-29T10:29:48.016Z [WARN] server-a474fc5c-4561-0ade-9e9a-7af9bdff2ff1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:48.016Z [INFO] server-a474fc5c-4561-0ade-9e9a-7af9bdff2ff1: entering candidate state: node="Node at a474fc5c-4561-0ade-9e9a-7af9bdff2ff1 [Candidate]" term=2 +2026-11-29T10:29:48.016Z [INFO] server-a474fc5c-4561-0ade-9e9a-7af9bdff2ff1: election won: term=2 tally=2 +2026-11-29T10:29:48.016Z [INFO] server-a474fc5c-4561-0ade-9e9a-7af9bdff2ff1: entering leader state: leader="Node at a474fc5c-4561-0ade-9e9a-7af9bdff2ff1 [Leader]" +2026-11-29T10:29:48.016Z [INFO] server-a474fc5c-4561-0ade-9e9a-7af9bdff2ff1: added peer, starting replication: peer=server-1587595c-9554-3f9d-6ebe-49761431a695 +2026-11-29T10:29:48.016Z [INFO] server-a474fc5c-4561-0ade-9e9a-7af9bdff2ff1: added peer, starting replication: peer=server-c15ed5ea-ced0-b18c-dd9c-daa95e04814f +2026-11-29T10:29:48.016Z [INFO] server-a474fc5c-4561-0ade-9e9a-7af9bdff2ff1: pipelining replication: peer="{Voter server-1587595c-9554-3f9d-6ebe-49761431a695 1587595c-9554-3f9d-6ebe-49761431a695}" +2026-11-29T10:29:48.016Z [INFO] server-a474fc5c-4561-0ade-9e9a-7af9bdff2ff1: pipelining replication: peer="{Voter server-c15ed5ea-ced0-b18c-dd9c-daa95e04814f c15ed5ea-ced0-b18c-dd9c-daa95e04814f}" +2026-11-29T10:29:48.122Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:48.016825619 +0000 UTC m=+98.751879681 (1 nodes), 60.579ms from start of poll, 60.80724ms from cluster start. Timeout at 2026-11-29 10:29:48.121828459 +0000 UTC m=+98.856882561, 105.00288ms after stability +2026-11-29T10:29:48.124Z [INFO] server-a474fc5c-4561-0ade-9e9a-7af9bdff2ff1: aborting pipeline replication: peer="{Voter server-c15ed5ea-ced0-b18c-dd9c-daa95e04814f c15ed5ea-ced0-b18c-dd9c-daa95e04814f}" +2026-11-29T10:29:48.125Z [INFO] server-a474fc5c-4561-0ade-9e9a-7af9bdff2ff1: aborting pipeline replication: peer="{Voter server-1587595c-9554-3f9d-6ebe-49761431a695 1587595c-9554-3f9d-6ebe-49761431a695}" +--- PASS: TestRaft_Barrier (0.17s) === RUN TestRaft_VerifyLeader -2025-10-27T04:03:45.748Z [INFO] server-b6f559fb-4fa3-d4bd-839b-3e8ada398f53: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b6f559fb-4fa3-d4bd-839b-3e8ada398f53 Address:b6f559fb-4fa3-d4bd-839b-3e8ada398f53} {Suffrage:Voter ID:server-f8ffda1d-07df-0595-a285-8d39d5532dc5 Address:f8ffda1d-07df-0595-a285-8d39d5532dc5} {Suffrage:Voter ID:server-b6c7dc14-406e-e3a2-3cb2-1017acfe07d9 Address:b6c7dc14-406e-e3a2-3cb2-1017acfe07d9}]" -2025-10-27T04:03:45.748Z [INFO] server-b6f559fb-4fa3-d4bd-839b-3e8ada398f53: entering follower state: follower="Node at b6f559fb-4fa3-d4bd-839b-3e8ada398f53 [Follower]" leader-address= leader-id= -2025-10-27T04:03:45.748Z [INFO] server-f8ffda1d-07df-0595-a285-8d39d5532dc5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b6f559fb-4fa3-d4bd-839b-3e8ada398f53 Address:b6f559fb-4fa3-d4bd-839b-3e8ada398f53} {Suffrage:Voter ID:server-f8ffda1d-07df-0595-a285-8d39d5532dc5 Address:f8ffda1d-07df-0595-a285-8d39d5532dc5} {Suffrage:Voter ID:server-b6c7dc14-406e-e3a2-3cb2-1017acfe07d9 Address:b6c7dc14-406e-e3a2-3cb2-1017acfe07d9}]" -2025-10-27T04:03:45.748Z [INFO] server-f8ffda1d-07df-0595-a285-8d39d5532dc5: entering follower state: follower="Node at f8ffda1d-07df-0595-a285-8d39d5532dc5 [Follower]" leader-address= leader-id= -2025-10-27T04:03:45.748Z [INFO] server-b6c7dc14-406e-e3a2-3cb2-1017acfe07d9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b6f559fb-4fa3-d4bd-839b-3e8ada398f53 Address:b6f559fb-4fa3-d4bd-839b-3e8ada398f53} {Suffrage:Voter ID:server-f8ffda1d-07df-0595-a285-8d39d5532dc5 Address:f8ffda1d-07df-0595-a285-8d39d5532dc5} {Suffrage:Voter ID:server-b6c7dc14-406e-e3a2-3cb2-1017acfe07d9 Address:b6c7dc14-406e-e3a2-3cb2-1017acfe07d9}]" -2025-10-27T04:03:45.748Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:45.748Z [INFO] server-b6c7dc14-406e-e3a2-3cb2-1017acfe07d9: entering follower state: follower="Node at b6c7dc14-406e-e3a2-3cb2-1017acfe07d9 [Follower]" leader-address= leader-id= -2025-10-27T04:03:45.823Z [WARN] server-b6c7dc14-406e-e3a2-3cb2-1017acfe07d9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:45.823Z [INFO] server-b6c7dc14-406e-e3a2-3cb2-1017acfe07d9: entering candidate state: node="Node at b6c7dc14-406e-e3a2-3cb2-1017acfe07d9 [Candidate]" term=2 -2025-10-27T04:03:45.823Z [INFO] server-b6c7dc14-406e-e3a2-3cb2-1017acfe07d9: election won: term=2 tally=2 -2025-10-27T04:03:45.823Z [INFO] server-b6c7dc14-406e-e3a2-3cb2-1017acfe07d9: entering leader state: leader="Node at b6c7dc14-406e-e3a2-3cb2-1017acfe07d9 [Leader]" -2025-10-27T04:03:45.823Z [INFO] server-b6c7dc14-406e-e3a2-3cb2-1017acfe07d9: added peer, starting replication: peer=server-b6f559fb-4fa3-d4bd-839b-3e8ada398f53 -2025-10-27T04:03:45.823Z [INFO] server-b6c7dc14-406e-e3a2-3cb2-1017acfe07d9: added peer, starting replication: peer=server-f8ffda1d-07df-0595-a285-8d39d5532dc5 -2025-10-27T04:03:45.823Z [INFO] server-b6c7dc14-406e-e3a2-3cb2-1017acfe07d9: pipelining replication: peer="{Voter server-b6f559fb-4fa3-d4bd-839b-3e8ada398f53 b6f559fb-4fa3-d4bd-839b-3e8ada398f53}" -2025-10-27T04:03:45.823Z [INFO] server-b6c7dc14-406e-e3a2-3cb2-1017acfe07d9: pipelining replication: peer="{Voter server-f8ffda1d-07df-0595-a285-8d39d5532dc5 f8ffda1d-07df-0595-a285-8d39d5532dc5}" -2025-10-27T04:03:45.928Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:45.823622988 +0000 UTC m=+100.818495466 (1 nodes), 75.125793ms from start of poll, 75.459316ms from cluster start. Timeout at 2025-10-27 04:03:45.928624188 +0000 UTC m=+100.923496626, 105.00116ms after stability -2025-10-27T04:03:45.928Z [INFO] server-b6c7dc14-406e-e3a2-3cb2-1017acfe07d9: aborting pipeline replication: peer="{Voter server-b6f559fb-4fa3-d4bd-839b-3e8ada398f53 b6f559fb-4fa3-d4bd-839b-3e8ada398f53}" -2025-10-27T04:03:45.928Z [INFO] server-b6c7dc14-406e-e3a2-3cb2-1017acfe07d9: aborting pipeline replication: peer="{Voter server-f8ffda1d-07df-0595-a285-8d39d5532dc5 f8ffda1d-07df-0595-a285-8d39d5532dc5}" ---- PASS: TestRaft_VerifyLeader (0.18s) +2026-11-29T10:29:48.125Z [INFO] server-a60860a1-4a78-5fa8-6a2c-e5f17682640f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a60860a1-4a78-5fa8-6a2c-e5f17682640f Address:a60860a1-4a78-5fa8-6a2c-e5f17682640f} {Suffrage:Voter ID:server-937ed8f3-21fe-0b11-843b-068aa7232e29 Address:937ed8f3-21fe-0b11-843b-068aa7232e29} {Suffrage:Voter ID:server-032ff65e-7af9-d558-3ee0-dd182dcc11ce Address:032ff65e-7af9-d558-3ee0-dd182dcc11ce}]" +2026-11-29T10:29:48.125Z [INFO] server-937ed8f3-21fe-0b11-843b-068aa7232e29: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a60860a1-4a78-5fa8-6a2c-e5f17682640f Address:a60860a1-4a78-5fa8-6a2c-e5f17682640f} {Suffrage:Voter ID:server-937ed8f3-21fe-0b11-843b-068aa7232e29 Address:937ed8f3-21fe-0b11-843b-068aa7232e29} {Suffrage:Voter ID:server-032ff65e-7af9-d558-3ee0-dd182dcc11ce Address:032ff65e-7af9-d558-3ee0-dd182dcc11ce}]" +2026-11-29T10:29:48.125Z [INFO] server-032ff65e-7af9-d558-3ee0-dd182dcc11ce: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a60860a1-4a78-5fa8-6a2c-e5f17682640f Address:a60860a1-4a78-5fa8-6a2c-e5f17682640f} {Suffrage:Voter ID:server-937ed8f3-21fe-0b11-843b-068aa7232e29 Address:937ed8f3-21fe-0b11-843b-068aa7232e29} {Suffrage:Voter ID:server-032ff65e-7af9-d558-3ee0-dd182dcc11ce Address:032ff65e-7af9-d558-3ee0-dd182dcc11ce}]" +2026-11-29T10:29:48.125Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:48.125Z [INFO] server-a60860a1-4a78-5fa8-6a2c-e5f17682640f: entering follower state: follower="Node at a60860a1-4a78-5fa8-6a2c-e5f17682640f [Follower]" leader-address= leader-id= +2026-11-29T10:29:48.125Z [INFO] server-937ed8f3-21fe-0b11-843b-068aa7232e29: entering follower state: follower="Node at 937ed8f3-21fe-0b11-843b-068aa7232e29 [Follower]" leader-address= leader-id= +2026-11-29T10:29:48.125Z [INFO] server-032ff65e-7af9-d558-3ee0-dd182dcc11ce: entering follower state: follower="Node at 032ff65e-7af9-d558-3ee0-dd182dcc11ce [Follower]" leader-address= leader-id= +2026-11-29T10:29:48.188Z [WARN] server-032ff65e-7af9-d558-3ee0-dd182dcc11ce: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:48.188Z [INFO] server-032ff65e-7af9-d558-3ee0-dd182dcc11ce: entering candidate state: node="Node at 032ff65e-7af9-d558-3ee0-dd182dcc11ce [Candidate]" term=2 +2026-11-29T10:29:48.188Z [INFO] server-032ff65e-7af9-d558-3ee0-dd182dcc11ce: election won: term=2 tally=2 +2026-11-29T10:29:48.188Z [INFO] server-032ff65e-7af9-d558-3ee0-dd182dcc11ce: entering leader state: leader="Node at 032ff65e-7af9-d558-3ee0-dd182dcc11ce [Leader]" +2026-11-29T10:29:48.188Z [INFO] server-032ff65e-7af9-d558-3ee0-dd182dcc11ce: added peer, starting replication: peer=server-a60860a1-4a78-5fa8-6a2c-e5f17682640f +2026-11-29T10:29:48.188Z [INFO] server-032ff65e-7af9-d558-3ee0-dd182dcc11ce: added peer, starting replication: peer=server-937ed8f3-21fe-0b11-843b-068aa7232e29 +2026-11-29T10:29:48.189Z [INFO] server-032ff65e-7af9-d558-3ee0-dd182dcc11ce: pipelining replication: peer="{Voter server-937ed8f3-21fe-0b11-843b-068aa7232e29 937ed8f3-21fe-0b11-843b-068aa7232e29}" +2026-11-29T10:29:48.189Z [INFO] server-032ff65e-7af9-d558-3ee0-dd182dcc11ce: pipelining replication: peer="{Voter server-a60860a1-4a78-5fa8-6a2c-e5f17682640f a60860a1-4a78-5fa8-6a2c-e5f17682640f}" +2026-11-29T10:29:48.294Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:48.189012779 +0000 UTC m=+98.924066921 (1 nodes), 63.48868ms from start of poll, 63.65544ms from cluster start. Timeout at 2026-11-29 10:29:48.294014099 +0000 UTC m=+99.029068201, 105.00128ms after stability +2026-11-29T10:29:48.294Z [INFO] server-032ff65e-7af9-d558-3ee0-dd182dcc11ce: aborting pipeline replication: peer="{Voter server-937ed8f3-21fe-0b11-843b-068aa7232e29 937ed8f3-21fe-0b11-843b-068aa7232e29}" +2026-11-29T10:29:48.294Z [INFO] server-032ff65e-7af9-d558-3ee0-dd182dcc11ce: aborting pipeline replication: peer="{Voter server-a60860a1-4a78-5fa8-6a2c-e5f17682640f a60860a1-4a78-5fa8-6a2c-e5f17682640f}" +2026-11-29T10:29:48.795Z [ERROR] server-032ff65e-7af9-d558-3ee0-dd182dcc11ce: failed to heartbeat to: peer=a60860a1-4a78-5fa8-6a2c-e5f17682640f backoff time=10ms error="command timed out" +--- PASS: TestRaft_VerifyLeader (0.67s) === RUN TestRaft_VerifyLeader_Single -2025-10-27T04:03:45.929Z [INFO] server-d01a6415-0f20-3162-d393-cfcd56832f24: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d01a6415-0f20-3162-d393-cfcd56832f24 Address:d01a6415-0f20-3162-d393-cfcd56832f24}]" -2025-10-27T04:03:45.929Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:45.929Z [INFO] server-d01a6415-0f20-3162-d393-cfcd56832f24: entering follower state: follower="Node at d01a6415-0f20-3162-d393-cfcd56832f24 [Follower]" leader-address= leader-id= -2025-10-27T04:03:45.995Z [WARN] server-d01a6415-0f20-3162-d393-cfcd56832f24: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:45.995Z [INFO] server-d01a6415-0f20-3162-d393-cfcd56832f24: entering candidate state: node="Node at d01a6415-0f20-3162-d393-cfcd56832f24 [Candidate]" term=2 -2025-10-27T04:03:45.995Z [INFO] server-d01a6415-0f20-3162-d393-cfcd56832f24: election won: term=2 tally=1 -2025-10-27T04:03:45.995Z [INFO] server-d01a6415-0f20-3162-d393-cfcd56832f24: entering leader state: leader="Node at d01a6415-0f20-3162-d393-cfcd56832f24 [Leader]" -2025-10-27T04:03:46.100Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:45.995559562 +0000 UTC m=+100.990432000 (1 nodes), 65.94285ms from start of poll, 66.096931ms from cluster start. Timeout at 2025-10-27 04:03:46.100575042 +0000 UTC m=+101.095447520, 105.01552ms after stability ---- PASS: TestRaft_VerifyLeader_Single (0.17s) +2026-11-29T10:29:48.795Z [INFO] server-4a56dd67-4e41-6c3e-504b-d763f69e729b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4a56dd67-4e41-6c3e-504b-d763f69e729b Address:4a56dd67-4e41-6c3e-504b-d763f69e729b}]" +2026-11-29T10:29:48.795Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:48.795Z [INFO] server-4a56dd67-4e41-6c3e-504b-d763f69e729b: entering follower state: follower="Node at 4a56dd67-4e41-6c3e-504b-d763f69e729b [Follower]" leader-address= leader-id= +2026-11-29T10:29:48.877Z [WARN] server-4a56dd67-4e41-6c3e-504b-d763f69e729b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:48.877Z [INFO] server-4a56dd67-4e41-6c3e-504b-d763f69e729b: entering candidate state: node="Node at 4a56dd67-4e41-6c3e-504b-d763f69e729b [Candidate]" term=2 +2026-11-29T10:29:48.877Z [INFO] server-4a56dd67-4e41-6c3e-504b-d763f69e729b: election won: term=2 tally=1 +2026-11-29T10:29:48.877Z [INFO] server-4a56dd67-4e41-6c3e-504b-d763f69e729b: entering leader state: leader="Node at 4a56dd67-4e41-6c3e-504b-d763f69e729b [Leader]" +2026-11-29T10:29:48.982Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:48.877395459 +0000 UTC m=+99.612449561 (1 nodes), 81.56552ms from start of poll, 81.67076ms from cluster start. Timeout at 2026-11-29 10:29:48.982396899 +0000 UTC m=+99.717450961, 105.0014ms after stability +--- PASS: TestRaft_VerifyLeader_Single (0.19s) === RUN TestRaft_VerifyLeader_Fail -2025-10-27T04:03:46.101Z [INFO] server-1e20c082-f202-0215-8cf6-5b7ca21526c2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1e20c082-f202-0215-8cf6-5b7ca21526c2 Address:1e20c082-f202-0215-8cf6-5b7ca21526c2} {Suffrage:Voter ID:server-b90f2c9c-5cfe-1472-8211-d28518a72621 Address:b90f2c9c-5cfe-1472-8211-d28518a72621}]" -2025-10-27T04:03:46.101Z [INFO] server-b90f2c9c-5cfe-1472-8211-d28518a72621: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1e20c082-f202-0215-8cf6-5b7ca21526c2 Address:1e20c082-f202-0215-8cf6-5b7ca21526c2} {Suffrage:Voter ID:server-b90f2c9c-5cfe-1472-8211-d28518a72621 Address:b90f2c9c-5cfe-1472-8211-d28518a72621}]" -2025-10-27T04:03:46.101Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:46.101Z [INFO] server-1e20c082-f202-0215-8cf6-5b7ca21526c2: entering follower state: follower="Node at 1e20c082-f202-0215-8cf6-5b7ca21526c2 [Follower]" leader-address= leader-id= -2025-10-27T04:03:46.101Z [INFO] server-b90f2c9c-5cfe-1472-8211-d28518a72621: entering follower state: follower="Node at b90f2c9c-5cfe-1472-8211-d28518a72621 [Follower]" leader-address= leader-id= -2025-10-27T04:03:46.153Z [WARN] server-b90f2c9c-5cfe-1472-8211-d28518a72621: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:46.153Z [INFO] server-b90f2c9c-5cfe-1472-8211-d28518a72621: entering candidate state: node="Node at b90f2c9c-5cfe-1472-8211-d28518a72621 [Candidate]" term=2 -2025-10-27T04:03:46.153Z [INFO] server-b90f2c9c-5cfe-1472-8211-d28518a72621: election won: term=2 tally=2 -2025-10-27T04:03:46.153Z [INFO] server-b90f2c9c-5cfe-1472-8211-d28518a72621: entering leader state: leader="Node at b90f2c9c-5cfe-1472-8211-d28518a72621 [Leader]" -2025-10-27T04:03:46.153Z [INFO] server-b90f2c9c-5cfe-1472-8211-d28518a72621: added peer, starting replication: peer=server-1e20c082-f202-0215-8cf6-5b7ca21526c2 -2025-10-27T04:03:46.153Z [INFO] server-b90f2c9c-5cfe-1472-8211-d28518a72621: pipelining replication: peer="{Voter server-1e20c082-f202-0215-8cf6-5b7ca21526c2 1e20c082-f202-0215-8cf6-5b7ca21526c2}" -2025-10-27T04:03:46.258Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:46.153502201 +0000 UTC m=+101.148374679 (1 nodes), 51.867514ms from start of poll, 52.184916ms from cluster start. Timeout at 2025-10-27 04:03:46.258518441 +0000 UTC m=+101.253390919, 105.01624ms after stability -2025-10-27T04:03:46.258Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:03:46.364Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:03:46.25872932 +0000 UTC m=+101.253601798 (1 nodes), 2.48µs from start of poll, 157.412035ms from cluster start. Timeout at 2025-10-27 04:03:46.36374472 +0000 UTC m=+101.358617238, 105.01544ms after stability -2025-10-27T04:03:46.370Z [ERROR] server-b90f2c9c-5cfe-1472-8211-d28518a72621: peer has newer term, stopping replication: peer="{Voter server-1e20c082-f202-0215-8cf6-5b7ca21526c2 1e20c082-f202-0215-8cf6-5b7ca21526c2}" -2025-10-27T04:03:46.370Z [INFO] server-b90f2c9c-5cfe-1472-8211-d28518a72621: aborting pipeline replication: peer="{Voter server-1e20c082-f202-0215-8cf6-5b7ca21526c2 1e20c082-f202-0215-8cf6-5b7ca21526c2}" -2025-10-27T04:03:46.370Z [INFO] server-b90f2c9c-5cfe-1472-8211-d28518a72621: entering follower state: follower="Node at b90f2c9c-5cfe-1472-8211-d28518a72621 [Follower]" leader-address= leader-id= ---- PASS: TestRaft_VerifyLeader_Fail (0.27s) +2026-11-29T10:29:48.982Z [INFO] server-03cf2b88-c2d5-405f-bc4f-012f4d4e2fc8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-03cf2b88-c2d5-405f-bc4f-012f4d4e2fc8 Address:03cf2b88-c2d5-405f-bc4f-012f4d4e2fc8} {Suffrage:Voter ID:server-558e7ffe-73d3-c8a4-d34e-7ff6898107fd Address:558e7ffe-73d3-c8a4-d34e-7ff6898107fd}]" +2026-11-29T10:29:48.982Z [INFO] server-558e7ffe-73d3-c8a4-d34e-7ff6898107fd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-03cf2b88-c2d5-405f-bc4f-012f4d4e2fc8 Address:03cf2b88-c2d5-405f-bc4f-012f4d4e2fc8} {Suffrage:Voter ID:server-558e7ffe-73d3-c8a4-d34e-7ff6898107fd Address:558e7ffe-73d3-c8a4-d34e-7ff6898107fd}]" +2026-11-29T10:29:48.982Z [INFO] server-03cf2b88-c2d5-405f-bc4f-012f4d4e2fc8: entering follower state: follower="Node at 03cf2b88-c2d5-405f-bc4f-012f4d4e2fc8 [Follower]" leader-address= leader-id= +2026-11-29T10:29:48.982Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:48.982Z [INFO] server-558e7ffe-73d3-c8a4-d34e-7ff6898107fd: entering follower state: follower="Node at 558e7ffe-73d3-c8a4-d34e-7ff6898107fd [Follower]" leader-address= leader-id= +2026-11-29T10:29:49.046Z [WARN] server-558e7ffe-73d3-c8a4-d34e-7ff6898107fd: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:49.046Z [INFO] server-558e7ffe-73d3-c8a4-d34e-7ff6898107fd: entering candidate state: node="Node at 558e7ffe-73d3-c8a4-d34e-7ff6898107fd [Candidate]" term=2 +2026-11-29T10:29:49.046Z [INFO] server-558e7ffe-73d3-c8a4-d34e-7ff6898107fd: election won: term=2 tally=2 +2026-11-29T10:29:49.046Z [INFO] server-558e7ffe-73d3-c8a4-d34e-7ff6898107fd: entering leader state: leader="Node at 558e7ffe-73d3-c8a4-d34e-7ff6898107fd [Leader]" +2026-11-29T10:29:49.046Z [INFO] server-558e7ffe-73d3-c8a4-d34e-7ff6898107fd: added peer, starting replication: peer=server-03cf2b88-c2d5-405f-bc4f-012f4d4e2fc8 +2026-11-29T10:29:49.046Z [INFO] server-558e7ffe-73d3-c8a4-d34e-7ff6898107fd: pipelining replication: peer="{Voter server-03cf2b88-c2d5-405f-bc4f-012f4d4e2fc8 03cf2b88-c2d5-405f-bc4f-012f4d4e2fc8}" +2026-11-29T10:29:49.152Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:49.046600779 +0000 UTC m=+99.781654881 (1 nodes), 63.61248ms from start of poll, 63.78728ms from cluster start. Timeout at 2026-11-29 10:29:49.151601739 +0000 UTC m=+99.886655841, 105.00096ms after stability +2026-11-29T10:29:49.152Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:49.258Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:49.152616219 +0000 UTC m=+99.887670321 (1 nodes), 1.92µs from start of poll, 169.80272ms from cluster start. Timeout at 2026-11-29 10:29:49.257634499 +0000 UTC m=+99.992688601, 105.01828ms after stability +2026-11-29T10:29:49.265Z [ERROR] server-558e7ffe-73d3-c8a4-d34e-7ff6898107fd: peer has newer term, stopping replication: peer="{Voter server-03cf2b88-c2d5-405f-bc4f-012f4d4e2fc8 03cf2b88-c2d5-405f-bc4f-012f4d4e2fc8}" +2026-11-29T10:29:49.265Z [INFO] server-558e7ffe-73d3-c8a4-d34e-7ff6898107fd: aborting pipeline replication: peer="{Voter server-03cf2b88-c2d5-405f-bc4f-012f4d4e2fc8 03cf2b88-c2d5-405f-bc4f-012f4d4e2fc8}" +2026-11-29T10:29:49.265Z [INFO] server-558e7ffe-73d3-c8a4-d34e-7ff6898107fd: entering follower state: follower="Node at 558e7ffe-73d3-c8a4-d34e-7ff6898107fd [Follower]" leader-address= leader-id= +--- PASS: TestRaft_VerifyLeader_Fail (0.28s) === RUN TestRaft_VerifyLeader_PartialConnect -2025-10-27T04:03:46.371Z [INFO] server-d287929f-3e5d-8a63-463b-7da30be74087: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d287929f-3e5d-8a63-463b-7da30be74087 Address:d287929f-3e5d-8a63-463b-7da30be74087} {Suffrage:Voter ID:server-300c0ec8-00e1-ab72-c87a-029aead9ff46 Address:300c0ec8-00e1-ab72-c87a-029aead9ff46} {Suffrage:Voter ID:server-a93580bf-d405-1c0b-e2bb-0578e368d2f7 Address:a93580bf-d405-1c0b-e2bb-0578e368d2f7}]" -2025-10-27T04:03:46.371Z [INFO] server-300c0ec8-00e1-ab72-c87a-029aead9ff46: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d287929f-3e5d-8a63-463b-7da30be74087 Address:d287929f-3e5d-8a63-463b-7da30be74087} {Suffrage:Voter ID:server-300c0ec8-00e1-ab72-c87a-029aead9ff46 Address:300c0ec8-00e1-ab72-c87a-029aead9ff46} {Suffrage:Voter ID:server-a93580bf-d405-1c0b-e2bb-0578e368d2f7 Address:a93580bf-d405-1c0b-e2bb-0578e368d2f7}]" -2025-10-27T04:03:46.371Z [INFO] server-d287929f-3e5d-8a63-463b-7da30be74087: entering follower state: follower="Node at d287929f-3e5d-8a63-463b-7da30be74087 [Follower]" leader-address= leader-id= -2025-10-27T04:03:46.371Z [INFO] server-a93580bf-d405-1c0b-e2bb-0578e368d2f7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d287929f-3e5d-8a63-463b-7da30be74087 Address:d287929f-3e5d-8a63-463b-7da30be74087} {Suffrage:Voter ID:server-300c0ec8-00e1-ab72-c87a-029aead9ff46 Address:300c0ec8-00e1-ab72-c87a-029aead9ff46} {Suffrage:Voter ID:server-a93580bf-d405-1c0b-e2bb-0578e368d2f7 Address:a93580bf-d405-1c0b-e2bb-0578e368d2f7}]" -2025-10-27T04:03:46.371Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:46.371Z [INFO] server-300c0ec8-00e1-ab72-c87a-029aead9ff46: entering follower state: follower="Node at 300c0ec8-00e1-ab72-c87a-029aead9ff46 [Follower]" leader-address= leader-id= -2025-10-27T04:03:46.371Z [INFO] server-a93580bf-d405-1c0b-e2bb-0578e368d2f7: entering follower state: follower="Node at a93580bf-d405-1c0b-e2bb-0578e368d2f7 [Follower]" leader-address= leader-id= -2025-10-27T04:03:46.437Z [WARN] server-d287929f-3e5d-8a63-463b-7da30be74087: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:46.437Z [INFO] server-d287929f-3e5d-8a63-463b-7da30be74087: entering candidate state: node="Node at d287929f-3e5d-8a63-463b-7da30be74087 [Candidate]" term=2 -2025-10-27T04:03:46.437Z [INFO] server-d287929f-3e5d-8a63-463b-7da30be74087: election won: term=2 tally=2 -2025-10-27T04:03:46.437Z [INFO] server-d287929f-3e5d-8a63-463b-7da30be74087: entering leader state: leader="Node at d287929f-3e5d-8a63-463b-7da30be74087 [Leader]" -2025-10-27T04:03:46.437Z [INFO] server-d287929f-3e5d-8a63-463b-7da30be74087: added peer, starting replication: peer=server-300c0ec8-00e1-ab72-c87a-029aead9ff46 -2025-10-27T04:03:46.437Z [INFO] server-d287929f-3e5d-8a63-463b-7da30be74087: added peer, starting replication: peer=server-a93580bf-d405-1c0b-e2bb-0578e368d2f7 -2025-10-27T04:03:46.437Z [INFO] server-d287929f-3e5d-8a63-463b-7da30be74087: pipelining replication: peer="{Voter server-300c0ec8-00e1-ab72-c87a-029aead9ff46 300c0ec8-00e1-ab72-c87a-029aead9ff46}" -2025-10-27T04:03:46.437Z [INFO] server-d287929f-3e5d-8a63-463b-7da30be74087: pipelining replication: peer="{Voter server-a93580bf-d405-1c0b-e2bb-0578e368d2f7 a93580bf-d405-1c0b-e2bb-0578e368d2f7}" -2025-10-27T04:03:46.543Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:46.437616142 +0000 UTC m=+101.432488620 (1 nodes), 66.086691ms from start of poll, 66.370133ms from cluster start. Timeout at 2025-10-27 04:03:46.542618822 +0000 UTC m=+101.537491300, 105.00268ms after stability -2025-10-27T04:03:46.548Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:03:46.654Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:03:46.548962983 +0000 UTC m=+101.543835461 (2 nodes), 3.36µs from start of poll, 177.716974ms from cluster start. Timeout at 2025-10-27 04:03:46.653986943 +0000 UTC m=+101.648859421, 105.02396ms after stability - raft_test.go:1968: [INFO] Disconnecting Node at 300c0ec8-00e1-ab72-c87a-029aead9ff46 [Follower] -2025-10-27T04:03:46.654Z [ERROR] server-d287929f-3e5d-8a63-463b-7da30be74087: failed to heartbeat to: peer=300c0ec8-00e1-ab72-c87a-029aead9ff46 backoff time=10ms error="failed to connect to peer: 300c0ec8-00e1-ab72-c87a-029aead9ff46" -2025-10-27T04:03:46.654Z [INFO] server-d287929f-3e5d-8a63-463b-7da30be74087: aborting pipeline replication: peer="{Voter server-300c0ec8-00e1-ab72-c87a-029aead9ff46 300c0ec8-00e1-ab72-c87a-029aead9ff46}" -2025-10-27T04:03:46.654Z [INFO] server-d287929f-3e5d-8a63-463b-7da30be74087: aborting pipeline replication: peer="{Voter server-a93580bf-d405-1c0b-e2bb-0578e368d2f7 a93580bf-d405-1c0b-e2bb-0578e368d2f7}" ---- PASS: TestRaft_VerifyLeader_PartialConnect (0.28s) +2026-11-29T10:29:49.266Z [INFO] server-9a23b479-291c-a506-db45-da383aecefe2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9a23b479-291c-a506-db45-da383aecefe2 Address:9a23b479-291c-a506-db45-da383aecefe2} {Suffrage:Voter ID:server-271f5ebd-6bd1-a728-9b6d-326d9ac89094 Address:271f5ebd-6bd1-a728-9b6d-326d9ac89094} {Suffrage:Voter ID:server-5a8d6faa-ef06-0f86-c5b3-85ca2cb80f0b Address:5a8d6faa-ef06-0f86-c5b3-85ca2cb80f0b}]" +2026-11-29T10:29:49.266Z [INFO] server-9a23b479-291c-a506-db45-da383aecefe2: entering follower state: follower="Node at 9a23b479-291c-a506-db45-da383aecefe2 [Follower]" leader-address= leader-id= +2026-11-29T10:29:49.266Z [INFO] server-271f5ebd-6bd1-a728-9b6d-326d9ac89094: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9a23b479-291c-a506-db45-da383aecefe2 Address:9a23b479-291c-a506-db45-da383aecefe2} {Suffrage:Voter ID:server-271f5ebd-6bd1-a728-9b6d-326d9ac89094 Address:271f5ebd-6bd1-a728-9b6d-326d9ac89094} {Suffrage:Voter ID:server-5a8d6faa-ef06-0f86-c5b3-85ca2cb80f0b Address:5a8d6faa-ef06-0f86-c5b3-85ca2cb80f0b}]" +2026-11-29T10:29:49.266Z [INFO] server-271f5ebd-6bd1-a728-9b6d-326d9ac89094: entering follower state: follower="Node at 271f5ebd-6bd1-a728-9b6d-326d9ac89094 [Follower]" leader-address= leader-id= +2026-11-29T10:29:49.266Z [INFO] server-5a8d6faa-ef06-0f86-c5b3-85ca2cb80f0b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9a23b479-291c-a506-db45-da383aecefe2 Address:9a23b479-291c-a506-db45-da383aecefe2} {Suffrage:Voter ID:server-271f5ebd-6bd1-a728-9b6d-326d9ac89094 Address:271f5ebd-6bd1-a728-9b6d-326d9ac89094} {Suffrage:Voter ID:server-5a8d6faa-ef06-0f86-c5b3-85ca2cb80f0b Address:5a8d6faa-ef06-0f86-c5b3-85ca2cb80f0b}]" +2026-11-29T10:29:49.266Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:49.266Z [INFO] server-5a8d6faa-ef06-0f86-c5b3-85ca2cb80f0b: entering follower state: follower="Node at 5a8d6faa-ef06-0f86-c5b3-85ca2cb80f0b [Follower]" leader-address= leader-id= +2026-11-29T10:29:49.318Z [WARN] server-5a8d6faa-ef06-0f86-c5b3-85ca2cb80f0b: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:49.318Z [INFO] server-5a8d6faa-ef06-0f86-c5b3-85ca2cb80f0b: entering candidate state: node="Node at 5a8d6faa-ef06-0f86-c5b3-85ca2cb80f0b [Candidate]" term=2 +2026-11-29T10:29:49.318Z [INFO] server-5a8d6faa-ef06-0f86-c5b3-85ca2cb80f0b: election won: term=2 tally=2 +2026-11-29T10:29:49.318Z [INFO] server-5a8d6faa-ef06-0f86-c5b3-85ca2cb80f0b: entering leader state: leader="Node at 5a8d6faa-ef06-0f86-c5b3-85ca2cb80f0b [Leader]" +2026-11-29T10:29:49.318Z [INFO] server-5a8d6faa-ef06-0f86-c5b3-85ca2cb80f0b: added peer, starting replication: peer=server-9a23b479-291c-a506-db45-da383aecefe2 +2026-11-29T10:29:49.318Z [INFO] server-5a8d6faa-ef06-0f86-c5b3-85ca2cb80f0b: added peer, starting replication: peer=server-271f5ebd-6bd1-a728-9b6d-326d9ac89094 +2026-11-29T10:29:49.318Z [INFO] server-5a8d6faa-ef06-0f86-c5b3-85ca2cb80f0b: pipelining replication: peer="{Voter server-271f5ebd-6bd1-a728-9b6d-326d9ac89094 271f5ebd-6bd1-a728-9b6d-326d9ac89094}" +2026-11-29T10:29:49.318Z [INFO] server-5a8d6faa-ef06-0f86-c5b3-85ca2cb80f0b: pipelining replication: peer="{Voter server-9a23b479-291c-a506-db45-da383aecefe2 9a23b479-291c-a506-db45-da383aecefe2}" +2026-11-29T10:29:49.423Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:49.318730499 +0000 UTC m=+100.053784601 (1 nodes), 52.04732ms from start of poll, 52.39148ms from cluster start. Timeout at 2026-11-29 10:29:49.423733379 +0000 UTC m=+100.158787481, 105.00288ms after stability +2026-11-29T10:29:49.430Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:49.535Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:49.430265459 +0000 UTC m=+100.165319561 (2 nodes), 1.64µs from start of poll, 163.92644ms from cluster start. Timeout at 2026-11-29 10:29:49.535283419 +0000 UTC m=+100.270337521, 105.01796ms after stability + raft_test.go:1968: [INFO] Disconnecting Node at 9a23b479-291c-a506-db45-da383aecefe2 [Follower] +2026-11-29T10:29:49.535Z [INFO] server-5a8d6faa-ef06-0f86-c5b3-85ca2cb80f0b: aborting pipeline replication: peer="{Voter server-271f5ebd-6bd1-a728-9b6d-326d9ac89094 271f5ebd-6bd1-a728-9b6d-326d9ac89094}" +2026-11-29T10:29:49.535Z [ERROR] server-5a8d6faa-ef06-0f86-c5b3-85ca2cb80f0b: failed to heartbeat to: peer=9a23b479-291c-a506-db45-da383aecefe2 backoff time=10ms error="failed to connect to peer: 9a23b479-291c-a506-db45-da383aecefe2" +2026-11-29T10:29:49.535Z [INFO] server-5a8d6faa-ef06-0f86-c5b3-85ca2cb80f0b: aborting pipeline replication: peer="{Voter server-9a23b479-291c-a506-db45-da383aecefe2 9a23b479-291c-a506-db45-da383aecefe2}" +--- PASS: TestRaft_VerifyLeader_PartialConnect (0.27s) === RUN TestRaft_NotifyCh -2025-10-27T04:03:46.655Z [INFO] server-a9c663d1-0710-e32c-ca06-7fd3c35d9691: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a9c663d1-0710-e32c-ca06-7fd3c35d9691 Address:a9c663d1-0710-e32c-ca06-7fd3c35d9691}]" -2025-10-27T04:03:46.655Z [INFO] server-a9c663d1-0710-e32c-ca06-7fd3c35d9691: entering follower state: follower="Node at a9c663d1-0710-e32c-ca06-7fd3c35d9691 [Follower]" leader-address= leader-id= -2025-10-27T04:03:46.733Z [WARN] server-a9c663d1-0710-e32c-ca06-7fd3c35d9691: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:46.733Z [INFO] server-a9c663d1-0710-e32c-ca06-7fd3c35d9691: entering candidate state: node="Node at a9c663d1-0710-e32c-ca06-7fd3c35d9691 [Candidate]" term=2 -2025-10-27T04:03:46.733Z [INFO] server-a9c663d1-0710-e32c-ca06-7fd3c35d9691: election won: term=2 tally=1 -2025-10-27T04:03:46.734Z [INFO] server-a9c663d1-0710-e32c-ca06-7fd3c35d9691: entering leader state: leader="Node at a9c663d1-0710-e32c-ca06-7fd3c35d9691 [Leader]" ---- PASS: TestRaft_NotifyCh (0.08s) +2026-11-29T10:29:49.536Z [INFO] server-e05e6782-f7e5-768a-ca99-1f2277083f52: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e05e6782-f7e5-768a-ca99-1f2277083f52 Address:e05e6782-f7e5-768a-ca99-1f2277083f52}]" +2026-11-29T10:29:49.536Z [INFO] server-e05e6782-f7e5-768a-ca99-1f2277083f52: entering follower state: follower="Node at e05e6782-f7e5-768a-ca99-1f2277083f52 [Follower]" leader-address= leader-id= +2026-11-29T10:29:49.598Z [WARN] server-e05e6782-f7e5-768a-ca99-1f2277083f52: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:49.598Z [INFO] server-e05e6782-f7e5-768a-ca99-1f2277083f52: entering candidate state: node="Node at e05e6782-f7e5-768a-ca99-1f2277083f52 [Candidate]" term=2 +2026-11-29T10:29:49.598Z [INFO] server-e05e6782-f7e5-768a-ca99-1f2277083f52: election won: term=2 tally=1 +2026-11-29T10:29:49.598Z [INFO] server-e05e6782-f7e5-768a-ca99-1f2277083f52: entering leader state: leader="Node at e05e6782-f7e5-768a-ca99-1f2277083f52 [Leader]" +--- PASS: TestRaft_NotifyCh (0.06s) === RUN TestRaft_AppendEntry -2025-10-27T04:03:46.734Z [INFO] server-ce4c80c0-f516-8222-74fd-7f1ebde37bd8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ce4c80c0-f516-8222-74fd-7f1ebde37bd8 Address:ce4c80c0-f516-8222-74fd-7f1ebde37bd8} {Suffrage:Voter ID:server-ecbff22f-e0cc-9c3d-9b12-737ef0e34a5b Address:ecbff22f-e0cc-9c3d-9b12-737ef0e34a5b} {Suffrage:Voter ID:server-d73706d7-de27-b6e8-b443-0e12268e23fe Address:d73706d7-de27-b6e8-b443-0e12268e23fe}]" -2025-10-27T04:03:46.734Z [INFO] server-ce4c80c0-f516-8222-74fd-7f1ebde37bd8: entering follower state: follower="Node at ce4c80c0-f516-8222-74fd-7f1ebde37bd8 [Follower]" leader-address= leader-id= -2025-10-27T04:03:46.734Z [INFO] server-ecbff22f-e0cc-9c3d-9b12-737ef0e34a5b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ce4c80c0-f516-8222-74fd-7f1ebde37bd8 Address:ce4c80c0-f516-8222-74fd-7f1ebde37bd8} {Suffrage:Voter ID:server-ecbff22f-e0cc-9c3d-9b12-737ef0e34a5b Address:ecbff22f-e0cc-9c3d-9b12-737ef0e34a5b} {Suffrage:Voter ID:server-d73706d7-de27-b6e8-b443-0e12268e23fe Address:d73706d7-de27-b6e8-b443-0e12268e23fe}]" -2025-10-27T04:03:46.735Z [INFO] server-ecbff22f-e0cc-9c3d-9b12-737ef0e34a5b: entering follower state: follower="Node at ecbff22f-e0cc-9c3d-9b12-737ef0e34a5b [Follower]" leader-address= leader-id= -2025-10-27T04:03:46.735Z [INFO] server-d73706d7-de27-b6e8-b443-0e12268e23fe: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ce4c80c0-f516-8222-74fd-7f1ebde37bd8 Address:ce4c80c0-f516-8222-74fd-7f1ebde37bd8} {Suffrage:Voter ID:server-ecbff22f-e0cc-9c3d-9b12-737ef0e34a5b Address:ecbff22f-e0cc-9c3d-9b12-737ef0e34a5b} {Suffrage:Voter ID:server-d73706d7-de27-b6e8-b443-0e12268e23fe Address:d73706d7-de27-b6e8-b443-0e12268e23fe}]" -2025-10-27T04:03:46.735Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:03:46.735Z [INFO] server-d73706d7-de27-b6e8-b443-0e12268e23fe: entering follower state: follower="Node at d73706d7-de27-b6e8-b443-0e12268e23fe [Follower]" leader-address= leader-id= -2025-10-27T04:03:46.790Z [WARN] server-d73706d7-de27-b6e8-b443-0e12268e23fe: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:46.790Z [INFO] server-d73706d7-de27-b6e8-b443-0e12268e23fe: entering candidate state: node="Node at d73706d7-de27-b6e8-b443-0e12268e23fe [Candidate]" term=2 -2025-10-27T04:03:46.790Z [INFO] server-d73706d7-de27-b6e8-b443-0e12268e23fe: election won: term=2 tally=2 -2025-10-27T04:03:46.790Z [INFO] server-d73706d7-de27-b6e8-b443-0e12268e23fe: entering leader state: leader="Node at d73706d7-de27-b6e8-b443-0e12268e23fe [Leader]" -2025-10-27T04:03:46.790Z [INFO] server-d73706d7-de27-b6e8-b443-0e12268e23fe: added peer, starting replication: peer=server-ce4c80c0-f516-8222-74fd-7f1ebde37bd8 -2025-10-27T04:03:46.790Z [INFO] server-d73706d7-de27-b6e8-b443-0e12268e23fe: added peer, starting replication: peer=server-ecbff22f-e0cc-9c3d-9b12-737ef0e34a5b -2025-10-27T04:03:46.791Z [INFO] server-d73706d7-de27-b6e8-b443-0e12268e23fe: pipelining replication: peer="{Voter server-ecbff22f-e0cc-9c3d-9b12-737ef0e34a5b ecbff22f-e0cc-9c3d-9b12-737ef0e34a5b}" -2025-10-27T04:03:46.791Z [INFO] server-d73706d7-de27-b6e8-b443-0e12268e23fe: pipelining replication: peer="{Voter server-ce4c80c0-f516-8222-74fd-7f1ebde37bd8 ce4c80c0-f516-8222-74fd-7f1ebde37bd8}" -2025-10-27T04:03:46.896Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:03:46.791015557 +0000 UTC m=+101.785887995 (2 nodes), 55.746101ms from start of poll, 56.298705ms from cluster start. Timeout at 2025-10-27 04:03:46.896017197 +0000 UTC m=+101.890889675, 105.00168ms after stability -2025-10-27T04:03:46.896Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:47.002Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:46.896759999 +0000 UTC m=+101.891632477 (1 nodes), 1.96µs from start of poll, 162.043187ms from cluster start. Timeout at 2025-10-27 04:03:47.001774799 +0000 UTC m=+101.996647237, 105.01476ms after stability -2025-10-27T04:03:47.002Z [WARN] server-ce4c80c0-f516-8222-74fd-7f1ebde37bd8: clearing log suffix: from=1 to=2 -2025-10-27T04:03:47.002Z [WARN] server-ce4c80c0-f516-8222-74fd-7f1ebde37bd8: skipping application of old log: index=1 -2025-10-27T04:03:47.002Z [WARN] server-ce4c80c0-f516-8222-74fd-7f1ebde37bd8: skipping application of old log: index=1 -2025-10-27T04:03:47.002Z [INFO] server-d73706d7-de27-b6e8-b443-0e12268e23fe: aborting pipeline replication: peer="{Voter server-ecbff22f-e0cc-9c3d-9b12-737ef0e34a5b ecbff22f-e0cc-9c3d-9b12-737ef0e34a5b}" -2025-10-27T04:03:47.002Z [INFO] server-d73706d7-de27-b6e8-b443-0e12268e23fe: aborting pipeline replication: peer="{Voter server-ce4c80c0-f516-8222-74fd-7f1ebde37bd8 ce4c80c0-f516-8222-74fd-7f1ebde37bd8}" +2026-11-29T10:29:49.598Z [INFO] server-576d0e7b-a350-4d75-04a0-be7f305d8461: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-576d0e7b-a350-4d75-04a0-be7f305d8461 Address:576d0e7b-a350-4d75-04a0-be7f305d8461} {Suffrage:Voter ID:server-5cc153ca-8cb6-13c9-bf9b-29652962cec5 Address:5cc153ca-8cb6-13c9-bf9b-29652962cec5} {Suffrage:Voter ID:server-98045809-462c-af96-3822-8be85436f5e9 Address:98045809-462c-af96-3822-8be85436f5e9}]" +2026-11-29T10:29:49.598Z [INFO] server-5cc153ca-8cb6-13c9-bf9b-29652962cec5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-576d0e7b-a350-4d75-04a0-be7f305d8461 Address:576d0e7b-a350-4d75-04a0-be7f305d8461} {Suffrage:Voter ID:server-5cc153ca-8cb6-13c9-bf9b-29652962cec5 Address:5cc153ca-8cb6-13c9-bf9b-29652962cec5} {Suffrage:Voter ID:server-98045809-462c-af96-3822-8be85436f5e9 Address:98045809-462c-af96-3822-8be85436f5e9}]" +2026-11-29T10:29:49.599Z [INFO] server-98045809-462c-af96-3822-8be85436f5e9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-576d0e7b-a350-4d75-04a0-be7f305d8461 Address:576d0e7b-a350-4d75-04a0-be7f305d8461} {Suffrage:Voter ID:server-5cc153ca-8cb6-13c9-bf9b-29652962cec5 Address:5cc153ca-8cb6-13c9-bf9b-29652962cec5} {Suffrage:Voter ID:server-98045809-462c-af96-3822-8be85436f5e9 Address:98045809-462c-af96-3822-8be85436f5e9}]" +2026-11-29T10:29:49.599Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:49.599Z [INFO] server-576d0e7b-a350-4d75-04a0-be7f305d8461: entering follower state: follower="Node at 576d0e7b-a350-4d75-04a0-be7f305d8461 [Follower]" leader-address= leader-id= +2026-11-29T10:29:49.599Z [INFO] server-5cc153ca-8cb6-13c9-bf9b-29652962cec5: entering follower state: follower="Node at 5cc153ca-8cb6-13c9-bf9b-29652962cec5 [Follower]" leader-address= leader-id= +2026-11-29T10:29:49.599Z [INFO] server-98045809-462c-af96-3822-8be85436f5e9: entering follower state: follower="Node at 98045809-462c-af96-3822-8be85436f5e9 [Follower]" leader-address= leader-id= +2026-11-29T10:29:49.661Z [WARN] server-576d0e7b-a350-4d75-04a0-be7f305d8461: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:49.661Z [INFO] server-576d0e7b-a350-4d75-04a0-be7f305d8461: entering candidate state: node="Node at 576d0e7b-a350-4d75-04a0-be7f305d8461 [Candidate]" term=2 +2026-11-29T10:29:49.661Z [INFO] server-576d0e7b-a350-4d75-04a0-be7f305d8461: election won: term=2 tally=2 +2026-11-29T10:29:49.661Z [INFO] server-576d0e7b-a350-4d75-04a0-be7f305d8461: entering leader state: leader="Node at 576d0e7b-a350-4d75-04a0-be7f305d8461 [Leader]" +2026-11-29T10:29:49.661Z [INFO] server-576d0e7b-a350-4d75-04a0-be7f305d8461: added peer, starting replication: peer=server-5cc153ca-8cb6-13c9-bf9b-29652962cec5 +2026-11-29T10:29:49.661Z [INFO] server-576d0e7b-a350-4d75-04a0-be7f305d8461: added peer, starting replication: peer=server-98045809-462c-af96-3822-8be85436f5e9 +2026-11-29T10:29:49.662Z [INFO] server-576d0e7b-a350-4d75-04a0-be7f305d8461: pipelining replication: peer="{Voter server-5cc153ca-8cb6-13c9-bf9b-29652962cec5 5cc153ca-8cb6-13c9-bf9b-29652962cec5}" +2026-11-29T10:29:49.662Z [INFO] server-576d0e7b-a350-4d75-04a0-be7f305d8461: pipelining replication: peer="{Voter server-98045809-462c-af96-3822-8be85436f5e9 98045809-462c-af96-3822-8be85436f5e9}" +2026-11-29T10:29:49.767Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:49.662001179 +0000 UTC m=+100.397055281 (2 nodes), 62.94364ms from start of poll, 63.13832ms from cluster start. Timeout at 2026-11-29 10:29:49.767004819 +0000 UTC m=+100.502058881, 105.0036ms after stability +2026-11-29T10:29:49.767Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:49.873Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:49.767841619 +0000 UTC m=+100.502895721 (1 nodes), 1.48µs from start of poll, 168.97876ms from cluster start. Timeout at 2026-11-29 10:29:49.872868899 +0000 UTC m=+100.607922961, 105.02724ms after stability +2026-11-29T10:29:49.873Z [WARN] server-5cc153ca-8cb6-13c9-bf9b-29652962cec5: clearing log suffix: from=1 to=2 +2026-11-29T10:29:49.873Z [WARN] server-5cc153ca-8cb6-13c9-bf9b-29652962cec5: skipping application of old log: index=1 +2026-11-29T10:29:49.873Z [WARN] server-5cc153ca-8cb6-13c9-bf9b-29652962cec5: skipping application of old log: index=1 +2026-11-29T10:29:49.873Z [INFO] server-576d0e7b-a350-4d75-04a0-be7f305d8461: aborting pipeline replication: peer="{Voter server-98045809-462c-af96-3822-8be85436f5e9 98045809-462c-af96-3822-8be85436f5e9}" +2026-11-29T10:29:49.873Z [INFO] server-576d0e7b-a350-4d75-04a0-be7f305d8461: aborting pipeline replication: peer="{Voter server-5cc153ca-8cb6-13c9-bf9b-29652962cec5 5cc153ca-8cb6-13c9-bf9b-29652962cec5}" --- PASS: TestRaft_AppendEntry (0.27s) === RUN TestRaft_VotingGrant_WhenLeaderAvailable -2025-10-27T04:03:47.003Z [INFO] server-4e3c8842-08fb-9e58-8479-b83d145a9885: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4e3c8842-08fb-9e58-8479-b83d145a9885 Address:4e3c8842-08fb-9e58-8479-b83d145a9885} {Suffrage:Voter ID:server-1661d9b6-91d1-b20a-e949-2fdc78cf7a41 Address:1661d9b6-91d1-b20a-e949-2fdc78cf7a41} {Suffrage:Voter ID:server-20885ba8-30e4-218f-3c43-5555b3045752 Address:20885ba8-30e4-218f-3c43-5555b3045752}]" -2025-10-27T04:03:47.003Z [INFO] server-4e3c8842-08fb-9e58-8479-b83d145a9885: entering follower state: follower="Node at 4e3c8842-08fb-9e58-8479-b83d145a9885 [Follower]" leader-address= leader-id= -2025-10-27T04:03:47.003Z [INFO] server-1661d9b6-91d1-b20a-e949-2fdc78cf7a41: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4e3c8842-08fb-9e58-8479-b83d145a9885 Address:4e3c8842-08fb-9e58-8479-b83d145a9885} {Suffrage:Voter ID:server-1661d9b6-91d1-b20a-e949-2fdc78cf7a41 Address:1661d9b6-91d1-b20a-e949-2fdc78cf7a41} {Suffrage:Voter ID:server-20885ba8-30e4-218f-3c43-5555b3045752 Address:20885ba8-30e4-218f-3c43-5555b3045752}]" -2025-10-27T04:03:47.003Z [INFO] server-20885ba8-30e4-218f-3c43-5555b3045752: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4e3c8842-08fb-9e58-8479-b83d145a9885 Address:4e3c8842-08fb-9e58-8479-b83d145a9885} {Suffrage:Voter ID:server-1661d9b6-91d1-b20a-e949-2fdc78cf7a41 Address:1661d9b6-91d1-b20a-e949-2fdc78cf7a41} {Suffrage:Voter ID:server-20885ba8-30e4-218f-3c43-5555b3045752 Address:20885ba8-30e4-218f-3c43-5555b3045752}]" -2025-10-27T04:03:47.004Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:03:47.004Z [INFO] server-1661d9b6-91d1-b20a-e949-2fdc78cf7a41: entering follower state: follower="Node at 1661d9b6-91d1-b20a-e949-2fdc78cf7a41 [Follower]" leader-address= leader-id= -2025-10-27T04:03:47.004Z [INFO] server-20885ba8-30e4-218f-3c43-5555b3045752: entering follower state: follower="Node at 20885ba8-30e4-218f-3c43-5555b3045752 [Follower]" leader-address= leader-id= -2025-10-27T04:03:47.056Z [WARN] server-4e3c8842-08fb-9e58-8479-b83d145a9885: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:47.056Z [INFO] server-4e3c8842-08fb-9e58-8479-b83d145a9885: entering candidate state: node="Node at 4e3c8842-08fb-9e58-8479-b83d145a9885 [Candidate]" term=2 -2025-10-27T04:03:47.056Z [INFO] server-4e3c8842-08fb-9e58-8479-b83d145a9885: election won: term=2 tally=2 -2025-10-27T04:03:47.056Z [INFO] server-4e3c8842-08fb-9e58-8479-b83d145a9885: entering leader state: leader="Node at 4e3c8842-08fb-9e58-8479-b83d145a9885 [Leader]" -2025-10-27T04:03:47.056Z [INFO] server-4e3c8842-08fb-9e58-8479-b83d145a9885: added peer, starting replication: peer=server-1661d9b6-91d1-b20a-e949-2fdc78cf7a41 -2025-10-27T04:03:47.056Z [INFO] server-4e3c8842-08fb-9e58-8479-b83d145a9885: added peer, starting replication: peer=server-20885ba8-30e4-218f-3c43-5555b3045752 -2025-10-27T04:03:47.056Z [INFO] server-4e3c8842-08fb-9e58-8479-b83d145a9885: pipelining replication: peer="{Voter server-20885ba8-30e4-218f-3c43-5555b3045752 20885ba8-30e4-218f-3c43-5555b3045752}" -2025-10-27T04:03:47.056Z [WARN] server-1661d9b6-91d1-b20a-e949-2fdc78cf7a41: rejecting vote request since our last term is greater: candidate=4e3c8842-08fb-9e58-8479-b83d145a9885 last-term=2 last-candidate-term=1 -2025-10-27T04:03:47.056Z [INFO] server-4e3c8842-08fb-9e58-8479-b83d145a9885: pipelining replication: peer="{Voter server-1661d9b6-91d1-b20a-e949-2fdc78cf7a41 1661d9b6-91d1-b20a-e949-2fdc78cf7a41}" -2025-10-27T04:03:47.162Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:03:47.056876213 +0000 UTC m=+102.051748651 (2 nodes), 52.844641ms from start of poll, 53.841808ms from cluster start. Timeout at 2025-10-27 04:03:47.161880693 +0000 UTC m=+102.156753171, 105.00452ms after stability -2025-10-27T04:03:47.162Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:47.267Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:47.162126212 +0000 UTC m=+102.156998690 (1 nodes), 1.72µs from start of poll, 159.091847ms from cluster start. Timeout at 2025-10-27 04:03:47.267156812 +0000 UTC m=+102.262029290, 105.0306ms after stability -2025-10-27T04:03:47.267Z [WARN] server-20885ba8-30e4-218f-3c43-5555b3045752: rejecting vote request since we have a leader: from=1661d9b6-91d1-b20a-e949-2fdc78cf7a41 leader=4e3c8842-08fb-9e58-8479-b83d145a9885 leader-id=server-4e3c8842-08fb-9e58-8479-b83d145a9885 -2025-10-27T04:03:47.267Z [INFO] server-4e3c8842-08fb-9e58-8479-b83d145a9885: aborting pipeline replication: peer="{Voter server-20885ba8-30e4-218f-3c43-5555b3045752 20885ba8-30e4-218f-3c43-5555b3045752}" -2025-10-27T04:03:47.267Z [INFO] server-4e3c8842-08fb-9e58-8479-b83d145a9885: aborting pipeline replication: peer="{Voter server-1661d9b6-91d1-b20a-e949-2fdc78cf7a41 1661d9b6-91d1-b20a-e949-2fdc78cf7a41}" ---- PASS: TestRaft_VotingGrant_WhenLeaderAvailable (0.27s) +2026-11-29T10:29:49.873Z [INFO] server-c393750f-a4a1-e495-6f39-f8d077910fcb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c393750f-a4a1-e495-6f39-f8d077910fcb Address:c393750f-a4a1-e495-6f39-f8d077910fcb} {Suffrage:Voter ID:server-293bd59e-ffa8-7f64-4239-700e16940bdc Address:293bd59e-ffa8-7f64-4239-700e16940bdc} {Suffrage:Voter ID:server-0eebbae9-312e-d103-e48c-ed6094069c87 Address:0eebbae9-312e-d103-e48c-ed6094069c87}]" +2026-11-29T10:29:49.873Z [INFO] server-293bd59e-ffa8-7f64-4239-700e16940bdc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c393750f-a4a1-e495-6f39-f8d077910fcb Address:c393750f-a4a1-e495-6f39-f8d077910fcb} {Suffrage:Voter ID:server-293bd59e-ffa8-7f64-4239-700e16940bdc Address:293bd59e-ffa8-7f64-4239-700e16940bdc} {Suffrage:Voter ID:server-0eebbae9-312e-d103-e48c-ed6094069c87 Address:0eebbae9-312e-d103-e48c-ed6094069c87}]" +2026-11-29T10:29:49.873Z [INFO] server-0eebbae9-312e-d103-e48c-ed6094069c87: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c393750f-a4a1-e495-6f39-f8d077910fcb Address:c393750f-a4a1-e495-6f39-f8d077910fcb} {Suffrage:Voter ID:server-293bd59e-ffa8-7f64-4239-700e16940bdc Address:293bd59e-ffa8-7f64-4239-700e16940bdc} {Suffrage:Voter ID:server-0eebbae9-312e-d103-e48c-ed6094069c87 Address:0eebbae9-312e-d103-e48c-ed6094069c87}]" +2026-11-29T10:29:49.873Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:49.873Z [INFO] server-c393750f-a4a1-e495-6f39-f8d077910fcb: entering follower state: follower="Node at c393750f-a4a1-e495-6f39-f8d077910fcb [Follower]" leader-address= leader-id= +2026-11-29T10:29:49.873Z [INFO] server-293bd59e-ffa8-7f64-4239-700e16940bdc: entering follower state: follower="Node at 293bd59e-ffa8-7f64-4239-700e16940bdc [Follower]" leader-address= leader-id= +2026-11-29T10:29:49.873Z [INFO] server-0eebbae9-312e-d103-e48c-ed6094069c87: entering follower state: follower="Node at 0eebbae9-312e-d103-e48c-ed6094069c87 [Follower]" leader-address= leader-id= +2026-11-29T10:29:49.960Z [WARN] server-293bd59e-ffa8-7f64-4239-700e16940bdc: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:49.960Z [INFO] server-293bd59e-ffa8-7f64-4239-700e16940bdc: entering candidate state: node="Node at 293bd59e-ffa8-7f64-4239-700e16940bdc [Candidate]" term=2 +2026-11-29T10:29:49.960Z [INFO] server-293bd59e-ffa8-7f64-4239-700e16940bdc: election won: term=2 tally=2 +2026-11-29T10:29:49.960Z [INFO] server-293bd59e-ffa8-7f64-4239-700e16940bdc: entering leader state: leader="Node at 293bd59e-ffa8-7f64-4239-700e16940bdc [Leader]" +2026-11-29T10:29:49.960Z [INFO] server-293bd59e-ffa8-7f64-4239-700e16940bdc: added peer, starting replication: peer=server-c393750f-a4a1-e495-6f39-f8d077910fcb +2026-11-29T10:29:49.960Z [INFO] server-293bd59e-ffa8-7f64-4239-700e16940bdc: added peer, starting replication: peer=server-0eebbae9-312e-d103-e48c-ed6094069c87 +2026-11-29T10:29:49.960Z [INFO] server-293bd59e-ffa8-7f64-4239-700e16940bdc: pipelining replication: peer="{Voter server-c393750f-a4a1-e495-6f39-f8d077910fcb c393750f-a4a1-e495-6f39-f8d077910fcb}" +2026-11-29T10:29:49.960Z [INFO] server-293bd59e-ffa8-7f64-4239-700e16940bdc: pipelining replication: peer="{Voter server-0eebbae9-312e-d103-e48c-ed6094069c87 0eebbae9-312e-d103-e48c-ed6094069c87}" +2026-11-29T10:29:50.066Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:49.960455459 +0000 UTC m=+100.695509561 (2 nodes), 86.72816ms from start of poll, 86.89148ms from cluster start. Timeout at 2026-11-29 10:29:50.065459819 +0000 UTC m=+100.800513921, 105.00436ms after stability +2026-11-29T10:29:50.066Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:50.171Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:50.066111099 +0000 UTC m=+100.801165161 (1 nodes), 1.64µs from start of poll, 192.54708ms from cluster start. Timeout at 2026-11-29 10:29:50.171129139 +0000 UTC m=+100.906183241, 105.01808ms after stability +2026-11-29T10:29:50.171Z [WARN] server-0eebbae9-312e-d103-e48c-ed6094069c87: rejecting vote request since we have a leader: from=c393750f-a4a1-e495-6f39-f8d077910fcb leader=293bd59e-ffa8-7f64-4239-700e16940bdc leader-id=server-293bd59e-ffa8-7f64-4239-700e16940bdc +2026-11-29T10:29:50.171Z [INFO] server-293bd59e-ffa8-7f64-4239-700e16940bdc: aborting pipeline replication: peer="{Voter server-0eebbae9-312e-d103-e48c-ed6094069c87 0eebbae9-312e-d103-e48c-ed6094069c87}" +2026-11-29T10:29:50.171Z [INFO] server-293bd59e-ffa8-7f64-4239-700e16940bdc: aborting pipeline replication: peer="{Voter server-c393750f-a4a1-e495-6f39-f8d077910fcb c393750f-a4a1-e495-6f39-f8d077910fcb}" +--- PASS: TestRaft_VotingGrant_WhenLeaderAvailable (0.30s) === RUN TestRaft_ProtocolVersion_RejectRPC -2025-10-27T04:03:47.269Z [INFO] server-a8f6bd15-556c-9f80-1fde-176c3c1ad1a7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a8f6bd15-556c-9f80-1fde-176c3c1ad1a7 Address:a8f6bd15-556c-9f80-1fde-176c3c1ad1a7} {Suffrage:Voter ID:server-e3afdeec-373d-510a-3081-d530c1a61702 Address:e3afdeec-373d-510a-3081-d530c1a61702} {Suffrage:Voter ID:server-c139b08d-ba79-4a78-1b16-0e09f2d63ef6 Address:c139b08d-ba79-4a78-1b16-0e09f2d63ef6}]" -2025-10-27T04:03:47.270Z [INFO] server-a8f6bd15-556c-9f80-1fde-176c3c1ad1a7: entering follower state: follower="Node at a8f6bd15-556c-9f80-1fde-176c3c1ad1a7 [Follower]" leader-address= leader-id= -2025-10-27T04:03:47.270Z [INFO] server-e3afdeec-373d-510a-3081-d530c1a61702: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a8f6bd15-556c-9f80-1fde-176c3c1ad1a7 Address:a8f6bd15-556c-9f80-1fde-176c3c1ad1a7} {Suffrage:Voter ID:server-e3afdeec-373d-510a-3081-d530c1a61702 Address:e3afdeec-373d-510a-3081-d530c1a61702} {Suffrage:Voter ID:server-c139b08d-ba79-4a78-1b16-0e09f2d63ef6 Address:c139b08d-ba79-4a78-1b16-0e09f2d63ef6}]" -2025-10-27T04:03:47.270Z [INFO] server-c139b08d-ba79-4a78-1b16-0e09f2d63ef6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a8f6bd15-556c-9f80-1fde-176c3c1ad1a7 Address:a8f6bd15-556c-9f80-1fde-176c3c1ad1a7} {Suffrage:Voter ID:server-e3afdeec-373d-510a-3081-d530c1a61702 Address:e3afdeec-373d-510a-3081-d530c1a61702} {Suffrage:Voter ID:server-c139b08d-ba79-4a78-1b16-0e09f2d63ef6 Address:c139b08d-ba79-4a78-1b16-0e09f2d63ef6}]" -2025-10-27T04:03:47.270Z [INFO] server-e3afdeec-373d-510a-3081-d530c1a61702: entering follower state: follower="Node at e3afdeec-373d-510a-3081-d530c1a61702 [Follower]" leader-address= leader-id= -2025-10-27T04:03:47.270Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:03:47.270Z [INFO] server-c139b08d-ba79-4a78-1b16-0e09f2d63ef6: entering follower state: follower="Node at c139b08d-ba79-4a78-1b16-0e09f2d63ef6 [Follower]" leader-address= leader-id= -2025-10-27T04:03:47.332Z [WARN] server-a8f6bd15-556c-9f80-1fde-176c3c1ad1a7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:47.332Z [WARN] server-e3afdeec-373d-510a-3081-d530c1a61702: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:47.332Z [INFO] server-a8f6bd15-556c-9f80-1fde-176c3c1ad1a7: entering candidate state: node="Node at a8f6bd15-556c-9f80-1fde-176c3c1ad1a7 [Candidate]" term=2 -2025-10-27T04:03:47.332Z [INFO] server-e3afdeec-373d-510a-3081-d530c1a61702: entering candidate state: node="Node at e3afdeec-373d-510a-3081-d530c1a61702 [Candidate]" term=2 -2025-10-27T04:03:47.332Z [INFO] server-e3afdeec-373d-510a-3081-d530c1a61702: duplicate requestVote for same term: term=2 -2025-10-27T04:03:47.332Z [INFO] server-a8f6bd15-556c-9f80-1fde-176c3c1ad1a7: duplicate requestVote for same term: term=2 -2025-10-27T04:03:47.332Z [INFO] server-c139b08d-ba79-4a78-1b16-0e09f2d63ef6: duplicate requestVote for same term: term=2 -2025-10-27T04:03:47.332Z [INFO] server-a8f6bd15-556c-9f80-1fde-176c3c1ad1a7: election won: term=2 tally=2 -2025-10-27T04:03:47.332Z [INFO] server-a8f6bd15-556c-9f80-1fde-176c3c1ad1a7: entering leader state: leader="Node at a8f6bd15-556c-9f80-1fde-176c3c1ad1a7 [Leader]" -2025-10-27T04:03:47.332Z [INFO] server-a8f6bd15-556c-9f80-1fde-176c3c1ad1a7: added peer, starting replication: peer=server-e3afdeec-373d-510a-3081-d530c1a61702 -2025-10-27T04:03:47.332Z [INFO] server-a8f6bd15-556c-9f80-1fde-176c3c1ad1a7: added peer, starting replication: peer=server-c139b08d-ba79-4a78-1b16-0e09f2d63ef6 -2025-10-27T04:03:47.332Z [INFO] server-a8f6bd15-556c-9f80-1fde-176c3c1ad1a7: pipelining replication: peer="{Voter server-c139b08d-ba79-4a78-1b16-0e09f2d63ef6 c139b08d-ba79-4a78-1b16-0e09f2d63ef6}" -2025-10-27T04:03:47.332Z [INFO] server-e3afdeec-373d-510a-3081-d530c1a61702: entering follower state: follower="Node at e3afdeec-373d-510a-3081-d530c1a61702 [Follower]" leader-address=a8f6bd15-556c-9f80-1fde-176c3c1ad1a7 leader-id=server-a8f6bd15-556c-9f80-1fde-176c3c1ad1a7 -2025-10-27T04:03:47.332Z [INFO] server-a8f6bd15-556c-9f80-1fde-176c3c1ad1a7: pipelining replication: peer="{Voter server-e3afdeec-373d-510a-3081-d530c1a61702 e3afdeec-373d-510a-3081-d530c1a61702}" -2025-10-27T04:03:47.437Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:03:47.332804418 +0000 UTC m=+102.327676856 (2 nodes), 62.612068ms from start of poll, 63.00119ms from cluster start. Timeout at 2025-10-27 04:03:47.437808258 +0000 UTC m=+102.432680736, 105.00388ms after stability -2025-10-27T04:03:47.437Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:47.544Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:47.437932216 +0000 UTC m=+102.432804694 (1 nodes), 2.6µs from start of poll, 168.129028ms from cluster start. Timeout at 2025-10-27 04:03:47.542952656 +0000 UTC m=+102.537825094, 105.0204ms after stability -2025-10-27T04:03:47.544Z [INFO] server-a8f6bd15-556c-9f80-1fde-176c3c1ad1a7: aborting pipeline replication: peer="{Voter server-c139b08d-ba79-4a78-1b16-0e09f2d63ef6 c139b08d-ba79-4a78-1b16-0e09f2d63ef6}" -2025-10-27T04:03:47.544Z [INFO] server-a8f6bd15-556c-9f80-1fde-176c3c1ad1a7: aborting pipeline replication: peer="{Voter server-e3afdeec-373d-510a-3081-d530c1a61702 e3afdeec-373d-510a-3081-d530c1a61702}" ---- PASS: TestRaft_ProtocolVersion_RejectRPC (0.27s) +2026-11-29T10:29:50.172Z [INFO] server-27c9407c-5467-5a76-c097-676f67d3bfc2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-27c9407c-5467-5a76-c097-676f67d3bfc2 Address:27c9407c-5467-5a76-c097-676f67d3bfc2} {Suffrage:Voter ID:server-f84bdcb8-de9b-2d7d-f2c5-b9ef0301178f Address:f84bdcb8-de9b-2d7d-f2c5-b9ef0301178f} {Suffrage:Voter ID:server-2ba38a58-2488-128d-6661-007927f8479e Address:2ba38a58-2488-128d-6661-007927f8479e}]" +2026-11-29T10:29:50.172Z [INFO] server-f84bdcb8-de9b-2d7d-f2c5-b9ef0301178f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-27c9407c-5467-5a76-c097-676f67d3bfc2 Address:27c9407c-5467-5a76-c097-676f67d3bfc2} {Suffrage:Voter ID:server-f84bdcb8-de9b-2d7d-f2c5-b9ef0301178f Address:f84bdcb8-de9b-2d7d-f2c5-b9ef0301178f} {Suffrage:Voter ID:server-2ba38a58-2488-128d-6661-007927f8479e Address:2ba38a58-2488-128d-6661-007927f8479e}]" +2026-11-29T10:29:50.172Z [INFO] server-2ba38a58-2488-128d-6661-007927f8479e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-27c9407c-5467-5a76-c097-676f67d3bfc2 Address:27c9407c-5467-5a76-c097-676f67d3bfc2} {Suffrage:Voter ID:server-f84bdcb8-de9b-2d7d-f2c5-b9ef0301178f Address:f84bdcb8-de9b-2d7d-f2c5-b9ef0301178f} {Suffrage:Voter ID:server-2ba38a58-2488-128d-6661-007927f8479e Address:2ba38a58-2488-128d-6661-007927f8479e}]" +2026-11-29T10:29:50.172Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:50.172Z [INFO] server-27c9407c-5467-5a76-c097-676f67d3bfc2: entering follower state: follower="Node at 27c9407c-5467-5a76-c097-676f67d3bfc2 [Follower]" leader-address= leader-id= +2026-11-29T10:29:50.172Z [INFO] server-f84bdcb8-de9b-2d7d-f2c5-b9ef0301178f: entering follower state: follower="Node at f84bdcb8-de9b-2d7d-f2c5-b9ef0301178f [Follower]" leader-address= leader-id= +2026-11-29T10:29:50.172Z [INFO] server-2ba38a58-2488-128d-6661-007927f8479e: entering follower state: follower="Node at 2ba38a58-2488-128d-6661-007927f8479e [Follower]" leader-address= leader-id= +2026-11-29T10:29:50.224Z [WARN] server-27c9407c-5467-5a76-c097-676f67d3bfc2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:50.224Z [INFO] server-27c9407c-5467-5a76-c097-676f67d3bfc2: entering candidate state: node="Node at 27c9407c-5467-5a76-c097-676f67d3bfc2 [Candidate]" term=2 +2026-11-29T10:29:50.224Z [INFO] server-27c9407c-5467-5a76-c097-676f67d3bfc2: election won: term=2 tally=2 +2026-11-29T10:29:50.224Z [INFO] server-27c9407c-5467-5a76-c097-676f67d3bfc2: entering leader state: leader="Node at 27c9407c-5467-5a76-c097-676f67d3bfc2 [Leader]" +2026-11-29T10:29:50.224Z [INFO] server-27c9407c-5467-5a76-c097-676f67d3bfc2: added peer, starting replication: peer=server-f84bdcb8-de9b-2d7d-f2c5-b9ef0301178f +2026-11-29T10:29:50.224Z [INFO] server-27c9407c-5467-5a76-c097-676f67d3bfc2: added peer, starting replication: peer=server-2ba38a58-2488-128d-6661-007927f8479e +2026-11-29T10:29:50.224Z [INFO] server-27c9407c-5467-5a76-c097-676f67d3bfc2: pipelining replication: peer="{Voter server-2ba38a58-2488-128d-6661-007927f8479e 2ba38a58-2488-128d-6661-007927f8479e}" +2026-11-29T10:29:50.224Z [INFO] server-27c9407c-5467-5a76-c097-676f67d3bfc2: pipelining replication: peer="{Voter server-f84bdcb8-de9b-2d7d-f2c5-b9ef0301178f f84bdcb8-de9b-2d7d-f2c5-b9ef0301178f}" +2026-11-29T10:29:50.329Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:50.224169379 +0000 UTC m=+100.959223521 (2 nodes), 51.81792ms from start of poll, 52.00056ms from cluster start. Timeout at 2026-11-29 10:29:50.329170739 +0000 UTC m=+101.064224841, 105.00132ms after stability +2026-11-29T10:29:50.329Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:50.434Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:50.329243859 +0000 UTC m=+101.064297961 (1 nodes), 1.76µs from start of poll, 157.075ms from cluster start. Timeout at 2026-11-29 10:29:50.434278739 +0000 UTC m=+101.169332841, 105.03488ms after stability +2026-11-29T10:29:50.434Z [INFO] server-27c9407c-5467-5a76-c097-676f67d3bfc2: aborting pipeline replication: peer="{Voter server-2ba38a58-2488-128d-6661-007927f8479e 2ba38a58-2488-128d-6661-007927f8479e}" +2026-11-29T10:29:50.434Z [INFO] server-27c9407c-5467-5a76-c097-676f67d3bfc2: aborting pipeline replication: peer="{Voter server-f84bdcb8-de9b-2d7d-f2c5-b9ef0301178f f84bdcb8-de9b-2d7d-f2c5-b9ef0301178f}" +--- PASS: TestRaft_ProtocolVersion_RejectRPC (0.26s) === RUN TestRaft_ProtocolVersion_Upgrade_1_2 raft_test.go:2151: Skipping unreliable test --- SKIP: TestRaft_ProtocolVersion_Upgrade_1_2 (0.00s) @@ -3597,269 +3626,267 @@ raft_test.go:2196: Skipping unreliable test --- SKIP: TestRaft_ProtocolVersion_Upgrade_2_3 (0.00s) === RUN TestRaft_LeaderID_Propagated -2025-10-27T04:03:47.544Z [INFO] server-af28f4b5-e635-9437-26c1-abaea1ec358f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-af28f4b5-e635-9437-26c1-abaea1ec358f Address:af28f4b5-e635-9437-26c1-abaea1ec358f} {Suffrage:Voter ID:server-aeb8d0b7-e442-4488-f6bb-2482f964aa70 Address:aeb8d0b7-e442-4488-f6bb-2482f964aa70} {Suffrage:Voter ID:server-80c7e100-29c8-bcf9-d78f-c971835bf4da Address:80c7e100-29c8-bcf9-d78f-c971835bf4da}]" -2025-10-27T04:03:47.544Z [INFO] server-aeb8d0b7-e442-4488-f6bb-2482f964aa70: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-af28f4b5-e635-9437-26c1-abaea1ec358f Address:af28f4b5-e635-9437-26c1-abaea1ec358f} {Suffrage:Voter ID:server-aeb8d0b7-e442-4488-f6bb-2482f964aa70 Address:aeb8d0b7-e442-4488-f6bb-2482f964aa70} {Suffrage:Voter ID:server-80c7e100-29c8-bcf9-d78f-c971835bf4da Address:80c7e100-29c8-bcf9-d78f-c971835bf4da}]" -2025-10-27T04:03:47.544Z [INFO] server-af28f4b5-e635-9437-26c1-abaea1ec358f: entering follower state: follower="Node at af28f4b5-e635-9437-26c1-abaea1ec358f [Follower]" leader-address= leader-id= -2025-10-27T04:03:47.544Z [INFO] server-aeb8d0b7-e442-4488-f6bb-2482f964aa70: entering follower state: follower="Node at aeb8d0b7-e442-4488-f6bb-2482f964aa70 [Follower]" leader-address= leader-id= -2025-10-27T04:03:47.544Z [INFO] server-80c7e100-29c8-bcf9-d78f-c971835bf4da: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-af28f4b5-e635-9437-26c1-abaea1ec358f Address:af28f4b5-e635-9437-26c1-abaea1ec358f} {Suffrage:Voter ID:server-aeb8d0b7-e442-4488-f6bb-2482f964aa70 Address:aeb8d0b7-e442-4488-f6bb-2482f964aa70} {Suffrage:Voter ID:server-80c7e100-29c8-bcf9-d78f-c971835bf4da Address:80c7e100-29c8-bcf9-d78f-c971835bf4da}]" -2025-10-27T04:03:47.544Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:47.544Z [INFO] server-80c7e100-29c8-bcf9-d78f-c971835bf4da: entering follower state: follower="Node at 80c7e100-29c8-bcf9-d78f-c971835bf4da [Follower]" leader-address= leader-id= -2025-10-27T04:03:47.616Z [WARN] server-aeb8d0b7-e442-4488-f6bb-2482f964aa70: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:47.616Z [INFO] server-aeb8d0b7-e442-4488-f6bb-2482f964aa70: entering candidate state: node="Node at aeb8d0b7-e442-4488-f6bb-2482f964aa70 [Candidate]" term=2 -2025-10-27T04:03:47.616Z [INFO] server-aeb8d0b7-e442-4488-f6bb-2482f964aa70: election won: term=2 tally=2 -2025-10-27T04:03:47.616Z [INFO] server-aeb8d0b7-e442-4488-f6bb-2482f964aa70: entering leader state: leader="Node at aeb8d0b7-e442-4488-f6bb-2482f964aa70 [Leader]" -2025-10-27T04:03:47.616Z [INFO] server-aeb8d0b7-e442-4488-f6bb-2482f964aa70: added peer, starting replication: peer=server-af28f4b5-e635-9437-26c1-abaea1ec358f -2025-10-27T04:03:47.616Z [INFO] server-aeb8d0b7-e442-4488-f6bb-2482f964aa70: added peer, starting replication: peer=server-80c7e100-29c8-bcf9-d78f-c971835bf4da -2025-10-27T04:03:47.616Z [INFO] server-aeb8d0b7-e442-4488-f6bb-2482f964aa70: pipelining replication: peer="{Voter server-af28f4b5-e635-9437-26c1-abaea1ec358f af28f4b5-e635-9437-26c1-abaea1ec358f}" -2025-10-27T04:03:47.616Z [INFO] server-aeb8d0b7-e442-4488-f6bb-2482f964aa70: pipelining replication: peer="{Voter server-80c7e100-29c8-bcf9-d78f-c971835bf4da 80c7e100-29c8-bcf9-d78f-c971835bf4da}" -2025-10-27T04:03:47.721Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:47.616193074 +0000 UTC m=+102.611065512 (1 nodes), 71.282087ms from start of poll, 71.522169ms from cluster start. Timeout at 2025-10-27 04:03:47.721199474 +0000 UTC m=+102.716071952, 105.00644ms after stability -2025-10-27T04:03:47.721Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:47.826Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:47.721701115 +0000 UTC m=+102.716573593 (1 nodes), 1.96µs from start of poll, 177.03025ms from cluster start. Timeout at 2025-10-27 04:03:47.826715035 +0000 UTC m=+102.821587473, 105.01388ms after stability -2025-10-27T04:03:47.826Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:47.932Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:47.826949394 +0000 UTC m=+102.821821832 (1 nodes), 920ns from start of poll, 282.278489ms from cluster start. Timeout at 2025-10-27 04:03:47.931968234 +0000 UTC m=+102.926840672, 105.01884ms after stability -2025-10-27T04:03:47.932Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:48.038Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:47.932195513 +0000 UTC m=+102.927067951 (1 nodes), 680ns from start of poll, 387.524608ms from cluster start. Timeout at 2025-10-27 04:03:48.037203113 +0000 UTC m=+103.032075551, 105.0076ms after stability -2025-10-27T04:03:48.038Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:48.144Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:48.038236917 +0000 UTC m=+103.033109355 (1 nodes), 720ns from start of poll, 493.566012ms from cluster start. Timeout at 2025-10-27 04:03:48.143249478 +0000 UTC m=+103.138121916, 105.012561ms after stability -2025-10-27T04:03:48.144Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:48.249Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:48.144300962 +0000 UTC m=+103.139173400 (1 nodes), 840ns from start of poll, 599.630057ms from cluster start. Timeout at 2025-10-27 04:03:48.249326243 +0000 UTC m=+103.244198681, 105.025281ms after stability -2025-10-27T04:03:48.354Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:48.460Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:48.355002841 +0000 UTC m=+103.349875239 (1 nodes), 1.48µs from start of poll, 810.331896ms from cluster start. Timeout at 2025-10-27 04:03:48.460019561 +0000 UTC m=+103.454891999, 105.01676ms after stability -2025-10-27T04:03:48.460Z [INFO] server-aeb8d0b7-e442-4488-f6bb-2482f964aa70: aborting pipeline replication: peer="{Voter server-80c7e100-29c8-bcf9-d78f-c971835bf4da 80c7e100-29c8-bcf9-d78f-c971835bf4da}" -2025-10-27T04:03:48.460Z [INFO] server-aeb8d0b7-e442-4488-f6bb-2482f964aa70: aborting pipeline replication: peer="{Voter server-af28f4b5-e635-9437-26c1-abaea1ec358f af28f4b5-e635-9437-26c1-abaea1ec358f}" ---- PASS: TestRaft_LeaderID_Propagated (0.92s) +2026-11-29T10:29:50.434Z [INFO] server-8572d727-cf8c-d935-add6-c50c645696a1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8572d727-cf8c-d935-add6-c50c645696a1 Address:8572d727-cf8c-d935-add6-c50c645696a1} {Suffrage:Voter ID:server-5253e6ed-3135-5064-f2ac-d9cf96e7eec1 Address:5253e6ed-3135-5064-f2ac-d9cf96e7eec1} {Suffrage:Voter ID:server-99591446-cc47-10d3-ece7-865c400c43f2 Address:99591446-cc47-10d3-ece7-865c400c43f2}]" +2026-11-29T10:29:50.435Z [INFO] server-5253e6ed-3135-5064-f2ac-d9cf96e7eec1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8572d727-cf8c-d935-add6-c50c645696a1 Address:8572d727-cf8c-d935-add6-c50c645696a1} {Suffrage:Voter ID:server-5253e6ed-3135-5064-f2ac-d9cf96e7eec1 Address:5253e6ed-3135-5064-f2ac-d9cf96e7eec1} {Suffrage:Voter ID:server-99591446-cc47-10d3-ece7-865c400c43f2 Address:99591446-cc47-10d3-ece7-865c400c43f2}]" +2026-11-29T10:29:50.434Z [INFO] server-8572d727-cf8c-d935-add6-c50c645696a1: entering follower state: follower="Node at 8572d727-cf8c-d935-add6-c50c645696a1 [Follower]" leader-address= leader-id= +2026-11-29T10:29:50.435Z [INFO] server-99591446-cc47-10d3-ece7-865c400c43f2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8572d727-cf8c-d935-add6-c50c645696a1 Address:8572d727-cf8c-d935-add6-c50c645696a1} {Suffrage:Voter ID:server-5253e6ed-3135-5064-f2ac-d9cf96e7eec1 Address:5253e6ed-3135-5064-f2ac-d9cf96e7eec1} {Suffrage:Voter ID:server-99591446-cc47-10d3-ece7-865c400c43f2 Address:99591446-cc47-10d3-ece7-865c400c43f2}]" +2026-11-29T10:29:50.435Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:50.435Z [INFO] server-5253e6ed-3135-5064-f2ac-d9cf96e7eec1: entering follower state: follower="Node at 5253e6ed-3135-5064-f2ac-d9cf96e7eec1 [Follower]" leader-address= leader-id= +2026-11-29T10:29:50.435Z [INFO] server-99591446-cc47-10d3-ece7-865c400c43f2: entering follower state: follower="Node at 99591446-cc47-10d3-ece7-865c400c43f2 [Follower]" leader-address= leader-id= +2026-11-29T10:29:50.488Z [WARN] server-8572d727-cf8c-d935-add6-c50c645696a1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:50.488Z [INFO] server-8572d727-cf8c-d935-add6-c50c645696a1: entering candidate state: node="Node at 8572d727-cf8c-d935-add6-c50c645696a1 [Candidate]" term=2 +2026-11-29T10:29:50.489Z [INFO] server-8572d727-cf8c-d935-add6-c50c645696a1: election won: term=2 tally=2 +2026-11-29T10:29:50.489Z [INFO] server-8572d727-cf8c-d935-add6-c50c645696a1: entering leader state: leader="Node at 8572d727-cf8c-d935-add6-c50c645696a1 [Leader]" +2026-11-29T10:29:50.489Z [INFO] server-8572d727-cf8c-d935-add6-c50c645696a1: added peer, starting replication: peer=server-5253e6ed-3135-5064-f2ac-d9cf96e7eec1 +2026-11-29T10:29:50.489Z [INFO] server-8572d727-cf8c-d935-add6-c50c645696a1: added peer, starting replication: peer=server-99591446-cc47-10d3-ece7-865c400c43f2 +2026-11-29T10:29:50.489Z [INFO] server-8572d727-cf8c-d935-add6-c50c645696a1: pipelining replication: peer="{Voter server-99591446-cc47-10d3-ece7-865c400c43f2 99591446-cc47-10d3-ece7-865c400c43f2}" +2026-11-29T10:29:50.489Z [INFO] server-8572d727-cf8c-d935-add6-c50c645696a1: pipelining replication: peer="{Voter server-5253e6ed-3135-5064-f2ac-d9cf96e7eec1 5253e6ed-3135-5064-f2ac-d9cf96e7eec1}" +2026-11-29T10:29:50.594Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:50.489205059 +0000 UTC m=+101.224259161 (1 nodes), 54.13676ms from start of poll, 54.3226ms from cluster start. Timeout at 2026-11-29 10:29:50.594206219 +0000 UTC m=+101.329260321, 105.00116ms after stability +2026-11-29T10:29:50.594Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:50.699Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:50.594888739 +0000 UTC m=+101.329942841 (1 nodes), 1.24µs from start of poll, 160.00628ms from cluster start. Timeout at 2026-11-29 10:29:50.699915499 +0000 UTC m=+101.434969601, 105.02676ms after stability +2026-11-29T10:29:50.700Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:50.805Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:50.700325819 +0000 UTC m=+101.435379921 (1 nodes), 600ns from start of poll, 265.44336ms from cluster start. Timeout at 2026-11-29 10:29:50.805358499 +0000 UTC m=+101.540412641, 105.03272ms after stability +2026-11-29T10:29:50.805Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:50.911Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:50.805582579 +0000 UTC m=+101.540636721 (1 nodes), 680ns from start of poll, 370.70016ms from cluster start. Timeout at 2026-11-29 10:29:50.910609819 +0000 UTC m=+101.645663881, 105.02716ms after stability +2026-11-29T10:29:50.911Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:51.017Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:50.911805099 +0000 UTC m=+101.646859201 (1 nodes), 920ns from start of poll, 476.92264ms from cluster start. Timeout at 2026-11-29 10:29:51.016814739 +0000 UTC m=+101.751868841, 105.00964ms after stability +2026-11-29T10:29:51.017Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:51.122Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:51.017632339 +0000 UTC m=+101.752686441 (1 nodes), 800ns from start of poll, 582.74988ms from cluster start. Timeout at 2026-11-29 10:29:51.122659019 +0000 UTC m=+101.857713121, 105.02668ms after stability +2026-11-29T10:29:51.228Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:51.333Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:51.228587459 +0000 UTC m=+101.963641561 (1 nodes), 1.52µs from start of poll, 793.705ms from cluster start. Timeout at 2026-11-29 10:29:51.333606659 +0000 UTC m=+102.068660841, 105.01928ms after stability +2026-11-29T10:29:51.333Z [INFO] server-8572d727-cf8c-d935-add6-c50c645696a1: aborting pipeline replication: peer="{Voter server-99591446-cc47-10d3-ece7-865c400c43f2 99591446-cc47-10d3-ece7-865c400c43f2}" +2026-11-29T10:29:51.333Z [INFO] server-8572d727-cf8c-d935-add6-c50c645696a1: aborting pipeline replication: peer="{Voter server-5253e6ed-3135-5064-f2ac-d9cf96e7eec1 5253e6ed-3135-5064-f2ac-d9cf96e7eec1}" +--- PASS: TestRaft_LeaderID_Propagated (0.90s) === RUN TestRaft_LeadershipTransferInProgress --- PASS: TestRaft_LeadershipTransferInProgress (0.00s) === RUN TestRaft_LeadershipTransferPickServer --- PASS: TestRaft_LeadershipTransferPickServer (0.00s) === RUN TestRaft_LeadershipTransfer -2025-10-27T04:03:48.461Z [INFO] server-53cd33be-5d0c-a21b-16f2-eb5f10c431b9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-53cd33be-5d0c-a21b-16f2-eb5f10c431b9 Address:53cd33be-5d0c-a21b-16f2-eb5f10c431b9} {Suffrage:Voter ID:server-4470a8de-d871-3a93-f7b7-d5ce3dc6d12c Address:4470a8de-d871-3a93-f7b7-d5ce3dc6d12c} {Suffrage:Voter ID:server-c6dd3427-cc47-2693-11d3-4c07743b440d Address:c6dd3427-cc47-2693-11d3-4c07743b440d}]" -2025-10-27T04:03:48.461Z [INFO] server-4470a8de-d871-3a93-f7b7-d5ce3dc6d12c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-53cd33be-5d0c-a21b-16f2-eb5f10c431b9 Address:53cd33be-5d0c-a21b-16f2-eb5f10c431b9} {Suffrage:Voter ID:server-4470a8de-d871-3a93-f7b7-d5ce3dc6d12c Address:4470a8de-d871-3a93-f7b7-d5ce3dc6d12c} {Suffrage:Voter ID:server-c6dd3427-cc47-2693-11d3-4c07743b440d Address:c6dd3427-cc47-2693-11d3-4c07743b440d}]" -2025-10-27T04:03:48.461Z [INFO] server-c6dd3427-cc47-2693-11d3-4c07743b440d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-53cd33be-5d0c-a21b-16f2-eb5f10c431b9 Address:53cd33be-5d0c-a21b-16f2-eb5f10c431b9} {Suffrage:Voter ID:server-4470a8de-d871-3a93-f7b7-d5ce3dc6d12c Address:4470a8de-d871-3a93-f7b7-d5ce3dc6d12c} {Suffrage:Voter ID:server-c6dd3427-cc47-2693-11d3-4c07743b440d Address:c6dd3427-cc47-2693-11d3-4c07743b440d}]" -2025-10-27T04:03:48.461Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:48.461Z [INFO] server-53cd33be-5d0c-a21b-16f2-eb5f10c431b9: entering follower state: follower="Node at 53cd33be-5d0c-a21b-16f2-eb5f10c431b9 [Follower]" leader-address= leader-id= -2025-10-27T04:03:48.461Z [INFO] server-4470a8de-d871-3a93-f7b7-d5ce3dc6d12c: entering follower state: follower="Node at 4470a8de-d871-3a93-f7b7-d5ce3dc6d12c [Follower]" leader-address= leader-id= -2025-10-27T04:03:48.461Z [INFO] server-c6dd3427-cc47-2693-11d3-4c07743b440d: entering follower state: follower="Node at c6dd3427-cc47-2693-11d3-4c07743b440d [Follower]" leader-address= leader-id= -2025-10-27T04:03:48.526Z [WARN] server-c6dd3427-cc47-2693-11d3-4c07743b440d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:48.526Z [INFO] server-c6dd3427-cc47-2693-11d3-4c07743b440d: entering candidate state: node="Node at c6dd3427-cc47-2693-11d3-4c07743b440d [Candidate]" term=2 -2025-10-27T04:03:48.526Z [INFO] server-c6dd3427-cc47-2693-11d3-4c07743b440d: election won: term=2 tally=2 -2025-10-27T04:03:48.527Z [INFO] server-c6dd3427-cc47-2693-11d3-4c07743b440d: entering leader state: leader="Node at c6dd3427-cc47-2693-11d3-4c07743b440d [Leader]" -2025-10-27T04:03:48.527Z [INFO] server-c6dd3427-cc47-2693-11d3-4c07743b440d: added peer, starting replication: peer=server-53cd33be-5d0c-a21b-16f2-eb5f10c431b9 -2025-10-27T04:03:48.527Z [INFO] server-c6dd3427-cc47-2693-11d3-4c07743b440d: added peer, starting replication: peer=server-4470a8de-d871-3a93-f7b7-d5ce3dc6d12c -2025-10-27T04:03:48.527Z [INFO] server-c6dd3427-cc47-2693-11d3-4c07743b440d: pipelining replication: peer="{Voter server-4470a8de-d871-3a93-f7b7-d5ce3dc6d12c 4470a8de-d871-3a93-f7b7-d5ce3dc6d12c}" -2025-10-27T04:03:48.527Z [INFO] server-c6dd3427-cc47-2693-11d3-4c07743b440d: pipelining replication: peer="{Voter server-53cd33be-5d0c-a21b-16f2-eb5f10c431b9 53cd33be-5d0c-a21b-16f2-eb5f10c431b9}" -2025-10-27T04:03:48.632Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:48.527014497 +0000 UTC m=+103.521886975 (1 nodes), 65.386327ms from start of poll, 65.603929ms from cluster start. Timeout at 2025-10-27 04:03:48.632018137 +0000 UTC m=+103.626890655, 105.00368ms after stability -2025-10-27T04:03:48.632Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:48.738Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:48.63293646 +0000 UTC m=+103.627808978 (1 nodes), 2.36µs from start of poll, 171.525932ms from cluster start. Timeout at 2025-10-27 04:03:48.73795706 +0000 UTC m=+103.732829498, 105.02052ms after stability -2025-10-27T04:03:48.738Z [INFO] server-53cd33be-5d0c-a21b-16f2-eb5f10c431b9: entering candidate state: node="Node at 53cd33be-5d0c-a21b-16f2-eb5f10c431b9 [Candidate]" term=3 -2025-10-27T04:03:48.738Z [INFO] server-c6dd3427-cc47-2693-11d3-4c07743b440d: entering follower state: follower="Node at c6dd3427-cc47-2693-11d3-4c07743b440d [Follower]" leader-address= leader-id= -2025-10-27T04:03:48.738Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:48.738Z [INFO] server-c6dd3427-cc47-2693-11d3-4c07743b440d: aborting pipeline replication: peer="{Voter server-4470a8de-d871-3a93-f7b7-d5ce3dc6d12c 4470a8de-d871-3a93-f7b7-d5ce3dc6d12c}" -2025-10-27T04:03:48.738Z [INFO] server-53cd33be-5d0c-a21b-16f2-eb5f10c431b9: election won: term=3 tally=2 -2025-10-27T04:03:48.738Z [INFO] server-53cd33be-5d0c-a21b-16f2-eb5f10c431b9: entering leader state: leader="Node at 53cd33be-5d0c-a21b-16f2-eb5f10c431b9 [Leader]" -2025-10-27T04:03:48.738Z [INFO] server-c6dd3427-cc47-2693-11d3-4c07743b440d: aborting pipeline replication: peer="{Voter server-53cd33be-5d0c-a21b-16f2-eb5f10c431b9 53cd33be-5d0c-a21b-16f2-eb5f10c431b9}" -2025-10-27T04:03:48.739Z [INFO] server-53cd33be-5d0c-a21b-16f2-eb5f10c431b9: added peer, starting replication: peer=server-4470a8de-d871-3a93-f7b7-d5ce3dc6d12c -2025-10-27T04:03:48.739Z [INFO] server-53cd33be-5d0c-a21b-16f2-eb5f10c431b9: added peer, starting replication: peer=server-c6dd3427-cc47-2693-11d3-4c07743b440d -2025-10-27T04:03:48.739Z [INFO] server-53cd33be-5d0c-a21b-16f2-eb5f10c431b9: pipelining replication: peer="{Voter server-4470a8de-d871-3a93-f7b7-d5ce3dc6d12c 4470a8de-d871-3a93-f7b7-d5ce3dc6d12c}" -2025-10-27T04:03:48.739Z [INFO] server-53cd33be-5d0c-a21b-16f2-eb5f10c431b9: pipelining replication: peer="{Voter server-c6dd3427-cc47-2693-11d3-4c07743b440d c6dd3427-cc47-2693-11d3-4c07743b440d}" -2025-10-27T04:03:48.844Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:48.739025505 +0000 UTC m=+103.733897943 (1 nodes), 134.561µs from start of poll, 277.614897ms from cluster start. Timeout at 2025-10-27 04:03:48.844043706 +0000 UTC m=+103.838916144, 105.018201ms after stability -2025-10-27T04:03:48.844Z [INFO] server-53cd33be-5d0c-a21b-16f2-eb5f10c431b9: aborting pipeline replication: peer="{Voter server-c6dd3427-cc47-2693-11d3-4c07743b440d c6dd3427-cc47-2693-11d3-4c07743b440d}" -2025-10-27T04:03:48.844Z [INFO] server-53cd33be-5d0c-a21b-16f2-eb5f10c431b9: aborting pipeline replication: peer="{Voter server-4470a8de-d871-3a93-f7b7-d5ce3dc6d12c 4470a8de-d871-3a93-f7b7-d5ce3dc6d12c}" ---- PASS: TestRaft_LeadershipTransfer (0.38s) +2026-11-29T10:29:51.334Z [INFO] server-0f062889-1d04-b3b4-d9b0-aa2963e73356: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0f062889-1d04-b3b4-d9b0-aa2963e73356 Address:0f062889-1d04-b3b4-d9b0-aa2963e73356} {Suffrage:Voter ID:server-16f3840a-819d-aa66-4a5e-2117592ae5ae Address:16f3840a-819d-aa66-4a5e-2117592ae5ae} {Suffrage:Voter ID:server-c1b8bdf6-dea9-1675-4b2e-caf7ed2f5254 Address:c1b8bdf6-dea9-1675-4b2e-caf7ed2f5254}]" +2026-11-29T10:29:51.334Z [INFO] server-16f3840a-819d-aa66-4a5e-2117592ae5ae: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0f062889-1d04-b3b4-d9b0-aa2963e73356 Address:0f062889-1d04-b3b4-d9b0-aa2963e73356} {Suffrage:Voter ID:server-16f3840a-819d-aa66-4a5e-2117592ae5ae Address:16f3840a-819d-aa66-4a5e-2117592ae5ae} {Suffrage:Voter ID:server-c1b8bdf6-dea9-1675-4b2e-caf7ed2f5254 Address:c1b8bdf6-dea9-1675-4b2e-caf7ed2f5254}]" +2026-11-29T10:29:51.334Z [INFO] server-c1b8bdf6-dea9-1675-4b2e-caf7ed2f5254: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0f062889-1d04-b3b4-d9b0-aa2963e73356 Address:0f062889-1d04-b3b4-d9b0-aa2963e73356} {Suffrage:Voter ID:server-16f3840a-819d-aa66-4a5e-2117592ae5ae Address:16f3840a-819d-aa66-4a5e-2117592ae5ae} {Suffrage:Voter ID:server-c1b8bdf6-dea9-1675-4b2e-caf7ed2f5254 Address:c1b8bdf6-dea9-1675-4b2e-caf7ed2f5254}]" +2026-11-29T10:29:51.334Z [INFO] server-0f062889-1d04-b3b4-d9b0-aa2963e73356: entering follower state: follower="Node at 0f062889-1d04-b3b4-d9b0-aa2963e73356 [Follower]" leader-address= leader-id= +2026-11-29T10:29:51.334Z [INFO] server-c1b8bdf6-dea9-1675-4b2e-caf7ed2f5254: entering follower state: follower="Node at c1b8bdf6-dea9-1675-4b2e-caf7ed2f5254 [Follower]" leader-address= leader-id= +2026-11-29T10:29:51.334Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:51.334Z [INFO] server-16f3840a-819d-aa66-4a5e-2117592ae5ae: entering follower state: follower="Node at 16f3840a-819d-aa66-4a5e-2117592ae5ae [Follower]" leader-address= leader-id= +2026-11-29T10:29:51.386Z [WARN] server-0f062889-1d04-b3b4-d9b0-aa2963e73356: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:51.386Z [INFO] server-0f062889-1d04-b3b4-d9b0-aa2963e73356: entering candidate state: node="Node at 0f062889-1d04-b3b4-d9b0-aa2963e73356 [Candidate]" term=2 +2026-11-29T10:29:51.386Z [INFO] server-0f062889-1d04-b3b4-d9b0-aa2963e73356: election won: term=2 tally=2 +2026-11-29T10:29:51.386Z [INFO] server-0f062889-1d04-b3b4-d9b0-aa2963e73356: entering leader state: leader="Node at 0f062889-1d04-b3b4-d9b0-aa2963e73356 [Leader]" +2026-11-29T10:29:51.386Z [INFO] server-0f062889-1d04-b3b4-d9b0-aa2963e73356: added peer, starting replication: peer=server-16f3840a-819d-aa66-4a5e-2117592ae5ae +2026-11-29T10:29:51.386Z [INFO] server-0f062889-1d04-b3b4-d9b0-aa2963e73356: added peer, starting replication: peer=server-c1b8bdf6-dea9-1675-4b2e-caf7ed2f5254 +2026-11-29T10:29:51.386Z [INFO] server-0f062889-1d04-b3b4-d9b0-aa2963e73356: pipelining replication: peer="{Voter server-c1b8bdf6-dea9-1675-4b2e-caf7ed2f5254 c1b8bdf6-dea9-1675-4b2e-caf7ed2f5254}" +2026-11-29T10:29:51.386Z [INFO] server-0f062889-1d04-b3b4-d9b0-aa2963e73356: pipelining replication: peer="{Voter server-16f3840a-819d-aa66-4a5e-2117592ae5ae 16f3840a-819d-aa66-4a5e-2117592ae5ae}" +2026-11-29T10:29:51.492Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:51.386544219 +0000 UTC m=+102.121598361 (1 nodes), 52.0982ms from start of poll, 52.35728ms from cluster start. Timeout at 2026-11-29 10:29:51.491546219 +0000 UTC m=+102.226600321, 105.00196ms after stability +2026-11-29T10:29:51.492Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:51.597Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:51.492522579 +0000 UTC m=+102.227576641 (1 nodes), 1.72µs from start of poll, 158.33556ms from cluster start. Timeout at 2026-11-29 10:29:51.597547459 +0000 UTC m=+102.332601561, 105.02492ms after stability +2026-11-29T10:29:51.597Z [INFO] server-16f3840a-819d-aa66-4a5e-2117592ae5ae: entering candidate state: node="Node at 16f3840a-819d-aa66-4a5e-2117592ae5ae [Candidate]" term=3 +2026-11-29T10:29:51.597Z [INFO] server-16f3840a-819d-aa66-4a5e-2117592ae5ae: election won: term=3 tally=2 +2026-11-29T10:29:51.597Z [INFO] server-16f3840a-819d-aa66-4a5e-2117592ae5ae: entering leader state: leader="Node at 16f3840a-819d-aa66-4a5e-2117592ae5ae [Leader]" +2026-11-29T10:29:51.597Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:51.597Z [INFO] server-16f3840a-819d-aa66-4a5e-2117592ae5ae: added peer, starting replication: peer=server-0f062889-1d04-b3b4-d9b0-aa2963e73356 +2026-11-29T10:29:51.597Z [INFO] server-16f3840a-819d-aa66-4a5e-2117592ae5ae: added peer, starting replication: peer=server-c1b8bdf6-dea9-1675-4b2e-caf7ed2f5254 +2026-11-29T10:29:51.597Z [INFO] server-0f062889-1d04-b3b4-d9b0-aa2963e73356: entering follower state: follower="Node at 0f062889-1d04-b3b4-d9b0-aa2963e73356 [Follower]" leader-address= leader-id= +2026-11-29T10:29:51.597Z [INFO] server-0f062889-1d04-b3b4-d9b0-aa2963e73356: aborting pipeline replication: peer="{Voter server-16f3840a-819d-aa66-4a5e-2117592ae5ae 16f3840a-819d-aa66-4a5e-2117592ae5ae}" +2026-11-29T10:29:51.597Z [INFO] server-0f062889-1d04-b3b4-d9b0-aa2963e73356: aborting pipeline replication: peer="{Voter server-c1b8bdf6-dea9-1675-4b2e-caf7ed2f5254 c1b8bdf6-dea9-1675-4b2e-caf7ed2f5254}" +2026-11-29T10:29:51.598Z [INFO] server-16f3840a-819d-aa66-4a5e-2117592ae5ae: pipelining replication: peer="{Voter server-0f062889-1d04-b3b4-d9b0-aa2963e73356 0f062889-1d04-b3b4-d9b0-aa2963e73356}" +2026-11-29T10:29:51.598Z [INFO] server-16f3840a-819d-aa66-4a5e-2117592ae5ae: pipelining replication: peer="{Voter server-c1b8bdf6-dea9-1675-4b2e-caf7ed2f5254 c1b8bdf6-dea9-1675-4b2e-caf7ed2f5254}" +2026-11-29T10:29:51.704Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:51.597991259 +0000 UTC m=+102.333045361 (1 nodes), 16.96µs from start of poll, 263.80428ms from cluster start. Timeout at 2026-11-29 10:29:51.703012939 +0000 UTC m=+102.438067001, 105.02164ms after stability +2026-11-29T10:29:51.704Z [INFO] server-16f3840a-819d-aa66-4a5e-2117592ae5ae: aborting pipeline replication: peer="{Voter server-c1b8bdf6-dea9-1675-4b2e-caf7ed2f5254 c1b8bdf6-dea9-1675-4b2e-caf7ed2f5254}" +2026-11-29T10:29:51.704Z [INFO] server-16f3840a-819d-aa66-4a5e-2117592ae5ae: aborting pipeline replication: peer="{Voter server-0f062889-1d04-b3b4-d9b0-aa2963e73356 0f062889-1d04-b3b4-d9b0-aa2963e73356}" +--- PASS: TestRaft_LeadershipTransfer (0.37s) === RUN TestRaft_LeadershipTransferWithOneNode -2025-10-27T04:03:48.845Z [INFO] server-ffe0b9c2-583d-9fe5-1e6e-7ee60e103af5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ffe0b9c2-583d-9fe5-1e6e-7ee60e103af5 Address:ffe0b9c2-583d-9fe5-1e6e-7ee60e103af5}]" -2025-10-27T04:03:48.845Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:48.845Z [INFO] server-ffe0b9c2-583d-9fe5-1e6e-7ee60e103af5: entering follower state: follower="Node at ffe0b9c2-583d-9fe5-1e6e-7ee60e103af5 [Follower]" leader-address= leader-id= -2025-10-27T04:03:48.945Z [WARN] server-ffe0b9c2-583d-9fe5-1e6e-7ee60e103af5: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:48.945Z [INFO] server-ffe0b9c2-583d-9fe5-1e6e-7ee60e103af5: entering candidate state: node="Node at ffe0b9c2-583d-9fe5-1e6e-7ee60e103af5 [Candidate]" term=2 -2025-10-27T04:03:48.945Z [INFO] server-ffe0b9c2-583d-9fe5-1e6e-7ee60e103af5: election won: term=2 tally=1 -2025-10-27T04:03:48.945Z [INFO] server-ffe0b9c2-583d-9fe5-1e6e-7ee60e103af5: entering leader state: leader="Node at ffe0b9c2-583d-9fe5-1e6e-7ee60e103af5 [Leader]" -2025-10-27T04:03:49.050Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:48.945515156 +0000 UTC m=+103.940387554 (1 nodes), 99.864603ms from start of poll, 100.057604ms from cluster start. Timeout at 2025-10-27 04:03:49.050527876 +0000 UTC m=+104.045400314, 105.01276ms after stability ---- PASS: TestRaft_LeadershipTransferWithOneNode (0.21s) +2026-11-29T10:29:51.704Z [INFO] server-85fe6d7e-3d0b-afce-4c7f-125a3fa1ccc7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-85fe6d7e-3d0b-afce-4c7f-125a3fa1ccc7 Address:85fe6d7e-3d0b-afce-4c7f-125a3fa1ccc7}]" +2026-11-29T10:29:51.704Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:51.704Z [INFO] server-85fe6d7e-3d0b-afce-4c7f-125a3fa1ccc7: entering follower state: follower="Node at 85fe6d7e-3d0b-afce-4c7f-125a3fa1ccc7 [Follower]" leader-address= leader-id= +2026-11-29T10:29:51.791Z [WARN] server-85fe6d7e-3d0b-afce-4c7f-125a3fa1ccc7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:51.791Z [INFO] server-85fe6d7e-3d0b-afce-4c7f-125a3fa1ccc7: entering candidate state: node="Node at 85fe6d7e-3d0b-afce-4c7f-125a3fa1ccc7 [Candidate]" term=2 +2026-11-29T10:29:51.791Z [INFO] server-85fe6d7e-3d0b-afce-4c7f-125a3fa1ccc7: election won: term=2 tally=1 +2026-11-29T10:29:51.791Z [INFO] server-85fe6d7e-3d0b-afce-4c7f-125a3fa1ccc7: entering leader state: leader="Node at 85fe6d7e-3d0b-afce-4c7f-125a3fa1ccc7 [Leader]" +2026-11-29T10:29:51.897Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:51.791932979 +0000 UTC m=+102.526987081 (1 nodes), 87.16532ms from start of poll, 87.32016ms from cluster start. Timeout at 2026-11-29 10:29:51.896940499 +0000 UTC m=+102.631994641, 105.00756ms after stability +--- PASS: TestRaft_LeadershipTransferWithOneNode (0.19s) === RUN TestRaft_LeadershipTransferWithSevenNodes -2025-10-27T04:03:49.051Z [INFO] server-5c569b65-fa95-17b7-11ed-18b3ced491d3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5c569b65-fa95-17b7-11ed-18b3ced491d3 Address:5c569b65-fa95-17b7-11ed-18b3ced491d3} {Suffrage:Voter ID:server-3b8a6284-9959-35d0-14b4-6c5076aa69a6 Address:3b8a6284-9959-35d0-14b4-6c5076aa69a6} {Suffrage:Voter ID:server-2834bd0c-00ff-c247-8130-de890b8995ed Address:2834bd0c-00ff-c247-8130-de890b8995ed} {Suffrage:Voter ID:server-3590827f-deba-b006-11f8-68842fe0cfc1 Address:3590827f-deba-b006-11f8-68842fe0cfc1} {Suffrage:Voter ID:server-a40091af-1525-1cee-4890-76db3935066c Address:a40091af-1525-1cee-4890-76db3935066c} {Suffrage:Voter ID:server-8c4b5bef-7a83-7b28-894c-5f89f2ec8d15 Address:8c4b5bef-7a83-7b28-894c-5f89f2ec8d15} {Suffrage:Voter ID:server-f9657fed-32c5-3775-39b9-58219cc1a442 Address:f9657fed-32c5-3775-39b9-58219cc1a442}]" -2025-10-27T04:03:49.051Z [INFO] server-3b8a6284-9959-35d0-14b4-6c5076aa69a6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5c569b65-fa95-17b7-11ed-18b3ced491d3 Address:5c569b65-fa95-17b7-11ed-18b3ced491d3} {Suffrage:Voter ID:server-3b8a6284-9959-35d0-14b4-6c5076aa69a6 Address:3b8a6284-9959-35d0-14b4-6c5076aa69a6} {Suffrage:Voter ID:server-2834bd0c-00ff-c247-8130-de890b8995ed Address:2834bd0c-00ff-c247-8130-de890b8995ed} {Suffrage:Voter ID:server-3590827f-deba-b006-11f8-68842fe0cfc1 Address:3590827f-deba-b006-11f8-68842fe0cfc1} {Suffrage:Voter ID:server-a40091af-1525-1cee-4890-76db3935066c Address:a40091af-1525-1cee-4890-76db3935066c} {Suffrage:Voter ID:server-8c4b5bef-7a83-7b28-894c-5f89f2ec8d15 Address:8c4b5bef-7a83-7b28-894c-5f89f2ec8d15} {Suffrage:Voter ID:server-f9657fed-32c5-3775-39b9-58219cc1a442 Address:f9657fed-32c5-3775-39b9-58219cc1a442}]" -2025-10-27T04:03:49.051Z [INFO] server-5c569b65-fa95-17b7-11ed-18b3ced491d3: entering follower state: follower="Node at 5c569b65-fa95-17b7-11ed-18b3ced491d3 [Follower]" leader-address= leader-id= -2025-10-27T04:03:49.051Z [INFO] server-3b8a6284-9959-35d0-14b4-6c5076aa69a6: entering follower state: follower="Node at 3b8a6284-9959-35d0-14b4-6c5076aa69a6 [Follower]" leader-address= leader-id= -2025-10-27T04:03:49.051Z [INFO] server-2834bd0c-00ff-c247-8130-de890b8995ed: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5c569b65-fa95-17b7-11ed-18b3ced491d3 Address:5c569b65-fa95-17b7-11ed-18b3ced491d3} {Suffrage:Voter ID:server-3b8a6284-9959-35d0-14b4-6c5076aa69a6 Address:3b8a6284-9959-35d0-14b4-6c5076aa69a6} {Suffrage:Voter ID:server-2834bd0c-00ff-c247-8130-de890b8995ed Address:2834bd0c-00ff-c247-8130-de890b8995ed} {Suffrage:Voter ID:server-3590827f-deba-b006-11f8-68842fe0cfc1 Address:3590827f-deba-b006-11f8-68842fe0cfc1} {Suffrage:Voter ID:server-a40091af-1525-1cee-4890-76db3935066c Address:a40091af-1525-1cee-4890-76db3935066c} {Suffrage:Voter ID:server-8c4b5bef-7a83-7b28-894c-5f89f2ec8d15 Address:8c4b5bef-7a83-7b28-894c-5f89f2ec8d15} {Suffrage:Voter ID:server-f9657fed-32c5-3775-39b9-58219cc1a442 Address:f9657fed-32c5-3775-39b9-58219cc1a442}]" -2025-10-27T04:03:49.051Z [INFO] server-2834bd0c-00ff-c247-8130-de890b8995ed: entering follower state: follower="Node at 2834bd0c-00ff-c247-8130-de890b8995ed [Follower]" leader-address= leader-id= -2025-10-27T04:03:49.051Z [INFO] server-3590827f-deba-b006-11f8-68842fe0cfc1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5c569b65-fa95-17b7-11ed-18b3ced491d3 Address:5c569b65-fa95-17b7-11ed-18b3ced491d3} {Suffrage:Voter ID:server-3b8a6284-9959-35d0-14b4-6c5076aa69a6 Address:3b8a6284-9959-35d0-14b4-6c5076aa69a6} {Suffrage:Voter ID:server-2834bd0c-00ff-c247-8130-de890b8995ed Address:2834bd0c-00ff-c247-8130-de890b8995ed} {Suffrage:Voter ID:server-3590827f-deba-b006-11f8-68842fe0cfc1 Address:3590827f-deba-b006-11f8-68842fe0cfc1} {Suffrage:Voter ID:server-a40091af-1525-1cee-4890-76db3935066c Address:a40091af-1525-1cee-4890-76db3935066c} {Suffrage:Voter ID:server-8c4b5bef-7a83-7b28-894c-5f89f2ec8d15 Address:8c4b5bef-7a83-7b28-894c-5f89f2ec8d15} {Suffrage:Voter ID:server-f9657fed-32c5-3775-39b9-58219cc1a442 Address:f9657fed-32c5-3775-39b9-58219cc1a442}]" -2025-10-27T04:03:49.051Z [INFO] server-3590827f-deba-b006-11f8-68842fe0cfc1: entering follower state: follower="Node at 3590827f-deba-b006-11f8-68842fe0cfc1 [Follower]" leader-address= leader-id= -2025-10-27T04:03:49.051Z [INFO] server-a40091af-1525-1cee-4890-76db3935066c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5c569b65-fa95-17b7-11ed-18b3ced491d3 Address:5c569b65-fa95-17b7-11ed-18b3ced491d3} {Suffrage:Voter ID:server-3b8a6284-9959-35d0-14b4-6c5076aa69a6 Address:3b8a6284-9959-35d0-14b4-6c5076aa69a6} {Suffrage:Voter ID:server-2834bd0c-00ff-c247-8130-de890b8995ed Address:2834bd0c-00ff-c247-8130-de890b8995ed} {Suffrage:Voter ID:server-3590827f-deba-b006-11f8-68842fe0cfc1 Address:3590827f-deba-b006-11f8-68842fe0cfc1} {Suffrage:Voter ID:server-a40091af-1525-1cee-4890-76db3935066c Address:a40091af-1525-1cee-4890-76db3935066c} {Suffrage:Voter ID:server-8c4b5bef-7a83-7b28-894c-5f89f2ec8d15 Address:8c4b5bef-7a83-7b28-894c-5f89f2ec8d15} {Suffrage:Voter ID:server-f9657fed-32c5-3775-39b9-58219cc1a442 Address:f9657fed-32c5-3775-39b9-58219cc1a442}]" -2025-10-27T04:03:49.051Z [INFO] server-a40091af-1525-1cee-4890-76db3935066c: entering follower state: follower="Node at a40091af-1525-1cee-4890-76db3935066c [Follower]" leader-address= leader-id= -2025-10-27T04:03:49.051Z [INFO] server-8c4b5bef-7a83-7b28-894c-5f89f2ec8d15: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5c569b65-fa95-17b7-11ed-18b3ced491d3 Address:5c569b65-fa95-17b7-11ed-18b3ced491d3} {Suffrage:Voter ID:server-3b8a6284-9959-35d0-14b4-6c5076aa69a6 Address:3b8a6284-9959-35d0-14b4-6c5076aa69a6} {Suffrage:Voter ID:server-2834bd0c-00ff-c247-8130-de890b8995ed Address:2834bd0c-00ff-c247-8130-de890b8995ed} {Suffrage:Voter ID:server-3590827f-deba-b006-11f8-68842fe0cfc1 Address:3590827f-deba-b006-11f8-68842fe0cfc1} {Suffrage:Voter ID:server-a40091af-1525-1cee-4890-76db3935066c Address:a40091af-1525-1cee-4890-76db3935066c} {Suffrage:Voter ID:server-8c4b5bef-7a83-7b28-894c-5f89f2ec8d15 Address:8c4b5bef-7a83-7b28-894c-5f89f2ec8d15} {Suffrage:Voter ID:server-f9657fed-32c5-3775-39b9-58219cc1a442 Address:f9657fed-32c5-3775-39b9-58219cc1a442}]" -2025-10-27T04:03:49.051Z [INFO] server-8c4b5bef-7a83-7b28-894c-5f89f2ec8d15: entering follower state: follower="Node at 8c4b5bef-7a83-7b28-894c-5f89f2ec8d15 [Follower]" leader-address= leader-id= -2025-10-27T04:03:49.051Z [INFO] server-f9657fed-32c5-3775-39b9-58219cc1a442: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5c569b65-fa95-17b7-11ed-18b3ced491d3 Address:5c569b65-fa95-17b7-11ed-18b3ced491d3} {Suffrage:Voter ID:server-3b8a6284-9959-35d0-14b4-6c5076aa69a6 Address:3b8a6284-9959-35d0-14b4-6c5076aa69a6} {Suffrage:Voter ID:server-2834bd0c-00ff-c247-8130-de890b8995ed Address:2834bd0c-00ff-c247-8130-de890b8995ed} {Suffrage:Voter ID:server-3590827f-deba-b006-11f8-68842fe0cfc1 Address:3590827f-deba-b006-11f8-68842fe0cfc1} {Suffrage:Voter ID:server-a40091af-1525-1cee-4890-76db3935066c Address:a40091af-1525-1cee-4890-76db3935066c} {Suffrage:Voter ID:server-8c4b5bef-7a83-7b28-894c-5f89f2ec8d15 Address:8c4b5bef-7a83-7b28-894c-5f89f2ec8d15} {Suffrage:Voter ID:server-f9657fed-32c5-3775-39b9-58219cc1a442 Address:f9657fed-32c5-3775-39b9-58219cc1a442}]" -2025-10-27T04:03:49.051Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:49.051Z [INFO] server-f9657fed-32c5-3775-39b9-58219cc1a442: entering follower state: follower="Node at f9657fed-32c5-3775-39b9-58219cc1a442 [Follower]" leader-address= leader-id= -2025-10-27T04:03:49.110Z [WARN] server-3590827f-deba-b006-11f8-68842fe0cfc1: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:49.110Z [INFO] server-3590827f-deba-b006-11f8-68842fe0cfc1: entering candidate state: node="Node at 3590827f-deba-b006-11f8-68842fe0cfc1 [Candidate]" term=2 -2025-10-27T04:03:49.110Z [INFO] server-3590827f-deba-b006-11f8-68842fe0cfc1: election won: term=2 tally=4 -2025-10-27T04:03:49.110Z [INFO] server-3590827f-deba-b006-11f8-68842fe0cfc1: entering leader state: leader="Node at 3590827f-deba-b006-11f8-68842fe0cfc1 [Leader]" -2025-10-27T04:03:49.110Z [INFO] server-3590827f-deba-b006-11f8-68842fe0cfc1: added peer, starting replication: peer=server-5c569b65-fa95-17b7-11ed-18b3ced491d3 -2025-10-27T04:03:49.110Z [INFO] server-3590827f-deba-b006-11f8-68842fe0cfc1: added peer, starting replication: peer=server-3b8a6284-9959-35d0-14b4-6c5076aa69a6 -2025-10-27T04:03:49.110Z [INFO] server-3590827f-deba-b006-11f8-68842fe0cfc1: added peer, starting replication: peer=server-2834bd0c-00ff-c247-8130-de890b8995ed -2025-10-27T04:03:49.110Z [INFO] server-3590827f-deba-b006-11f8-68842fe0cfc1: added peer, starting replication: peer=server-a40091af-1525-1cee-4890-76db3935066c -2025-10-27T04:03:49.110Z [INFO] server-3590827f-deba-b006-11f8-68842fe0cfc1: added peer, starting replication: peer=server-8c4b5bef-7a83-7b28-894c-5f89f2ec8d15 -2025-10-27T04:03:49.110Z [INFO] server-3590827f-deba-b006-11f8-68842fe0cfc1: added peer, starting replication: peer=server-f9657fed-32c5-3775-39b9-58219cc1a442 -2025-10-27T04:03:49.110Z [INFO] server-3590827f-deba-b006-11f8-68842fe0cfc1: pipelining replication: peer="{Voter server-f9657fed-32c5-3775-39b9-58219cc1a442 f9657fed-32c5-3775-39b9-58219cc1a442}" -2025-10-27T04:03:49.110Z [WARN] server-f9657fed-32c5-3775-39b9-58219cc1a442: rejecting vote request since our last term is greater: candidate=3590827f-deba-b006-11f8-68842fe0cfc1 last-term=2 last-candidate-term=1 -2025-10-27T04:03:49.110Z [INFO] server-3590827f-deba-b006-11f8-68842fe0cfc1: pipelining replication: peer="{Voter server-5c569b65-fa95-17b7-11ed-18b3ced491d3 5c569b65-fa95-17b7-11ed-18b3ced491d3}" -2025-10-27T04:03:49.110Z [INFO] server-3590827f-deba-b006-11f8-68842fe0cfc1: pipelining replication: peer="{Voter server-3b8a6284-9959-35d0-14b4-6c5076aa69a6 3b8a6284-9959-35d0-14b4-6c5076aa69a6}" -2025-10-27T04:03:49.111Z [INFO] server-3590827f-deba-b006-11f8-68842fe0cfc1: pipelining replication: peer="{Voter server-2834bd0c-00ff-c247-8130-de890b8995ed 2834bd0c-00ff-c247-8130-de890b8995ed}" -2025-10-27T04:03:49.111Z [INFO] server-3590827f-deba-b006-11f8-68842fe0cfc1: pipelining replication: peer="{Voter server-a40091af-1525-1cee-4890-76db3935066c a40091af-1525-1cee-4890-76db3935066c}" -2025-10-27T04:03:49.111Z [INFO] server-3590827f-deba-b006-11f8-68842fe0cfc1: pipelining replication: peer="{Voter server-8c4b5bef-7a83-7b28-894c-5f89f2ec8d15 8c4b5bef-7a83-7b28-894c-5f89f2ec8d15}" -2025-10-27T04:03:49.216Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:49.110920446 +0000 UTC m=+104.105792884 (1 nodes), 58.951163ms from start of poll, 59.717648ms from cluster start. Timeout at 2025-10-27 04:03:49.215921926 +0000 UTC m=+104.210794364, 105.00148ms after stability -2025-10-27T04:03:49.216Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:03:49.322Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:03:49.216766769 +0000 UTC m=+104.211639207 (6 nodes), 2µs from start of poll, 165.563971ms from cluster start. Timeout at 2025-10-27 04:03:49.32179733 +0000 UTC m=+104.316669768, 105.030561ms after stability -2025-10-27T04:03:49.322Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:49.428Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:49.322510931 +0000 UTC m=+104.317383369 (1 nodes), 2.16µs from start of poll, 271.308133ms from cluster start. Timeout at 2025-10-27 04:03:49.427547892 +0000 UTC m=+104.422420370, 105.037001ms after stability -2025-10-27T04:03:49.428Z [INFO] server-5c569b65-fa95-17b7-11ed-18b3ced491d3: entering candidate state: node="Node at 5c569b65-fa95-17b7-11ed-18b3ced491d3 [Candidate]" term=3 -2025-10-27T04:03:49.428Z [INFO] server-3590827f-deba-b006-11f8-68842fe0cfc1: entering follower state: follower="Node at 3590827f-deba-b006-11f8-68842fe0cfc1 [Follower]" leader-address= leader-id= -2025-10-27T04:03:49.428Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:49.428Z [INFO] server-3590827f-deba-b006-11f8-68842fe0cfc1: aborting pipeline replication: peer="{Voter server-f9657fed-32c5-3775-39b9-58219cc1a442 f9657fed-32c5-3775-39b9-58219cc1a442}" -2025-10-27T04:03:49.428Z [INFO] server-5c569b65-fa95-17b7-11ed-18b3ced491d3: election won: term=3 tally=4 -2025-10-27T04:03:49.428Z [INFO] server-5c569b65-fa95-17b7-11ed-18b3ced491d3: entering leader state: leader="Node at 5c569b65-fa95-17b7-11ed-18b3ced491d3 [Leader]" -2025-10-27T04:03:49.428Z [INFO] server-5c569b65-fa95-17b7-11ed-18b3ced491d3: added peer, starting replication: peer=server-3b8a6284-9959-35d0-14b4-6c5076aa69a6 -2025-10-27T04:03:49.428Z [INFO] server-5c569b65-fa95-17b7-11ed-18b3ced491d3: added peer, starting replication: peer=server-2834bd0c-00ff-c247-8130-de890b8995ed -2025-10-27T04:03:49.428Z [INFO] server-3590827f-deba-b006-11f8-68842fe0cfc1: aborting pipeline replication: peer="{Voter server-3b8a6284-9959-35d0-14b4-6c5076aa69a6 3b8a6284-9959-35d0-14b4-6c5076aa69a6}" -2025-10-27T04:03:49.428Z [INFO] server-5c569b65-fa95-17b7-11ed-18b3ced491d3: added peer, starting replication: peer=server-3590827f-deba-b006-11f8-68842fe0cfc1 -2025-10-27T04:03:49.428Z [INFO] server-5c569b65-fa95-17b7-11ed-18b3ced491d3: added peer, starting replication: peer=server-a40091af-1525-1cee-4890-76db3935066c -2025-10-27T04:03:49.428Z [INFO] server-5c569b65-fa95-17b7-11ed-18b3ced491d3: added peer, starting replication: peer=server-8c4b5bef-7a83-7b28-894c-5f89f2ec8d15 -2025-10-27T04:03:49.428Z [INFO] server-5c569b65-fa95-17b7-11ed-18b3ced491d3: added peer, starting replication: peer=server-f9657fed-32c5-3775-39b9-58219cc1a442 -2025-10-27T04:03:49.428Z [INFO] server-3590827f-deba-b006-11f8-68842fe0cfc1: aborting pipeline replication: peer="{Voter server-5c569b65-fa95-17b7-11ed-18b3ced491d3 5c569b65-fa95-17b7-11ed-18b3ced491d3}" -2025-10-27T04:03:49.428Z [INFO] server-3590827f-deba-b006-11f8-68842fe0cfc1: aborting pipeline replication: peer="{Voter server-2834bd0c-00ff-c247-8130-de890b8995ed 2834bd0c-00ff-c247-8130-de890b8995ed}" -2025-10-27T04:03:49.428Z [INFO] server-3590827f-deba-b006-11f8-68842fe0cfc1: aborting pipeline replication: peer="{Voter server-a40091af-1525-1cee-4890-76db3935066c a40091af-1525-1cee-4890-76db3935066c}" -2025-10-27T04:03:49.428Z [INFO] server-3590827f-deba-b006-11f8-68842fe0cfc1: aborting pipeline replication: peer="{Voter server-8c4b5bef-7a83-7b28-894c-5f89f2ec8d15 8c4b5bef-7a83-7b28-894c-5f89f2ec8d15}" -2025-10-27T04:03:49.428Z [INFO] server-5c569b65-fa95-17b7-11ed-18b3ced491d3: pipelining replication: peer="{Voter server-f9657fed-32c5-3775-39b9-58219cc1a442 f9657fed-32c5-3775-39b9-58219cc1a442}" -2025-10-27T04:03:49.428Z [INFO] server-5c569b65-fa95-17b7-11ed-18b3ced491d3: pipelining replication: peer="{Voter server-3b8a6284-9959-35d0-14b4-6c5076aa69a6 3b8a6284-9959-35d0-14b4-6c5076aa69a6}" -2025-10-27T04:03:49.428Z [INFO] server-5c569b65-fa95-17b7-11ed-18b3ced491d3: pipelining replication: peer="{Voter server-2834bd0c-00ff-c247-8130-de890b8995ed 2834bd0c-00ff-c247-8130-de890b8995ed}" -2025-10-27T04:03:49.428Z [INFO] server-5c569b65-fa95-17b7-11ed-18b3ced491d3: pipelining replication: peer="{Voter server-8c4b5bef-7a83-7b28-894c-5f89f2ec8d15 8c4b5bef-7a83-7b28-894c-5f89f2ec8d15}" -2025-10-27T04:03:49.429Z [INFO] server-5c569b65-fa95-17b7-11ed-18b3ced491d3: pipelining replication: peer="{Voter server-3590827f-deba-b006-11f8-68842fe0cfc1 3590827f-deba-b006-11f8-68842fe0cfc1}" -2025-10-27T04:03:49.429Z [INFO] server-5c569b65-fa95-17b7-11ed-18b3ced491d3: pipelining replication: peer="{Voter server-a40091af-1525-1cee-4890-76db3935066c a40091af-1525-1cee-4890-76db3935066c}" -2025-10-27T04:03:49.534Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:49.428858618 +0000 UTC m=+104.423731096 (1 nodes), 298.602µs from start of poll, 377.65586ms from cluster start. Timeout at 2025-10-27 04:03:49.533876059 +0000 UTC m=+104.528748537, 105.017441ms after stability -2025-10-27T04:03:49.534Z [INFO] server-5c569b65-fa95-17b7-11ed-18b3ced491d3: aborting pipeline replication: peer="{Voter server-2834bd0c-00ff-c247-8130-de890b8995ed 2834bd0c-00ff-c247-8130-de890b8995ed}" -2025-10-27T04:03:49.534Z [INFO] server-5c569b65-fa95-17b7-11ed-18b3ced491d3: aborting pipeline replication: peer="{Voter server-3590827f-deba-b006-11f8-68842fe0cfc1 3590827f-deba-b006-11f8-68842fe0cfc1}" -2025-10-27T04:03:49.534Z [INFO] server-5c569b65-fa95-17b7-11ed-18b3ced491d3: aborting pipeline replication: peer="{Voter server-8c4b5bef-7a83-7b28-894c-5f89f2ec8d15 8c4b5bef-7a83-7b28-894c-5f89f2ec8d15}" -2025-10-27T04:03:49.534Z [INFO] server-5c569b65-fa95-17b7-11ed-18b3ced491d3: aborting pipeline replication: peer="{Voter server-f9657fed-32c5-3775-39b9-58219cc1a442 f9657fed-32c5-3775-39b9-58219cc1a442}" -2025-10-27T04:03:49.534Z [INFO] server-5c569b65-fa95-17b7-11ed-18b3ced491d3: aborting pipeline replication: peer="{Voter server-3b8a6284-9959-35d0-14b4-6c5076aa69a6 3b8a6284-9959-35d0-14b4-6c5076aa69a6}" -2025-10-27T04:03:49.534Z [INFO] server-5c569b65-fa95-17b7-11ed-18b3ced491d3: aborting pipeline replication: peer="{Voter server-a40091af-1525-1cee-4890-76db3935066c a40091af-1525-1cee-4890-76db3935066c}" -2025-10-27T04:03:50.034Z [ERROR] server-5c569b65-fa95-17b7-11ed-18b3ced491d3: failed to heartbeat to: peer=8c4b5bef-7a83-7b28-894c-5f89f2ec8d15 backoff time=10ms error="command timed out" ---- PASS: TestRaft_LeadershipTransferWithSevenNodes (0.98s) +2026-11-29T10:29:51.898Z [INFO] server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f Address:cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f} {Suffrage:Voter ID:server-ad17a85a-f802-e502-e696-2a142f5f5b93 Address:ad17a85a-f802-e502-e696-2a142f5f5b93} {Suffrage:Voter ID:server-b136858c-d554-c972-acc7-20fdad63e72d Address:b136858c-d554-c972-acc7-20fdad63e72d} {Suffrage:Voter ID:server-2d0a4ba3-9213-e972-e735-582d16dbafde Address:2d0a4ba3-9213-e972-e735-582d16dbafde} {Suffrage:Voter ID:server-e17c10b6-3636-b8c2-8924-5a768210bac8 Address:e17c10b6-3636-b8c2-8924-5a768210bac8} {Suffrage:Voter ID:server-ef89dea2-ba83-da71-55bf-738f4975221c Address:ef89dea2-ba83-da71-55bf-738f4975221c} {Suffrage:Voter ID:server-b8ebe6be-694f-f034-0ae4-5c64f3c9247a Address:b8ebe6be-694f-f034-0ae4-5c64f3c9247a}]" +2026-11-29T10:29:51.898Z [INFO] server-ad17a85a-f802-e502-e696-2a142f5f5b93: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f Address:cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f} {Suffrage:Voter ID:server-ad17a85a-f802-e502-e696-2a142f5f5b93 Address:ad17a85a-f802-e502-e696-2a142f5f5b93} {Suffrage:Voter ID:server-b136858c-d554-c972-acc7-20fdad63e72d Address:b136858c-d554-c972-acc7-20fdad63e72d} {Suffrage:Voter ID:server-2d0a4ba3-9213-e972-e735-582d16dbafde Address:2d0a4ba3-9213-e972-e735-582d16dbafde} {Suffrage:Voter ID:server-e17c10b6-3636-b8c2-8924-5a768210bac8 Address:e17c10b6-3636-b8c2-8924-5a768210bac8} {Suffrage:Voter ID:server-ef89dea2-ba83-da71-55bf-738f4975221c Address:ef89dea2-ba83-da71-55bf-738f4975221c} {Suffrage:Voter ID:server-b8ebe6be-694f-f034-0ae4-5c64f3c9247a Address:b8ebe6be-694f-f034-0ae4-5c64f3c9247a}]" +2026-11-29T10:29:51.898Z [INFO] server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f: entering follower state: follower="Node at cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f [Follower]" leader-address= leader-id= +2026-11-29T10:29:51.898Z [INFO] server-b136858c-d554-c972-acc7-20fdad63e72d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f Address:cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f} {Suffrage:Voter ID:server-ad17a85a-f802-e502-e696-2a142f5f5b93 Address:ad17a85a-f802-e502-e696-2a142f5f5b93} {Suffrage:Voter ID:server-b136858c-d554-c972-acc7-20fdad63e72d Address:b136858c-d554-c972-acc7-20fdad63e72d} {Suffrage:Voter ID:server-2d0a4ba3-9213-e972-e735-582d16dbafde Address:2d0a4ba3-9213-e972-e735-582d16dbafde} {Suffrage:Voter ID:server-e17c10b6-3636-b8c2-8924-5a768210bac8 Address:e17c10b6-3636-b8c2-8924-5a768210bac8} {Suffrage:Voter ID:server-ef89dea2-ba83-da71-55bf-738f4975221c Address:ef89dea2-ba83-da71-55bf-738f4975221c} {Suffrage:Voter ID:server-b8ebe6be-694f-f034-0ae4-5c64f3c9247a Address:b8ebe6be-694f-f034-0ae4-5c64f3c9247a}]" +2026-11-29T10:29:51.898Z [INFO] server-2d0a4ba3-9213-e972-e735-582d16dbafde: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f Address:cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f} {Suffrage:Voter ID:server-ad17a85a-f802-e502-e696-2a142f5f5b93 Address:ad17a85a-f802-e502-e696-2a142f5f5b93} {Suffrage:Voter ID:server-b136858c-d554-c972-acc7-20fdad63e72d Address:b136858c-d554-c972-acc7-20fdad63e72d} {Suffrage:Voter ID:server-2d0a4ba3-9213-e972-e735-582d16dbafde Address:2d0a4ba3-9213-e972-e735-582d16dbafde} {Suffrage:Voter ID:server-e17c10b6-3636-b8c2-8924-5a768210bac8 Address:e17c10b6-3636-b8c2-8924-5a768210bac8} {Suffrage:Voter ID:server-ef89dea2-ba83-da71-55bf-738f4975221c Address:ef89dea2-ba83-da71-55bf-738f4975221c} {Suffrage:Voter ID:server-b8ebe6be-694f-f034-0ae4-5c64f3c9247a Address:b8ebe6be-694f-f034-0ae4-5c64f3c9247a}]" +2026-11-29T10:29:51.898Z [INFO] server-ad17a85a-f802-e502-e696-2a142f5f5b93: entering follower state: follower="Node at ad17a85a-f802-e502-e696-2a142f5f5b93 [Follower]" leader-address= leader-id= +2026-11-29T10:29:51.898Z [INFO] server-e17c10b6-3636-b8c2-8924-5a768210bac8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f Address:cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f} {Suffrage:Voter ID:server-ad17a85a-f802-e502-e696-2a142f5f5b93 Address:ad17a85a-f802-e502-e696-2a142f5f5b93} {Suffrage:Voter ID:server-b136858c-d554-c972-acc7-20fdad63e72d Address:b136858c-d554-c972-acc7-20fdad63e72d} {Suffrage:Voter ID:server-2d0a4ba3-9213-e972-e735-582d16dbafde Address:2d0a4ba3-9213-e972-e735-582d16dbafde} {Suffrage:Voter ID:server-e17c10b6-3636-b8c2-8924-5a768210bac8 Address:e17c10b6-3636-b8c2-8924-5a768210bac8} {Suffrage:Voter ID:server-ef89dea2-ba83-da71-55bf-738f4975221c Address:ef89dea2-ba83-da71-55bf-738f4975221c} {Suffrage:Voter ID:server-b8ebe6be-694f-f034-0ae4-5c64f3c9247a Address:b8ebe6be-694f-f034-0ae4-5c64f3c9247a}]" +2026-11-29T10:29:51.898Z [INFO] server-b136858c-d554-c972-acc7-20fdad63e72d: entering follower state: follower="Node at b136858c-d554-c972-acc7-20fdad63e72d [Follower]" leader-address= leader-id= +2026-11-29T10:29:51.898Z [INFO] server-2d0a4ba3-9213-e972-e735-582d16dbafde: entering follower state: follower="Node at 2d0a4ba3-9213-e972-e735-582d16dbafde [Follower]" leader-address= leader-id= +2026-11-29T10:29:51.898Z [INFO] server-ef89dea2-ba83-da71-55bf-738f4975221c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f Address:cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f} {Suffrage:Voter ID:server-ad17a85a-f802-e502-e696-2a142f5f5b93 Address:ad17a85a-f802-e502-e696-2a142f5f5b93} {Suffrage:Voter ID:server-b136858c-d554-c972-acc7-20fdad63e72d Address:b136858c-d554-c972-acc7-20fdad63e72d} {Suffrage:Voter ID:server-2d0a4ba3-9213-e972-e735-582d16dbafde Address:2d0a4ba3-9213-e972-e735-582d16dbafde} {Suffrage:Voter ID:server-e17c10b6-3636-b8c2-8924-5a768210bac8 Address:e17c10b6-3636-b8c2-8924-5a768210bac8} {Suffrage:Voter ID:server-ef89dea2-ba83-da71-55bf-738f4975221c Address:ef89dea2-ba83-da71-55bf-738f4975221c} {Suffrage:Voter ID:server-b8ebe6be-694f-f034-0ae4-5c64f3c9247a Address:b8ebe6be-694f-f034-0ae4-5c64f3c9247a}]" +2026-11-29T10:29:51.898Z [INFO] server-e17c10b6-3636-b8c2-8924-5a768210bac8: entering follower state: follower="Node at e17c10b6-3636-b8c2-8924-5a768210bac8 [Follower]" leader-address= leader-id= +2026-11-29T10:29:51.898Z [INFO] server-ef89dea2-ba83-da71-55bf-738f4975221c: entering follower state: follower="Node at ef89dea2-ba83-da71-55bf-738f4975221c [Follower]" leader-address= leader-id= +2026-11-29T10:29:51.898Z [INFO] server-b8ebe6be-694f-f034-0ae4-5c64f3c9247a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f Address:cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f} {Suffrage:Voter ID:server-ad17a85a-f802-e502-e696-2a142f5f5b93 Address:ad17a85a-f802-e502-e696-2a142f5f5b93} {Suffrage:Voter ID:server-b136858c-d554-c972-acc7-20fdad63e72d Address:b136858c-d554-c972-acc7-20fdad63e72d} {Suffrage:Voter ID:server-2d0a4ba3-9213-e972-e735-582d16dbafde Address:2d0a4ba3-9213-e972-e735-582d16dbafde} {Suffrage:Voter ID:server-e17c10b6-3636-b8c2-8924-5a768210bac8 Address:e17c10b6-3636-b8c2-8924-5a768210bac8} {Suffrage:Voter ID:server-ef89dea2-ba83-da71-55bf-738f4975221c Address:ef89dea2-ba83-da71-55bf-738f4975221c} {Suffrage:Voter ID:server-b8ebe6be-694f-f034-0ae4-5c64f3c9247a Address:b8ebe6be-694f-f034-0ae4-5c64f3c9247a}]" +2026-11-29T10:29:51.898Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:51.898Z [INFO] server-b8ebe6be-694f-f034-0ae4-5c64f3c9247a: entering follower state: follower="Node at b8ebe6be-694f-f034-0ae4-5c64f3c9247a [Follower]" leader-address= leader-id= +2026-11-29T10:29:51.950Z [WARN] server-ef89dea2-ba83-da71-55bf-738f4975221c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:51.950Z [INFO] server-ef89dea2-ba83-da71-55bf-738f4975221c: entering candidate state: node="Node at ef89dea2-ba83-da71-55bf-738f4975221c [Candidate]" term=2 +2026-11-29T10:29:51.950Z [INFO] server-ef89dea2-ba83-da71-55bf-738f4975221c: election won: term=2 tally=4 +2026-11-29T10:29:51.950Z [INFO] server-ef89dea2-ba83-da71-55bf-738f4975221c: entering leader state: leader="Node at ef89dea2-ba83-da71-55bf-738f4975221c [Leader]" +2026-11-29T10:29:51.950Z [INFO] server-ef89dea2-ba83-da71-55bf-738f4975221c: added peer, starting replication: peer=server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f +2026-11-29T10:29:51.950Z [INFO] server-ef89dea2-ba83-da71-55bf-738f4975221c: added peer, starting replication: peer=server-ad17a85a-f802-e502-e696-2a142f5f5b93 +2026-11-29T10:29:51.950Z [INFO] server-ef89dea2-ba83-da71-55bf-738f4975221c: added peer, starting replication: peer=server-b136858c-d554-c972-acc7-20fdad63e72d +2026-11-29T10:29:51.950Z [INFO] server-ef89dea2-ba83-da71-55bf-738f4975221c: added peer, starting replication: peer=server-2d0a4ba3-9213-e972-e735-582d16dbafde +2026-11-29T10:29:51.950Z [INFO] server-ef89dea2-ba83-da71-55bf-738f4975221c: added peer, starting replication: peer=server-e17c10b6-3636-b8c2-8924-5a768210bac8 +2026-11-29T10:29:51.950Z [INFO] server-ef89dea2-ba83-da71-55bf-738f4975221c: added peer, starting replication: peer=server-b8ebe6be-694f-f034-0ae4-5c64f3c9247a +2026-11-29T10:29:51.950Z [INFO] server-ef89dea2-ba83-da71-55bf-738f4975221c: pipelining replication: peer="{Voter server-b8ebe6be-694f-f034-0ae4-5c64f3c9247a b8ebe6be-694f-f034-0ae4-5c64f3c9247a}" +2026-11-29T10:29:51.950Z [INFO] server-ef89dea2-ba83-da71-55bf-738f4975221c: pipelining replication: peer="{Voter server-ad17a85a-f802-e502-e696-2a142f5f5b93 ad17a85a-f802-e502-e696-2a142f5f5b93}" +2026-11-29T10:29:51.950Z [INFO] server-ef89dea2-ba83-da71-55bf-738f4975221c: pipelining replication: peer="{Voter server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f}" +2026-11-29T10:29:51.950Z [INFO] server-ef89dea2-ba83-da71-55bf-738f4975221c: pipelining replication: peer="{Voter server-e17c10b6-3636-b8c2-8924-5a768210bac8 e17c10b6-3636-b8c2-8924-5a768210bac8}" +2026-11-29T10:29:51.950Z [INFO] server-ef89dea2-ba83-da71-55bf-738f4975221c: pipelining replication: peer="{Voter server-b136858c-d554-c972-acc7-20fdad63e72d b136858c-d554-c972-acc7-20fdad63e72d}" +2026-11-29T10:29:51.950Z [INFO] server-ef89dea2-ba83-da71-55bf-738f4975221c: pipelining replication: peer="{Voter server-2d0a4ba3-9213-e972-e735-582d16dbafde 2d0a4ba3-9213-e972-e735-582d16dbafde}" +2026-11-29T10:29:52.056Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:51.950722539 +0000 UTC m=+102.685776641 (1 nodes), 51.97008ms from start of poll, 52.38336ms from cluster start. Timeout at 2026-11-29 10:29:52.055727379 +0000 UTC m=+102.790781521, 105.00488ms after stability +2026-11-29T10:29:52.056Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:52.162Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:52.056508059 +0000 UTC m=+102.791562201 (6 nodes), 2.92µs from start of poll, 158.16892ms from cluster start. Timeout at 2026-11-29 10:29:52.161533059 +0000 UTC m=+102.896587201, 105.025ms after stability +2026-11-29T10:29:52.162Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:52.268Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:52.162622019 +0000 UTC m=+102.897676161 (1 nodes), 1.08µs from start of poll, 264.28288ms from cluster start. Timeout at 2026-11-29 10:29:52.267677539 +0000 UTC m=+103.002731681, 105.05552ms after stability +2026-11-29T10:29:52.268Z [INFO] server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f: entering candidate state: node="Node at cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f [Candidate]" term=3 +2026-11-29T10:29:52.268Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:52.268Z [INFO] server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f: election won: term=3 tally=4 +2026-11-29T10:29:52.268Z [INFO] server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f: entering leader state: leader="Node at cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f [Leader]" +2026-11-29T10:29:52.268Z [INFO] server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f: added peer, starting replication: peer=server-ad17a85a-f802-e502-e696-2a142f5f5b93 +2026-11-29T10:29:52.268Z [INFO] server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f: added peer, starting replication: peer=server-b136858c-d554-c972-acc7-20fdad63e72d +2026-11-29T10:29:52.268Z [INFO] server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f: added peer, starting replication: peer=server-2d0a4ba3-9213-e972-e735-582d16dbafde +2026-11-29T10:29:52.268Z [INFO] server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f: added peer, starting replication: peer=server-e17c10b6-3636-b8c2-8924-5a768210bac8 +2026-11-29T10:29:52.268Z [INFO] server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f: added peer, starting replication: peer=server-ef89dea2-ba83-da71-55bf-738f4975221c +2026-11-29T10:29:52.268Z [INFO] server-ef89dea2-ba83-da71-55bf-738f4975221c: entering follower state: follower="Node at ef89dea2-ba83-da71-55bf-738f4975221c [Follower]" leader-address= leader-id= +2026-11-29T10:29:52.268Z [INFO] server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f: added peer, starting replication: peer=server-b8ebe6be-694f-f034-0ae4-5c64f3c9247a +2026-11-29T10:29:52.268Z [INFO] server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f: pipelining replication: peer="{Voter server-ef89dea2-ba83-da71-55bf-738f4975221c ef89dea2-ba83-da71-55bf-738f4975221c}" +2026-11-29T10:29:52.268Z [INFO] server-ef89dea2-ba83-da71-55bf-738f4975221c: aborting pipeline replication: peer="{Voter server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f}" +2026-11-29T10:29:52.268Z [INFO] server-ef89dea2-ba83-da71-55bf-738f4975221c: aborting pipeline replication: peer="{Voter server-ad17a85a-f802-e502-e696-2a142f5f5b93 ad17a85a-f802-e502-e696-2a142f5f5b93}" +2026-11-29T10:29:52.268Z [INFO] server-ef89dea2-ba83-da71-55bf-738f4975221c: aborting pipeline replication: peer="{Voter server-b136858c-d554-c972-acc7-20fdad63e72d b136858c-d554-c972-acc7-20fdad63e72d}" +2026-11-29T10:29:52.268Z [INFO] server-ef89dea2-ba83-da71-55bf-738f4975221c: aborting pipeline replication: peer="{Voter server-2d0a4ba3-9213-e972-e735-582d16dbafde 2d0a4ba3-9213-e972-e735-582d16dbafde}" +2026-11-29T10:29:52.268Z [INFO] server-ef89dea2-ba83-da71-55bf-738f4975221c: aborting pipeline replication: peer="{Voter server-e17c10b6-3636-b8c2-8924-5a768210bac8 e17c10b6-3636-b8c2-8924-5a768210bac8}" +2026-11-29T10:29:52.268Z [INFO] server-ef89dea2-ba83-da71-55bf-738f4975221c: aborting pipeline replication: peer="{Voter server-b8ebe6be-694f-f034-0ae4-5c64f3c9247a b8ebe6be-694f-f034-0ae4-5c64f3c9247a}" +2026-11-29T10:29:52.268Z [INFO] server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f: pipelining replication: peer="{Voter server-ad17a85a-f802-e502-e696-2a142f5f5b93 ad17a85a-f802-e502-e696-2a142f5f5b93}" +2026-11-29T10:29:52.268Z [INFO] server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f: pipelining replication: peer="{Voter server-b136858c-d554-c972-acc7-20fdad63e72d b136858c-d554-c972-acc7-20fdad63e72d}" +2026-11-29T10:29:52.268Z [INFO] server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f: pipelining replication: peer="{Voter server-b8ebe6be-694f-f034-0ae4-5c64f3c9247a b8ebe6be-694f-f034-0ae4-5c64f3c9247a}" +2026-11-29T10:29:52.268Z [INFO] server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f: pipelining replication: peer="{Voter server-2d0a4ba3-9213-e972-e735-582d16dbafde 2d0a4ba3-9213-e972-e735-582d16dbafde}" +2026-11-29T10:29:52.268Z [INFO] server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f: pipelining replication: peer="{Voter server-e17c10b6-3636-b8c2-8924-5a768210bac8 e17c10b6-3636-b8c2-8924-5a768210bac8}" +2026-11-29T10:29:52.374Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:52.268893059 +0000 UTC m=+103.003947161 (1 nodes), 295.32µs from start of poll, 370.55388ms from cluster start. Timeout at 2026-11-29 10:29:52.373894019 +0000 UTC m=+103.108948121, 105.00096ms after stability +2026-11-29T10:29:52.375Z [INFO] server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f: aborting pipeline replication: peer="{Voter server-e17c10b6-3636-b8c2-8924-5a768210bac8 e17c10b6-3636-b8c2-8924-5a768210bac8}" +2026-11-29T10:29:52.375Z [INFO] server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f: aborting pipeline replication: peer="{Voter server-2d0a4ba3-9213-e972-e735-582d16dbafde 2d0a4ba3-9213-e972-e735-582d16dbafde}" +2026-11-29T10:29:52.375Z [INFO] server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f: aborting pipeline replication: peer="{Voter server-b136858c-d554-c972-acc7-20fdad63e72d b136858c-d554-c972-acc7-20fdad63e72d}" +2026-11-29T10:29:52.375Z [INFO] server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f: aborting pipeline replication: peer="{Voter server-ad17a85a-f802-e502-e696-2a142f5f5b93 ad17a85a-f802-e502-e696-2a142f5f5b93}" +2026-11-29T10:29:52.375Z [INFO] server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f: aborting pipeline replication: peer="{Voter server-ef89dea2-ba83-da71-55bf-738f4975221c ef89dea2-ba83-da71-55bf-738f4975221c}" +2026-11-29T10:29:52.375Z [INFO] server-cc2f6ea8-a75c-a79f-7c66-42d8fd3eda2f: aborting pipeline replication: peer="{Voter server-b8ebe6be-694f-f034-0ae4-5c64f3c9247a b8ebe6be-694f-f034-0ae4-5c64f3c9247a}" +--- PASS: TestRaft_LeadershipTransferWithSevenNodes (0.48s) === RUN TestRaft_LeadershipTransferToInvalidID -2025-10-27T04:03:50.040Z [INFO] server-9b8d9d19-6f2b-9e8f-f1cb-543941b5ff6d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9b8d9d19-6f2b-9e8f-f1cb-543941b5ff6d Address:9b8d9d19-6f2b-9e8f-f1cb-543941b5ff6d} {Suffrage:Voter ID:server-f7058a0b-b929-5cbf-b52f-b54829392656 Address:f7058a0b-b929-5cbf-b52f-b54829392656} {Suffrage:Voter ID:server-8f831599-b708-59be-d12a-38b9223a5e5e Address:8f831599-b708-59be-d12a-38b9223a5e5e}]" -2025-10-27T04:03:50.040Z [INFO] server-f7058a0b-b929-5cbf-b52f-b54829392656: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9b8d9d19-6f2b-9e8f-f1cb-543941b5ff6d Address:9b8d9d19-6f2b-9e8f-f1cb-543941b5ff6d} {Suffrage:Voter ID:server-f7058a0b-b929-5cbf-b52f-b54829392656 Address:f7058a0b-b929-5cbf-b52f-b54829392656} {Suffrage:Voter ID:server-8f831599-b708-59be-d12a-38b9223a5e5e Address:8f831599-b708-59be-d12a-38b9223a5e5e}]" -2025-10-27T04:03:50.040Z [INFO] server-8f831599-b708-59be-d12a-38b9223a5e5e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9b8d9d19-6f2b-9e8f-f1cb-543941b5ff6d Address:9b8d9d19-6f2b-9e8f-f1cb-543941b5ff6d} {Suffrage:Voter ID:server-f7058a0b-b929-5cbf-b52f-b54829392656 Address:f7058a0b-b929-5cbf-b52f-b54829392656} {Suffrage:Voter ID:server-8f831599-b708-59be-d12a-38b9223a5e5e Address:8f831599-b708-59be-d12a-38b9223a5e5e}]" -2025-10-27T04:03:50.040Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:50.041Z [INFO] server-f7058a0b-b929-5cbf-b52f-b54829392656: entering follower state: follower="Node at f7058a0b-b929-5cbf-b52f-b54829392656 [Follower]" leader-address= leader-id= -2025-10-27T04:03:50.041Z [INFO] server-9b8d9d19-6f2b-9e8f-f1cb-543941b5ff6d: entering follower state: follower="Node at 9b8d9d19-6f2b-9e8f-f1cb-543941b5ff6d [Follower]" leader-address= leader-id= -2025-10-27T04:03:50.041Z [INFO] server-8f831599-b708-59be-d12a-38b9223a5e5e: entering follower state: follower="Node at 8f831599-b708-59be-d12a-38b9223a5e5e [Follower]" leader-address= leader-id= -2025-10-27T04:03:50.123Z [WARN] server-9b8d9d19-6f2b-9e8f-f1cb-543941b5ff6d: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:50.123Z [INFO] server-9b8d9d19-6f2b-9e8f-f1cb-543941b5ff6d: entering candidate state: node="Node at 9b8d9d19-6f2b-9e8f-f1cb-543941b5ff6d [Candidate]" term=2 -2025-10-27T04:03:50.123Z [INFO] server-9b8d9d19-6f2b-9e8f-f1cb-543941b5ff6d: election won: term=2 tally=2 -2025-10-27T04:03:50.123Z [INFO] server-9b8d9d19-6f2b-9e8f-f1cb-543941b5ff6d: entering leader state: leader="Node at 9b8d9d19-6f2b-9e8f-f1cb-543941b5ff6d [Leader]" -2025-10-27T04:03:50.123Z [INFO] server-9b8d9d19-6f2b-9e8f-f1cb-543941b5ff6d: added peer, starting replication: peer=server-f7058a0b-b929-5cbf-b52f-b54829392656 -2025-10-27T04:03:50.123Z [INFO] server-9b8d9d19-6f2b-9e8f-f1cb-543941b5ff6d: added peer, starting replication: peer=server-8f831599-b708-59be-d12a-38b9223a5e5e -2025-10-27T04:03:50.123Z [INFO] server-9b8d9d19-6f2b-9e8f-f1cb-543941b5ff6d: pipelining replication: peer="{Voter server-8f831599-b708-59be-d12a-38b9223a5e5e 8f831599-b708-59be-d12a-38b9223a5e5e}" -2025-10-27T04:03:50.123Z [INFO] server-9b8d9d19-6f2b-9e8f-f1cb-543941b5ff6d: pipelining replication: peer="{Voter server-f7058a0b-b929-5cbf-b52f-b54829392656 f7058a0b-b929-5cbf-b52f-b54829392656}" -2025-10-27T04:03:50.229Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:50.123688084 +0000 UTC m=+105.118560562 (1 nodes), 82.902686ms from start of poll, 83.184568ms from cluster start. Timeout at 2025-10-27 04:03:50.228692004 +0000 UTC m=+105.223564442, 105.00388ms after stability -2025-10-27T04:03:50.229Z [INFO] server-9b8d9d19-6f2b-9e8f-f1cb-543941b5ff6d: aborting pipeline replication: peer="{Voter server-8f831599-b708-59be-d12a-38b9223a5e5e 8f831599-b708-59be-d12a-38b9223a5e5e}" -2025-10-27T04:03:50.229Z [INFO] server-9b8d9d19-6f2b-9e8f-f1cb-543941b5ff6d: aborting pipeline replication: peer="{Voter server-f7058a0b-b929-5cbf-b52f-b54829392656 f7058a0b-b929-5cbf-b52f-b54829392656}" ---- PASS: TestRaft_LeadershipTransferToInvalidID (0.19s) +2026-11-29T10:29:52.375Z [INFO] server-6c54e819-6a9d-8823-15eb-b990df924fd7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6c54e819-6a9d-8823-15eb-b990df924fd7 Address:6c54e819-6a9d-8823-15eb-b990df924fd7} {Suffrage:Voter ID:server-b30cc0fb-7e76-5192-230c-054a6aaff504 Address:b30cc0fb-7e76-5192-230c-054a6aaff504} {Suffrage:Voter ID:server-83bee370-7b16-3e12-9675-34c6336291a2 Address:83bee370-7b16-3e12-9675-34c6336291a2}]" +2026-11-29T10:29:52.375Z [INFO] server-b30cc0fb-7e76-5192-230c-054a6aaff504: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6c54e819-6a9d-8823-15eb-b990df924fd7 Address:6c54e819-6a9d-8823-15eb-b990df924fd7} {Suffrage:Voter ID:server-b30cc0fb-7e76-5192-230c-054a6aaff504 Address:b30cc0fb-7e76-5192-230c-054a6aaff504} {Suffrage:Voter ID:server-83bee370-7b16-3e12-9675-34c6336291a2 Address:83bee370-7b16-3e12-9675-34c6336291a2}]" +2026-11-29T10:29:52.375Z [INFO] server-83bee370-7b16-3e12-9675-34c6336291a2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6c54e819-6a9d-8823-15eb-b990df924fd7 Address:6c54e819-6a9d-8823-15eb-b990df924fd7} {Suffrage:Voter ID:server-b30cc0fb-7e76-5192-230c-054a6aaff504 Address:b30cc0fb-7e76-5192-230c-054a6aaff504} {Suffrage:Voter ID:server-83bee370-7b16-3e12-9675-34c6336291a2 Address:83bee370-7b16-3e12-9675-34c6336291a2}]" +2026-11-29T10:29:52.375Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:52.375Z [INFO] server-6c54e819-6a9d-8823-15eb-b990df924fd7: entering follower state: follower="Node at 6c54e819-6a9d-8823-15eb-b990df924fd7 [Follower]" leader-address= leader-id= +2026-11-29T10:29:52.375Z [INFO] server-b30cc0fb-7e76-5192-230c-054a6aaff504: entering follower state: follower="Node at b30cc0fb-7e76-5192-230c-054a6aaff504 [Follower]" leader-address= leader-id= +2026-11-29T10:29:52.375Z [INFO] server-83bee370-7b16-3e12-9675-34c6336291a2: entering follower state: follower="Node at 83bee370-7b16-3e12-9675-34c6336291a2 [Follower]" leader-address= leader-id= +2026-11-29T10:29:52.450Z [WARN] server-6c54e819-6a9d-8823-15eb-b990df924fd7: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:52.450Z [INFO] server-6c54e819-6a9d-8823-15eb-b990df924fd7: entering candidate state: node="Node at 6c54e819-6a9d-8823-15eb-b990df924fd7 [Candidate]" term=2 +2026-11-29T10:29:52.450Z [INFO] server-6c54e819-6a9d-8823-15eb-b990df924fd7: election won: term=2 tally=2 +2026-11-29T10:29:52.450Z [INFO] server-6c54e819-6a9d-8823-15eb-b990df924fd7: entering leader state: leader="Node at 6c54e819-6a9d-8823-15eb-b990df924fd7 [Leader]" +2026-11-29T10:29:52.450Z [INFO] server-6c54e819-6a9d-8823-15eb-b990df924fd7: added peer, starting replication: peer=server-b30cc0fb-7e76-5192-230c-054a6aaff504 +2026-11-29T10:29:52.450Z [INFO] server-6c54e819-6a9d-8823-15eb-b990df924fd7: added peer, starting replication: peer=server-83bee370-7b16-3e12-9675-34c6336291a2 +2026-11-29T10:29:52.450Z [INFO] server-6c54e819-6a9d-8823-15eb-b990df924fd7: pipelining replication: peer="{Voter server-b30cc0fb-7e76-5192-230c-054a6aaff504 b30cc0fb-7e76-5192-230c-054a6aaff504}" +2026-11-29T10:29:52.450Z [INFO] server-6c54e819-6a9d-8823-15eb-b990df924fd7: pipelining replication: peer="{Voter server-83bee370-7b16-3e12-9675-34c6336291a2 83bee370-7b16-3e12-9675-34c6336291a2}" +2026-11-29T10:29:52.555Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:52.450822339 +0000 UTC m=+103.185876441 (1 nodes), 74.83032ms from start of poll, 75.04596ms from cluster start. Timeout at 2026-11-29 10:29:52.555843539 +0000 UTC m=+103.290897641, 105.0212ms after stability +2026-11-29T10:29:52.556Z [INFO] server-6c54e819-6a9d-8823-15eb-b990df924fd7: aborting pipeline replication: peer="{Voter server-83bee370-7b16-3e12-9675-34c6336291a2 83bee370-7b16-3e12-9675-34c6336291a2}" +2026-11-29T10:29:52.556Z [INFO] server-6c54e819-6a9d-8823-15eb-b990df924fd7: aborting pipeline replication: peer="{Voter server-b30cc0fb-7e76-5192-230c-054a6aaff504 b30cc0fb-7e76-5192-230c-054a6aaff504}" +--- PASS: TestRaft_LeadershipTransferToInvalidID (0.18s) === RUN TestRaft_LeadershipTransferToInvalidAddress -2025-10-27T04:03:50.229Z [INFO] server-487f7f06-db88-20bd-8835-66ee77f40d5b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-487f7f06-db88-20bd-8835-66ee77f40d5b Address:487f7f06-db88-20bd-8835-66ee77f40d5b} {Suffrage:Voter ID:server-635e47d1-3f76-9713-8e8e-1a8072fc4afd Address:635e47d1-3f76-9713-8e8e-1a8072fc4afd} {Suffrage:Voter ID:server-281e5d54-78a4-9962-7af4-d3f7683446e3 Address:281e5d54-78a4-9962-7af4-d3f7683446e3}]" -2025-10-27T04:03:50.229Z [INFO] server-635e47d1-3f76-9713-8e8e-1a8072fc4afd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-487f7f06-db88-20bd-8835-66ee77f40d5b Address:487f7f06-db88-20bd-8835-66ee77f40d5b} {Suffrage:Voter ID:server-635e47d1-3f76-9713-8e8e-1a8072fc4afd Address:635e47d1-3f76-9713-8e8e-1a8072fc4afd} {Suffrage:Voter ID:server-281e5d54-78a4-9962-7af4-d3f7683446e3 Address:281e5d54-78a4-9962-7af4-d3f7683446e3}]" -2025-10-27T04:03:50.229Z [INFO] server-281e5d54-78a4-9962-7af4-d3f7683446e3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-487f7f06-db88-20bd-8835-66ee77f40d5b Address:487f7f06-db88-20bd-8835-66ee77f40d5b} {Suffrage:Voter ID:server-635e47d1-3f76-9713-8e8e-1a8072fc4afd Address:635e47d1-3f76-9713-8e8e-1a8072fc4afd} {Suffrage:Voter ID:server-281e5d54-78a4-9962-7af4-d3f7683446e3 Address:281e5d54-78a4-9962-7af4-d3f7683446e3}]" -2025-10-27T04:03:50.229Z [INFO] server-487f7f06-db88-20bd-8835-66ee77f40d5b: entering follower state: follower="Node at 487f7f06-db88-20bd-8835-66ee77f40d5b [Follower]" leader-address= leader-id= -2025-10-27T04:03:50.229Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:03:50.229Z [INFO] server-635e47d1-3f76-9713-8e8e-1a8072fc4afd: entering follower state: follower="Node at 635e47d1-3f76-9713-8e8e-1a8072fc4afd [Follower]" leader-address= leader-id= -2025-10-27T04:03:50.229Z [INFO] server-281e5d54-78a4-9962-7af4-d3f7683446e3: entering follower state: follower="Node at 281e5d54-78a4-9962-7af4-d3f7683446e3 [Follower]" leader-address= leader-id= -2025-10-27T04:03:50.280Z [WARN] server-635e47d1-3f76-9713-8e8e-1a8072fc4afd: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:50.280Z [INFO] server-635e47d1-3f76-9713-8e8e-1a8072fc4afd: entering candidate state: node="Node at 635e47d1-3f76-9713-8e8e-1a8072fc4afd [Candidate]" term=2 -2025-10-27T04:03:50.280Z [INFO] server-635e47d1-3f76-9713-8e8e-1a8072fc4afd: election won: term=2 tally=2 -2025-10-27T04:03:50.280Z [INFO] server-635e47d1-3f76-9713-8e8e-1a8072fc4afd: entering leader state: leader="Node at 635e47d1-3f76-9713-8e8e-1a8072fc4afd [Leader]" -2025-10-27T04:03:50.280Z [INFO] server-635e47d1-3f76-9713-8e8e-1a8072fc4afd: added peer, starting replication: peer=server-487f7f06-db88-20bd-8835-66ee77f40d5b -2025-10-27T04:03:50.280Z [INFO] server-635e47d1-3f76-9713-8e8e-1a8072fc4afd: added peer, starting replication: peer=server-281e5d54-78a4-9962-7af4-d3f7683446e3 -2025-10-27T04:03:50.280Z [INFO] server-635e47d1-3f76-9713-8e8e-1a8072fc4afd: pipelining replication: peer="{Voter server-281e5d54-78a4-9962-7af4-d3f7683446e3 281e5d54-78a4-9962-7af4-d3f7683446e3}" -2025-10-27T04:03:50.280Z [INFO] server-635e47d1-3f76-9713-8e8e-1a8072fc4afd: pipelining replication: peer="{Voter server-487f7f06-db88-20bd-8835-66ee77f40d5b 487f7f06-db88-20bd-8835-66ee77f40d5b}" -2025-10-27T04:03:50.385Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:03:50.280483356 +0000 UTC m=+105.275355794 (2 nodes), 50.556746ms from start of poll, 50.767227ms from cluster start. Timeout at 2025-10-27 04:03:50.385485556 +0000 UTC m=+105.380357994, 105.0022ms after stability -2025-10-27T04:03:50.385Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:50.491Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:50.385977476 +0000 UTC m=+105.380849914 (1 nodes), 1.64µs from start of poll, 156.261347ms from cluster start. Timeout at 2025-10-27 04:03:50.490998436 +0000 UTC m=+105.485870874, 105.02096ms after stability -2025-10-27T04:03:50.491Z [INFO] server-635e47d1-3f76-9713-8e8e-1a8072fc4afd: aborting pipeline replication: peer="{Voter server-281e5d54-78a4-9962-7af4-d3f7683446e3 281e5d54-78a4-9962-7af4-d3f7683446e3}" -2025-10-27T04:03:50.491Z [INFO] server-635e47d1-3f76-9713-8e8e-1a8072fc4afd: aborting pipeline replication: peer="{Voter server-487f7f06-db88-20bd-8835-66ee77f40d5b 487f7f06-db88-20bd-8835-66ee77f40d5b}" ---- PASS: TestRaft_LeadershipTransferToInvalidAddress (0.26s) +2026-11-29T10:29:52.556Z [INFO] server-1016f409-07a9-0096-5320-149511582746: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1016f409-07a9-0096-5320-149511582746 Address:1016f409-07a9-0096-5320-149511582746} {Suffrage:Voter ID:server-142d4ee2-95e7-9d8a-4882-f46705308749 Address:142d4ee2-95e7-9d8a-4882-f46705308749} {Suffrage:Voter ID:server-98855e63-a570-df2d-be37-fec0d5efca81 Address:98855e63-a570-df2d-be37-fec0d5efca81}]" +2026-11-29T10:29:52.556Z [INFO] server-1016f409-07a9-0096-5320-149511582746: entering follower state: follower="Node at 1016f409-07a9-0096-5320-149511582746 [Follower]" leader-address= leader-id= +2026-11-29T10:29:52.556Z [INFO] server-142d4ee2-95e7-9d8a-4882-f46705308749: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1016f409-07a9-0096-5320-149511582746 Address:1016f409-07a9-0096-5320-149511582746} {Suffrage:Voter ID:server-142d4ee2-95e7-9d8a-4882-f46705308749 Address:142d4ee2-95e7-9d8a-4882-f46705308749} {Suffrage:Voter ID:server-98855e63-a570-df2d-be37-fec0d5efca81 Address:98855e63-a570-df2d-be37-fec0d5efca81}]" +2026-11-29T10:29:52.556Z [INFO] server-142d4ee2-95e7-9d8a-4882-f46705308749: entering follower state: follower="Node at 142d4ee2-95e7-9d8a-4882-f46705308749 [Follower]" leader-address= leader-id= +2026-11-29T10:29:52.556Z [INFO] server-98855e63-a570-df2d-be37-fec0d5efca81: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1016f409-07a9-0096-5320-149511582746 Address:1016f409-07a9-0096-5320-149511582746} {Suffrage:Voter ID:server-142d4ee2-95e7-9d8a-4882-f46705308749 Address:142d4ee2-95e7-9d8a-4882-f46705308749} {Suffrage:Voter ID:server-98855e63-a570-df2d-be37-fec0d5efca81 Address:98855e63-a570-df2d-be37-fec0d5efca81}]" +2026-11-29T10:29:52.556Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:52.556Z [INFO] server-98855e63-a570-df2d-be37-fec0d5efca81: entering follower state: follower="Node at 98855e63-a570-df2d-be37-fec0d5efca81 [Follower]" leader-address= leader-id= +2026-11-29T10:29:52.620Z [WARN] server-98855e63-a570-df2d-be37-fec0d5efca81: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:52.620Z [INFO] server-98855e63-a570-df2d-be37-fec0d5efca81: entering candidate state: node="Node at 98855e63-a570-df2d-be37-fec0d5efca81 [Candidate]" term=2 +2026-11-29T10:29:52.620Z [INFO] server-98855e63-a570-df2d-be37-fec0d5efca81: election won: term=2 tally=2 +2026-11-29T10:29:52.620Z [INFO] server-98855e63-a570-df2d-be37-fec0d5efca81: entering leader state: leader="Node at 98855e63-a570-df2d-be37-fec0d5efca81 [Leader]" +2026-11-29T10:29:52.620Z [INFO] server-98855e63-a570-df2d-be37-fec0d5efca81: added peer, starting replication: peer=server-1016f409-07a9-0096-5320-149511582746 +2026-11-29T10:29:52.620Z [INFO] server-98855e63-a570-df2d-be37-fec0d5efca81: added peer, starting replication: peer=server-142d4ee2-95e7-9d8a-4882-f46705308749 +2026-11-29T10:29:52.620Z [INFO] server-98855e63-a570-df2d-be37-fec0d5efca81: pipelining replication: peer="{Voter server-142d4ee2-95e7-9d8a-4882-f46705308749 142d4ee2-95e7-9d8a-4882-f46705308749}" +2026-11-29T10:29:52.620Z [INFO] server-98855e63-a570-df2d-be37-fec0d5efca81: pipelining replication: peer="{Voter server-1016f409-07a9-0096-5320-149511582746 1016f409-07a9-0096-5320-149511582746}" +2026-11-29T10:29:52.726Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:52.620703739 +0000 UTC m=+103.355757841 (2 nodes), 63.91ms from start of poll, 64.16584ms from cluster start. Timeout at 2026-11-29 10:29:52.725710379 +0000 UTC m=+103.460764561, 105.00672ms after stability +2026-11-29T10:29:52.726Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:52.832Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:52.726335059 +0000 UTC m=+103.461389161 (1 nodes), 440ns from start of poll, 169.79716ms from cluster start. Timeout at 2026-11-29 10:29:52.831360099 +0000 UTC m=+103.566414201, 105.02504ms after stability +2026-11-29T10:29:52.832Z [INFO] server-98855e63-a570-df2d-be37-fec0d5efca81: aborting pipeline replication: peer="{Voter server-142d4ee2-95e7-9d8a-4882-f46705308749 142d4ee2-95e7-9d8a-4882-f46705308749}" +2026-11-29T10:29:52.832Z [INFO] server-98855e63-a570-df2d-be37-fec0d5efca81: aborting pipeline replication: peer="{Voter server-1016f409-07a9-0096-5320-149511582746 1016f409-07a9-0096-5320-149511582746}" +--- PASS: TestRaft_LeadershipTransferToInvalidAddress (0.28s) === RUN TestRaft_LeadershipTransferToBehindServer -2025-10-27T04:03:50.491Z [INFO] server-a203fd73-04c5-6379-8fe2-afc59e0d0136: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a203fd73-04c5-6379-8fe2-afc59e0d0136 Address:a203fd73-04c5-6379-8fe2-afc59e0d0136} {Suffrage:Voter ID:server-0238f148-5d06-faef-92e3-4dabf767b6cb Address:0238f148-5d06-faef-92e3-4dabf767b6cb} {Suffrage:Voter ID:server-feebf6c8-f109-635d-8165-439474e249c6 Address:feebf6c8-f109-635d-8165-439474e249c6}]" -2025-10-27T04:03:50.491Z [INFO] server-0238f148-5d06-faef-92e3-4dabf767b6cb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a203fd73-04c5-6379-8fe2-afc59e0d0136 Address:a203fd73-04c5-6379-8fe2-afc59e0d0136} {Suffrage:Voter ID:server-0238f148-5d06-faef-92e3-4dabf767b6cb Address:0238f148-5d06-faef-92e3-4dabf767b6cb} {Suffrage:Voter ID:server-feebf6c8-f109-635d-8165-439474e249c6 Address:feebf6c8-f109-635d-8165-439474e249c6}]" -2025-10-27T04:03:50.491Z [INFO] server-feebf6c8-f109-635d-8165-439474e249c6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a203fd73-04c5-6379-8fe2-afc59e0d0136 Address:a203fd73-04c5-6379-8fe2-afc59e0d0136} {Suffrage:Voter ID:server-0238f148-5d06-faef-92e3-4dabf767b6cb Address:0238f148-5d06-faef-92e3-4dabf767b6cb} {Suffrage:Voter ID:server-feebf6c8-f109-635d-8165-439474e249c6 Address:feebf6c8-f109-635d-8165-439474e249c6}]" -2025-10-27T04:03:50.491Z [INFO] server-0238f148-5d06-faef-92e3-4dabf767b6cb: entering follower state: follower="Node at 0238f148-5d06-faef-92e3-4dabf767b6cb [Follower]" leader-address= leader-id= -2025-10-27T04:03:50.491Z [INFO] server-a203fd73-04c5-6379-8fe2-afc59e0d0136: entering follower state: follower="Node at a203fd73-04c5-6379-8fe2-afc59e0d0136 [Follower]" leader-address= leader-id= -2025-10-27T04:03:50.491Z [INFO] server-feebf6c8-f109-635d-8165-439474e249c6: entering follower state: follower="Node at feebf6c8-f109-635d-8165-439474e249c6 [Follower]" leader-address= leader-id= -2025-10-27T04:03:50.491Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:50.546Z [WARN] server-0238f148-5d06-faef-92e3-4dabf767b6cb: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:50.546Z [INFO] server-0238f148-5d06-faef-92e3-4dabf767b6cb: entering candidate state: node="Node at 0238f148-5d06-faef-92e3-4dabf767b6cb [Candidate]" term=2 -2025-10-27T04:03:50.546Z [INFO] server-0238f148-5d06-faef-92e3-4dabf767b6cb: election won: term=2 tally=2 -2025-10-27T04:03:50.546Z [INFO] server-0238f148-5d06-faef-92e3-4dabf767b6cb: entering leader state: leader="Node at 0238f148-5d06-faef-92e3-4dabf767b6cb [Leader]" -2025-10-27T04:03:50.546Z [INFO] server-0238f148-5d06-faef-92e3-4dabf767b6cb: added peer, starting replication: peer=server-a203fd73-04c5-6379-8fe2-afc59e0d0136 -2025-10-27T04:03:50.546Z [INFO] server-0238f148-5d06-faef-92e3-4dabf767b6cb: added peer, starting replication: peer=server-feebf6c8-f109-635d-8165-439474e249c6 -2025-10-27T04:03:50.546Z [INFO] server-0238f148-5d06-faef-92e3-4dabf767b6cb: pipelining replication: peer="{Voter server-a203fd73-04c5-6379-8fe2-afc59e0d0136 a203fd73-04c5-6379-8fe2-afc59e0d0136}" -2025-10-27T04:03:50.546Z [INFO] server-0238f148-5d06-faef-92e3-4dabf767b6cb: pipelining replication: peer="{Voter server-feebf6c8-f109-635d-8165-439474e249c6 feebf6c8-f109-635d-8165-439474e249c6}" -2025-10-27T04:03:50.651Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:50.546193571 +0000 UTC m=+105.541066009 (1 nodes), 54.423252ms from start of poll, 54.653334ms from cluster start. Timeout at 2025-10-27 04:03:50.651201491 +0000 UTC m=+105.646073969, 105.00796ms after stability -2025-10-27T04:03:50.651Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:03:50.757Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:03:50.651932813 +0000 UTC m=+105.646805291 (2 nodes), 1.44µs from start of poll, 160.392616ms from cluster start. Timeout at 2025-10-27 04:03:50.756966894 +0000 UTC m=+105.751839412, 105.034121ms after stability -2025-10-27T04:03:50.805Z [INFO] server-a203fd73-04c5-6379-8fe2-afc59e0d0136: entering candidate state: node="Node at a203fd73-04c5-6379-8fe2-afc59e0d0136 [Candidate]" term=3 -2025-10-27T04:03:50.805Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:50.805Z [INFO] server-0238f148-5d06-faef-92e3-4dabf767b6cb: entering follower state: follower="Node at 0238f148-5d06-faef-92e3-4dabf767b6cb [Follower]" leader-address= leader-id= -2025-10-27T04:03:50.805Z [INFO] server-0238f148-5d06-faef-92e3-4dabf767b6cb: aborting pipeline replication: peer="{Voter server-feebf6c8-f109-635d-8165-439474e249c6 feebf6c8-f109-635d-8165-439474e249c6}" -2025-10-27T04:03:50.805Z [INFO] server-0238f148-5d06-faef-92e3-4dabf767b6cb: aborting pipeline replication: peer="{Voter server-a203fd73-04c5-6379-8fe2-afc59e0d0136 a203fd73-04c5-6379-8fe2-afc59e0d0136}" -2025-10-27T04:03:50.805Z [INFO] server-a203fd73-04c5-6379-8fe2-afc59e0d0136: election won: term=3 tally=2 -2025-10-27T04:03:50.805Z [INFO] server-a203fd73-04c5-6379-8fe2-afc59e0d0136: entering leader state: leader="Node at a203fd73-04c5-6379-8fe2-afc59e0d0136 [Leader]" -2025-10-27T04:03:50.805Z [INFO] server-a203fd73-04c5-6379-8fe2-afc59e0d0136: added peer, starting replication: peer=server-0238f148-5d06-faef-92e3-4dabf767b6cb -2025-10-27T04:03:50.805Z [INFO] server-a203fd73-04c5-6379-8fe2-afc59e0d0136: added peer, starting replication: peer=server-feebf6c8-f109-635d-8165-439474e249c6 -2025-10-27T04:03:50.805Z [INFO] server-a203fd73-04c5-6379-8fe2-afc59e0d0136: pipelining replication: peer="{Voter server-feebf6c8-f109-635d-8165-439474e249c6 feebf6c8-f109-635d-8165-439474e249c6}" -2025-10-27T04:03:50.805Z [INFO] server-a203fd73-04c5-6379-8fe2-afc59e0d0136: pipelining replication: peer="{Voter server-0238f148-5d06-faef-92e3-4dabf767b6cb 0238f148-5d06-faef-92e3-4dabf767b6cb}" -2025-10-27T04:03:50.911Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:50.805639303 +0000 UTC m=+105.800511781 (1 nodes), 430.123µs from start of poll, 314.099106ms from cluster start. Timeout at 2025-10-27 04:03:50.910678783 +0000 UTC m=+105.905551261, 105.03948ms after stability -2025-10-27T04:03:50.911Z [INFO] server-a203fd73-04c5-6379-8fe2-afc59e0d0136: aborting pipeline replication: peer="{Voter server-0238f148-5d06-faef-92e3-4dabf767b6cb 0238f148-5d06-faef-92e3-4dabf767b6cb}" -2025-10-27T04:03:50.911Z [INFO] server-a203fd73-04c5-6379-8fe2-afc59e0d0136: aborting pipeline replication: peer="{Voter server-feebf6c8-f109-635d-8165-439474e249c6 feebf6c8-f109-635d-8165-439474e249c6}" ---- PASS: TestRaft_LeadershipTransferToBehindServer (0.42s) +2026-11-29T10:29:52.833Z [INFO] server-26ca6cac-851d-a444-e4fa-21cf8e79c833: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-26ca6cac-851d-a444-e4fa-21cf8e79c833 Address:26ca6cac-851d-a444-e4fa-21cf8e79c833} {Suffrage:Voter ID:server-34d6af28-8626-39b9-44d9-7c368e43d1a6 Address:34d6af28-8626-39b9-44d9-7c368e43d1a6} {Suffrage:Voter ID:server-92d82364-b77c-9508-814d-96c7782fedd9 Address:92d82364-b77c-9508-814d-96c7782fedd9}]" +2026-11-29T10:29:52.833Z [INFO] server-34d6af28-8626-39b9-44d9-7c368e43d1a6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-26ca6cac-851d-a444-e4fa-21cf8e79c833 Address:26ca6cac-851d-a444-e4fa-21cf8e79c833} {Suffrage:Voter ID:server-34d6af28-8626-39b9-44d9-7c368e43d1a6 Address:34d6af28-8626-39b9-44d9-7c368e43d1a6} {Suffrage:Voter ID:server-92d82364-b77c-9508-814d-96c7782fedd9 Address:92d82364-b77c-9508-814d-96c7782fedd9}]" +2026-11-29T10:29:52.833Z [INFO] server-26ca6cac-851d-a444-e4fa-21cf8e79c833: entering follower state: follower="Node at 26ca6cac-851d-a444-e4fa-21cf8e79c833 [Follower]" leader-address= leader-id= +2026-11-29T10:29:52.833Z [INFO] server-34d6af28-8626-39b9-44d9-7c368e43d1a6: entering follower state: follower="Node at 34d6af28-8626-39b9-44d9-7c368e43d1a6 [Follower]" leader-address= leader-id= +2026-11-29T10:29:52.833Z [INFO] server-92d82364-b77c-9508-814d-96c7782fedd9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-26ca6cac-851d-a444-e4fa-21cf8e79c833 Address:26ca6cac-851d-a444-e4fa-21cf8e79c833} {Suffrage:Voter ID:server-34d6af28-8626-39b9-44d9-7c368e43d1a6 Address:34d6af28-8626-39b9-44d9-7c368e43d1a6} {Suffrage:Voter ID:server-92d82364-b77c-9508-814d-96c7782fedd9 Address:92d82364-b77c-9508-814d-96c7782fedd9}]" +2026-11-29T10:29:52.833Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:52.833Z [INFO] server-92d82364-b77c-9508-814d-96c7782fedd9: entering follower state: follower="Node at 92d82364-b77c-9508-814d-96c7782fedd9 [Follower]" leader-address= leader-id= +2026-11-29T10:29:52.884Z [WARN] server-26ca6cac-851d-a444-e4fa-21cf8e79c833: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:52.884Z [INFO] server-26ca6cac-851d-a444-e4fa-21cf8e79c833: entering candidate state: node="Node at 26ca6cac-851d-a444-e4fa-21cf8e79c833 [Candidate]" term=2 +2026-11-29T10:29:52.884Z [INFO] server-26ca6cac-851d-a444-e4fa-21cf8e79c833: election won: term=2 tally=2 +2026-11-29T10:29:52.884Z [INFO] server-26ca6cac-851d-a444-e4fa-21cf8e79c833: entering leader state: leader="Node at 26ca6cac-851d-a444-e4fa-21cf8e79c833 [Leader]" +2026-11-29T10:29:52.884Z [INFO] server-26ca6cac-851d-a444-e4fa-21cf8e79c833: added peer, starting replication: peer=server-34d6af28-8626-39b9-44d9-7c368e43d1a6 +2026-11-29T10:29:52.884Z [INFO] server-26ca6cac-851d-a444-e4fa-21cf8e79c833: added peer, starting replication: peer=server-92d82364-b77c-9508-814d-96c7782fedd9 +2026-11-29T10:29:52.884Z [INFO] server-26ca6cac-851d-a444-e4fa-21cf8e79c833: pipelining replication: peer="{Voter server-34d6af28-8626-39b9-44d9-7c368e43d1a6 34d6af28-8626-39b9-44d9-7c368e43d1a6}" +2026-11-29T10:29:52.884Z [INFO] server-26ca6cac-851d-a444-e4fa-21cf8e79c833: pipelining replication: peer="{Voter server-92d82364-b77c-9508-814d-96c7782fedd9 92d82364-b77c-9508-814d-96c7782fedd9}" +2026-11-29T10:29:52.989Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:52.884328619 +0000 UTC m=+103.619382721 (1 nodes), 50.93544ms from start of poll, 51.1558ms from cluster start. Timeout at 2026-11-29 10:29:52.989336419 +0000 UTC m=+103.724390521, 105.0078ms after stability +2026-11-29T10:29:52.989Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:53.095Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:52.989918339 +0000 UTC m=+103.724972441 (2 nodes), 1.28µs from start of poll, 156.74552ms from cluster start. Timeout at 2026-11-29 10:29:53.094940139 +0000 UTC m=+103.829994201, 105.02176ms after stability +2026-11-29T10:29:53.130Z [INFO] server-34d6af28-8626-39b9-44d9-7c368e43d1a6: entering candidate state: node="Node at 34d6af28-8626-39b9-44d9-7c368e43d1a6 [Candidate]" term=3 +2026-11-29T10:29:53.130Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:53.130Z [INFO] server-34d6af28-8626-39b9-44d9-7c368e43d1a6: election won: term=3 tally=2 +2026-11-29T10:29:53.130Z [INFO] server-34d6af28-8626-39b9-44d9-7c368e43d1a6: entering leader state: leader="Node at 34d6af28-8626-39b9-44d9-7c368e43d1a6 [Leader]" +2026-11-29T10:29:53.130Z [INFO] server-34d6af28-8626-39b9-44d9-7c368e43d1a6: added peer, starting replication: peer=server-26ca6cac-851d-a444-e4fa-21cf8e79c833 +2026-11-29T10:29:53.130Z [INFO] server-26ca6cac-851d-a444-e4fa-21cf8e79c833: entering follower state: follower="Node at 26ca6cac-851d-a444-e4fa-21cf8e79c833 [Follower]" leader-address= leader-id= +2026-11-29T10:29:53.130Z [INFO] server-26ca6cac-851d-a444-e4fa-21cf8e79c833: aborting pipeline replication: peer="{Voter server-34d6af28-8626-39b9-44d9-7c368e43d1a6 34d6af28-8626-39b9-44d9-7c368e43d1a6}" +2026-11-29T10:29:53.130Z [INFO] server-26ca6cac-851d-a444-e4fa-21cf8e79c833: aborting pipeline replication: peer="{Voter server-92d82364-b77c-9508-814d-96c7782fedd9 92d82364-b77c-9508-814d-96c7782fedd9}" +2026-11-29T10:29:53.130Z [INFO] server-34d6af28-8626-39b9-44d9-7c368e43d1a6: added peer, starting replication: peer=server-92d82364-b77c-9508-814d-96c7782fedd9 +2026-11-29T10:29:53.130Z [INFO] server-34d6af28-8626-39b9-44d9-7c368e43d1a6: pipelining replication: peer="{Voter server-26ca6cac-851d-a444-e4fa-21cf8e79c833 26ca6cac-851d-a444-e4fa-21cf8e79c833}" +2026-11-29T10:29:53.130Z [INFO] server-34d6af28-8626-39b9-44d9-7c368e43d1a6: pipelining replication: peer="{Voter server-92d82364-b77c-9508-814d-96c7782fedd9 92d82364-b77c-9508-814d-96c7782fedd9}" +2026-11-29T10:29:53.235Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:53.130093459 +0000 UTC m=+103.865147561 (1 nodes), 16.24µs from start of poll, 296.92064ms from cluster start. Timeout at 2026-11-29 10:29:53.235114939 +0000 UTC m=+103.970169041, 105.02148ms after stability +2026-11-29T10:29:53.235Z [INFO] server-34d6af28-8626-39b9-44d9-7c368e43d1a6: aborting pipeline replication: peer="{Voter server-92d82364-b77c-9508-814d-96c7782fedd9 92d82364-b77c-9508-814d-96c7782fedd9}" +2026-11-29T10:29:53.235Z [INFO] server-34d6af28-8626-39b9-44d9-7c368e43d1a6: aborting pipeline replication: peer="{Voter server-26ca6cac-851d-a444-e4fa-21cf8e79c833 26ca6cac-851d-a444-e4fa-21cf8e79c833}" +--- PASS: TestRaft_LeadershipTransferToBehindServer (0.40s) === RUN TestRaft_LeadershipTransferToItself -2025-10-27T04:03:50.912Z [INFO] server-fc3a9347-02ed-08e5-75ef-b3f9326e5a70: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fc3a9347-02ed-08e5-75ef-b3f9326e5a70 Address:fc3a9347-02ed-08e5-75ef-b3f9326e5a70} {Suffrage:Voter ID:server-76aa2de4-22bc-a273-b5c6-13026a51b821 Address:76aa2de4-22bc-a273-b5c6-13026a51b821} {Suffrage:Voter ID:server-945ae5e7-702a-9efd-e997-c31fb80ea0ee Address:945ae5e7-702a-9efd-e997-c31fb80ea0ee}]" -2025-10-27T04:03:50.912Z [INFO] server-76aa2de4-22bc-a273-b5c6-13026a51b821: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fc3a9347-02ed-08e5-75ef-b3f9326e5a70 Address:fc3a9347-02ed-08e5-75ef-b3f9326e5a70} {Suffrage:Voter ID:server-76aa2de4-22bc-a273-b5c6-13026a51b821 Address:76aa2de4-22bc-a273-b5c6-13026a51b821} {Suffrage:Voter ID:server-945ae5e7-702a-9efd-e997-c31fb80ea0ee Address:945ae5e7-702a-9efd-e997-c31fb80ea0ee}]" -2025-10-27T04:03:50.912Z [INFO] server-fc3a9347-02ed-08e5-75ef-b3f9326e5a70: entering follower state: follower="Node at fc3a9347-02ed-08e5-75ef-b3f9326e5a70 [Follower]" leader-address= leader-id= -2025-10-27T04:03:50.912Z [INFO] server-76aa2de4-22bc-a273-b5c6-13026a51b821: entering follower state: follower="Node at 76aa2de4-22bc-a273-b5c6-13026a51b821 [Follower]" leader-address= leader-id= -2025-10-27T04:03:50.912Z [INFO] server-945ae5e7-702a-9efd-e997-c31fb80ea0ee: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fc3a9347-02ed-08e5-75ef-b3f9326e5a70 Address:fc3a9347-02ed-08e5-75ef-b3f9326e5a70} {Suffrage:Voter ID:server-76aa2de4-22bc-a273-b5c6-13026a51b821 Address:76aa2de4-22bc-a273-b5c6-13026a51b821} {Suffrage:Voter ID:server-945ae5e7-702a-9efd-e997-c31fb80ea0ee Address:945ae5e7-702a-9efd-e997-c31fb80ea0ee}]" -2025-10-27T04:03:50.912Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:50.912Z [INFO] server-945ae5e7-702a-9efd-e997-c31fb80ea0ee: entering follower state: follower="Node at 945ae5e7-702a-9efd-e997-c31fb80ea0ee [Follower]" leader-address= leader-id= -2025-10-27T04:03:50.979Z [WARN] server-fc3a9347-02ed-08e5-75ef-b3f9326e5a70: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:50.979Z [INFO] server-fc3a9347-02ed-08e5-75ef-b3f9326e5a70: entering candidate state: node="Node at fc3a9347-02ed-08e5-75ef-b3f9326e5a70 [Candidate]" term=2 -2025-10-27T04:03:50.979Z [INFO] server-fc3a9347-02ed-08e5-75ef-b3f9326e5a70: election won: term=2 tally=2 -2025-10-27T04:03:50.979Z [INFO] server-fc3a9347-02ed-08e5-75ef-b3f9326e5a70: entering leader state: leader="Node at fc3a9347-02ed-08e5-75ef-b3f9326e5a70 [Leader]" -2025-10-27T04:03:50.979Z [INFO] server-fc3a9347-02ed-08e5-75ef-b3f9326e5a70: added peer, starting replication: peer=server-76aa2de4-22bc-a273-b5c6-13026a51b821 -2025-10-27T04:03:50.979Z [INFO] server-fc3a9347-02ed-08e5-75ef-b3f9326e5a70: added peer, starting replication: peer=server-945ae5e7-702a-9efd-e997-c31fb80ea0ee -2025-10-27T04:03:50.979Z [INFO] server-fc3a9347-02ed-08e5-75ef-b3f9326e5a70: pipelining replication: peer="{Voter server-945ae5e7-702a-9efd-e997-c31fb80ea0ee 945ae5e7-702a-9efd-e997-c31fb80ea0ee}" -2025-10-27T04:03:50.979Z [INFO] server-fc3a9347-02ed-08e5-75ef-b3f9326e5a70: pipelining replication: peer="{Voter server-76aa2de4-22bc-a273-b5c6-13026a51b821 76aa2de4-22bc-a273-b5c6-13026a51b821}" -2025-10-27T04:03:51.084Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:50.97938129 +0000 UTC m=+105.974253728 (1 nodes), 66.902577ms from start of poll, 67.31018ms from cluster start. Timeout at 2025-10-27 04:03:51.08438777 +0000 UTC m=+106.079260248, 105.00652ms after stability -2025-10-27T04:03:51.084Z [INFO] server-fc3a9347-02ed-08e5-75ef-b3f9326e5a70: cannot transfer leadership to itself -2025-10-27T04:03:51.084Z [INFO] server-fc3a9347-02ed-08e5-75ef-b3f9326e5a70: aborting pipeline replication: peer="{Voter server-76aa2de4-22bc-a273-b5c6-13026a51b821 76aa2de4-22bc-a273-b5c6-13026a51b821}" -2025-10-27T04:03:51.084Z [INFO] server-fc3a9347-02ed-08e5-75ef-b3f9326e5a70: aborting pipeline replication: peer="{Voter server-945ae5e7-702a-9efd-e997-c31fb80ea0ee 945ae5e7-702a-9efd-e997-c31fb80ea0ee}" ---- PASS: TestRaft_LeadershipTransferToItself (0.17s) +2026-11-29T10:29:53.235Z [INFO] server-3937c3bd-9a63-dfe0-f495-7798f2a7bb83: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3937c3bd-9a63-dfe0-f495-7798f2a7bb83 Address:3937c3bd-9a63-dfe0-f495-7798f2a7bb83} {Suffrage:Voter ID:server-4ff3afaf-4802-112f-acc3-27d24aa1ebfa Address:4ff3afaf-4802-112f-acc3-27d24aa1ebfa} {Suffrage:Voter ID:server-1033bf8b-3d0b-8eb9-a329-8c819199ae75 Address:1033bf8b-3d0b-8eb9-a329-8c819199ae75}]" +2026-11-29T10:29:53.235Z [INFO] server-4ff3afaf-4802-112f-acc3-27d24aa1ebfa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3937c3bd-9a63-dfe0-f495-7798f2a7bb83 Address:3937c3bd-9a63-dfe0-f495-7798f2a7bb83} {Suffrage:Voter ID:server-4ff3afaf-4802-112f-acc3-27d24aa1ebfa Address:4ff3afaf-4802-112f-acc3-27d24aa1ebfa} {Suffrage:Voter ID:server-1033bf8b-3d0b-8eb9-a329-8c819199ae75 Address:1033bf8b-3d0b-8eb9-a329-8c819199ae75}]" +2026-11-29T10:29:53.235Z [INFO] server-3937c3bd-9a63-dfe0-f495-7798f2a7bb83: entering follower state: follower="Node at 3937c3bd-9a63-dfe0-f495-7798f2a7bb83 [Follower]" leader-address= leader-id= +2026-11-29T10:29:53.235Z [INFO] server-1033bf8b-3d0b-8eb9-a329-8c819199ae75: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3937c3bd-9a63-dfe0-f495-7798f2a7bb83 Address:3937c3bd-9a63-dfe0-f495-7798f2a7bb83} {Suffrage:Voter ID:server-4ff3afaf-4802-112f-acc3-27d24aa1ebfa Address:4ff3afaf-4802-112f-acc3-27d24aa1ebfa} {Suffrage:Voter ID:server-1033bf8b-3d0b-8eb9-a329-8c819199ae75 Address:1033bf8b-3d0b-8eb9-a329-8c819199ae75}]" +2026-11-29T10:29:53.235Z [INFO] server-4ff3afaf-4802-112f-acc3-27d24aa1ebfa: entering follower state: follower="Node at 4ff3afaf-4802-112f-acc3-27d24aa1ebfa [Follower]" leader-address= leader-id= +2026-11-29T10:29:53.235Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:53.235Z [INFO] server-1033bf8b-3d0b-8eb9-a329-8c819199ae75: entering follower state: follower="Node at 1033bf8b-3d0b-8eb9-a329-8c819199ae75 [Follower]" leader-address= leader-id= +2026-11-29T10:29:53.310Z [WARN] server-1033bf8b-3d0b-8eb9-a329-8c819199ae75: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:53.310Z [INFO] server-1033bf8b-3d0b-8eb9-a329-8c819199ae75: entering candidate state: node="Node at 1033bf8b-3d0b-8eb9-a329-8c819199ae75 [Candidate]" term=2 +2026-11-29T10:29:53.310Z [INFO] server-1033bf8b-3d0b-8eb9-a329-8c819199ae75: election won: term=2 tally=2 +2026-11-29T10:29:53.310Z [INFO] server-1033bf8b-3d0b-8eb9-a329-8c819199ae75: entering leader state: leader="Node at 1033bf8b-3d0b-8eb9-a329-8c819199ae75 [Leader]" +2026-11-29T10:29:53.310Z [INFO] server-1033bf8b-3d0b-8eb9-a329-8c819199ae75: added peer, starting replication: peer=server-3937c3bd-9a63-dfe0-f495-7798f2a7bb83 +2026-11-29T10:29:53.310Z [INFO] server-1033bf8b-3d0b-8eb9-a329-8c819199ae75: added peer, starting replication: peer=server-4ff3afaf-4802-112f-acc3-27d24aa1ebfa +2026-11-29T10:29:53.310Z [INFO] server-1033bf8b-3d0b-8eb9-a329-8c819199ae75: pipelining replication: peer="{Voter server-4ff3afaf-4802-112f-acc3-27d24aa1ebfa 4ff3afaf-4802-112f-acc3-27d24aa1ebfa}" +2026-11-29T10:29:53.310Z [INFO] server-1033bf8b-3d0b-8eb9-a329-8c819199ae75: pipelining replication: peer="{Voter server-3937c3bd-9a63-dfe0-f495-7798f2a7bb83 3937c3bd-9a63-dfe0-f495-7798f2a7bb83}" +2026-11-29T10:29:53.416Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:53.310689699 +0000 UTC m=+104.045743761 (1 nodes), 74.70264ms from start of poll, 74.894ms from cluster start. Timeout at 2026-11-29 10:29:53.415691499 +0000 UTC m=+104.150745601, 105.00184ms after stability +2026-11-29T10:29:53.416Z [INFO] server-1033bf8b-3d0b-8eb9-a329-8c819199ae75: cannot transfer leadership to itself +2026-11-29T10:29:53.416Z [INFO] server-1033bf8b-3d0b-8eb9-a329-8c819199ae75: aborting pipeline replication: peer="{Voter server-4ff3afaf-4802-112f-acc3-27d24aa1ebfa 4ff3afaf-4802-112f-acc3-27d24aa1ebfa}" +2026-11-29T10:29:53.416Z [INFO] server-1033bf8b-3d0b-8eb9-a329-8c819199ae75: aborting pipeline replication: peer="{Voter server-3937c3bd-9a63-dfe0-f495-7798f2a7bb83 3937c3bd-9a63-dfe0-f495-7798f2a7bb83}" +--- PASS: TestRaft_LeadershipTransferToItself (0.18s) === RUN TestRaft_LeadershipTransferLeaderRejectsClientRequests -2025-10-27T04:03:51.085Z [INFO] server-cea66c26-5244-1b09-65f0-3197e3a340a6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cea66c26-5244-1b09-65f0-3197e3a340a6 Address:cea66c26-5244-1b09-65f0-3197e3a340a6} {Suffrage:Voter ID:server-605f64b3-042b-6803-b535-fcd5a8f2a878 Address:605f64b3-042b-6803-b535-fcd5a8f2a878} {Suffrage:Voter ID:server-8d428cc5-bd17-dcac-4589-61725287082b Address:8d428cc5-bd17-dcac-4589-61725287082b}]" -2025-10-27T04:03:51.085Z [INFO] server-605f64b3-042b-6803-b535-fcd5a8f2a878: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cea66c26-5244-1b09-65f0-3197e3a340a6 Address:cea66c26-5244-1b09-65f0-3197e3a340a6} {Suffrage:Voter ID:server-605f64b3-042b-6803-b535-fcd5a8f2a878 Address:605f64b3-042b-6803-b535-fcd5a8f2a878} {Suffrage:Voter ID:server-8d428cc5-bd17-dcac-4589-61725287082b Address:8d428cc5-bd17-dcac-4589-61725287082b}]" -2025-10-27T04:03:51.085Z [INFO] server-605f64b3-042b-6803-b535-fcd5a8f2a878: entering follower state: follower="Node at 605f64b3-042b-6803-b535-fcd5a8f2a878 [Follower]" leader-address= leader-id= -2025-10-27T04:03:51.085Z [INFO] server-8d428cc5-bd17-dcac-4589-61725287082b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cea66c26-5244-1b09-65f0-3197e3a340a6 Address:cea66c26-5244-1b09-65f0-3197e3a340a6} {Suffrage:Voter ID:server-605f64b3-042b-6803-b535-fcd5a8f2a878 Address:605f64b3-042b-6803-b535-fcd5a8f2a878} {Suffrage:Voter ID:server-8d428cc5-bd17-dcac-4589-61725287082b Address:8d428cc5-bd17-dcac-4589-61725287082b}]" -2025-10-27T04:03:51.085Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:51.085Z [INFO] server-cea66c26-5244-1b09-65f0-3197e3a340a6: entering follower state: follower="Node at cea66c26-5244-1b09-65f0-3197e3a340a6 [Follower]" leader-address= leader-id= -2025-10-27T04:03:51.085Z [INFO] server-8d428cc5-bd17-dcac-4589-61725287082b: entering follower state: follower="Node at 8d428cc5-bd17-dcac-4589-61725287082b [Follower]" leader-address= leader-id= -2025-10-27T04:03:51.172Z [WARN] server-cea66c26-5244-1b09-65f0-3197e3a340a6: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:51.172Z [INFO] server-cea66c26-5244-1b09-65f0-3197e3a340a6: entering candidate state: node="Node at cea66c26-5244-1b09-65f0-3197e3a340a6 [Candidate]" term=2 -2025-10-27T04:03:51.172Z [INFO] server-cea66c26-5244-1b09-65f0-3197e3a340a6: election won: term=2 tally=2 -2025-10-27T04:03:51.172Z [INFO] server-cea66c26-5244-1b09-65f0-3197e3a340a6: entering leader state: leader="Node at cea66c26-5244-1b09-65f0-3197e3a340a6 [Leader]" -2025-10-27T04:03:51.172Z [INFO] server-cea66c26-5244-1b09-65f0-3197e3a340a6: added peer, starting replication: peer=server-605f64b3-042b-6803-b535-fcd5a8f2a878 -2025-10-27T04:03:51.172Z [INFO] server-cea66c26-5244-1b09-65f0-3197e3a340a6: added peer, starting replication: peer=server-8d428cc5-bd17-dcac-4589-61725287082b -2025-10-27T04:03:51.172Z [INFO] server-cea66c26-5244-1b09-65f0-3197e3a340a6: pipelining replication: peer="{Voter server-8d428cc5-bd17-dcac-4589-61725287082b 8d428cc5-bd17-dcac-4589-61725287082b}" -2025-10-27T04:03:51.172Z [INFO] server-cea66c26-5244-1b09-65f0-3197e3a340a6: pipelining replication: peer="{Voter server-605f64b3-042b-6803-b535-fcd5a8f2a878 605f64b3-042b-6803-b535-fcd5a8f2a878}" -2025-10-27T04:03:51.280Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:51.17260429 +0000 UTC m=+106.167476728 (1 nodes), 86.830553ms from start of poll, 87.213316ms from cluster start. Timeout at 2025-10-27 04:03:51.27763357 +0000 UTC m=+106.272506008, 105.02928ms after stability +2026-11-29T10:29:53.417Z [INFO] server-4d913f16-244d-39fd-4d75-c9273159ba86: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d913f16-244d-39fd-4d75-c9273159ba86 Address:4d913f16-244d-39fd-4d75-c9273159ba86} {Suffrage:Voter ID:server-28cc72bb-0964-8a9d-6afe-5bb09ba86f4c Address:28cc72bb-0964-8a9d-6afe-5bb09ba86f4c} {Suffrage:Voter ID:server-46661b6b-16ab-a510-3be6-9333fd377993 Address:46661b6b-16ab-a510-3be6-9333fd377993}]" +2026-11-29T10:29:53.417Z [INFO] server-28cc72bb-0964-8a9d-6afe-5bb09ba86f4c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d913f16-244d-39fd-4d75-c9273159ba86 Address:4d913f16-244d-39fd-4d75-c9273159ba86} {Suffrage:Voter ID:server-28cc72bb-0964-8a9d-6afe-5bb09ba86f4c Address:28cc72bb-0964-8a9d-6afe-5bb09ba86f4c} {Suffrage:Voter ID:server-46661b6b-16ab-a510-3be6-9333fd377993 Address:46661b6b-16ab-a510-3be6-9333fd377993}]" +2026-11-29T10:29:53.417Z [INFO] server-4d913f16-244d-39fd-4d75-c9273159ba86: entering follower state: follower="Node at 4d913f16-244d-39fd-4d75-c9273159ba86 [Follower]" leader-address= leader-id= +2026-11-29T10:29:53.417Z [INFO] server-46661b6b-16ab-a510-3be6-9333fd377993: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d913f16-244d-39fd-4d75-c9273159ba86 Address:4d913f16-244d-39fd-4d75-c9273159ba86} {Suffrage:Voter ID:server-28cc72bb-0964-8a9d-6afe-5bb09ba86f4c Address:28cc72bb-0964-8a9d-6afe-5bb09ba86f4c} {Suffrage:Voter ID:server-46661b6b-16ab-a510-3be6-9333fd377993 Address:46661b6b-16ab-a510-3be6-9333fd377993}]" +2026-11-29T10:29:53.417Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:53.417Z [INFO] server-28cc72bb-0964-8a9d-6afe-5bb09ba86f4c: entering follower state: follower="Node at 28cc72bb-0964-8a9d-6afe-5bb09ba86f4c [Follower]" leader-address= leader-id= +2026-11-29T10:29:53.417Z [INFO] server-46661b6b-16ab-a510-3be6-9333fd377993: entering follower state: follower="Node at 46661b6b-16ab-a510-3be6-9333fd377993 [Follower]" leader-address= leader-id= +2026-11-29T10:29:53.476Z [WARN] server-28cc72bb-0964-8a9d-6afe-5bb09ba86f4c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:53.476Z [INFO] server-28cc72bb-0964-8a9d-6afe-5bb09ba86f4c: entering candidate state: node="Node at 28cc72bb-0964-8a9d-6afe-5bb09ba86f4c [Candidate]" term=2 +2026-11-29T10:29:53.476Z [INFO] server-28cc72bb-0964-8a9d-6afe-5bb09ba86f4c: election won: term=2 tally=2 +2026-11-29T10:29:53.476Z [INFO] server-28cc72bb-0964-8a9d-6afe-5bb09ba86f4c: entering leader state: leader="Node at 28cc72bb-0964-8a9d-6afe-5bb09ba86f4c [Leader]" +2026-11-29T10:29:53.476Z [INFO] server-28cc72bb-0964-8a9d-6afe-5bb09ba86f4c: added peer, starting replication: peer=server-4d913f16-244d-39fd-4d75-c9273159ba86 +2026-11-29T10:29:53.476Z [INFO] server-28cc72bb-0964-8a9d-6afe-5bb09ba86f4c: added peer, starting replication: peer=server-46661b6b-16ab-a510-3be6-9333fd377993 +2026-11-29T10:29:53.476Z [INFO] server-28cc72bb-0964-8a9d-6afe-5bb09ba86f4c: pipelining replication: peer="{Voter server-46661b6b-16ab-a510-3be6-9333fd377993 46661b6b-16ab-a510-3be6-9333fd377993}" +2026-11-29T10:29:53.476Z [INFO] server-28cc72bb-0964-8a9d-6afe-5bb09ba86f4c: pipelining replication: peer="{Voter server-4d913f16-244d-39fd-4d75-c9273159ba86 4d913f16-244d-39fd-4d75-c9273159ba86}" +2026-11-29T10:29:53.582Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:53.476671939 +0000 UTC m=+104.211726041 (1 nodes), 58.73064ms from start of poll, 58.92084ms from cluster start. Timeout at 2026-11-29 10:29:53.581674539 +0000 UTC m=+104.316728601, 105.00256ms after stability raft_test.go:2457: waiting on future 0 raft_test.go:2457: waiting on future 1 raft_test.go:2457: waiting on future 2 @@ -3867,351 +3894,338 @@ raft_test.go:2457: waiting on future 4 raft_test.go:2457: waiting on future 5 raft_test.go:2457: waiting on future 6 -2025-10-27T04:03:51.280Z [INFO] server-cea66c26-5244-1b09-65f0-3197e3a340a6: aborting pipeline replication: peer="{Voter server-8d428cc5-bd17-dcac-4589-61725287082b 8d428cc5-bd17-dcac-4589-61725287082b}" -2025-10-27T04:03:51.280Z [INFO] server-cea66c26-5244-1b09-65f0-3197e3a340a6: aborting pipeline replication: peer="{Voter server-605f64b3-042b-6803-b535-fcd5a8f2a878 605f64b3-042b-6803-b535-fcd5a8f2a878}" -2025-10-27T04:03:51.781Z [ERROR] server-cea66c26-5244-1b09-65f0-3197e3a340a6: failed to heartbeat to: peer=605f64b3-042b-6803-b535-fcd5a8f2a878 backoff time=10ms error="command timed out" ---- PASS: TestRaft_LeadershipTransferLeaderRejectsClientRequests (0.70s) +2026-11-29T10:29:53.582Z [INFO] server-28cc72bb-0964-8a9d-6afe-5bb09ba86f4c: aborting pipeline replication: peer="{Voter server-46661b6b-16ab-a510-3be6-9333fd377993 46661b6b-16ab-a510-3be6-9333fd377993}" +2026-11-29T10:29:53.582Z [INFO] server-28cc72bb-0964-8a9d-6afe-5bb09ba86f4c: aborting pipeline replication: peer="{Voter server-4d913f16-244d-39fd-4d75-c9273159ba86 4d913f16-244d-39fd-4d75-c9273159ba86}" +--- PASS: TestRaft_LeadershipTransferLeaderRejectsClientRequests (0.17s) === RUN TestRaft_LeadershipTransferLeaderReplicationTimeout -2025-10-27T04:03:51.781Z [INFO] server-237a718e-ed12-92a3-cbcf-87e682693368: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-237a718e-ed12-92a3-cbcf-87e682693368 Address:237a718e-ed12-92a3-cbcf-87e682693368} {Suffrage:Voter ID:server-f762deab-e62c-1d53-12fb-b1489567a159 Address:f762deab-e62c-1d53-12fb-b1489567a159} {Suffrage:Voter ID:server-7667158a-0153-bc75-6844-7a6be5378237 Address:7667158a-0153-bc75-6844-7a6be5378237}]" -2025-10-27T04:03:51.781Z [INFO] server-f762deab-e62c-1d53-12fb-b1489567a159: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-237a718e-ed12-92a3-cbcf-87e682693368 Address:237a718e-ed12-92a3-cbcf-87e682693368} {Suffrage:Voter ID:server-f762deab-e62c-1d53-12fb-b1489567a159 Address:f762deab-e62c-1d53-12fb-b1489567a159} {Suffrage:Voter ID:server-7667158a-0153-bc75-6844-7a6be5378237 Address:7667158a-0153-bc75-6844-7a6be5378237}]" -2025-10-27T04:03:51.781Z [INFO] server-7667158a-0153-bc75-6844-7a6be5378237: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-237a718e-ed12-92a3-cbcf-87e682693368 Address:237a718e-ed12-92a3-cbcf-87e682693368} {Suffrage:Voter ID:server-f762deab-e62c-1d53-12fb-b1489567a159 Address:f762deab-e62c-1d53-12fb-b1489567a159} {Suffrage:Voter ID:server-7667158a-0153-bc75-6844-7a6be5378237 Address:7667158a-0153-bc75-6844-7a6be5378237}]" -2025-10-27T04:03:51.781Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:51.781Z [INFO] server-237a718e-ed12-92a3-cbcf-87e682693368: entering follower state: follower="Node at 237a718e-ed12-92a3-cbcf-87e682693368 [Follower]" leader-address= leader-id= -2025-10-27T04:03:51.781Z [INFO] server-7667158a-0153-bc75-6844-7a6be5378237: entering follower state: follower="Node at 7667158a-0153-bc75-6844-7a6be5378237 [Follower]" leader-address= leader-id= -2025-10-27T04:03:51.781Z [INFO] server-f762deab-e62c-1d53-12fb-b1489567a159: entering follower state: follower="Node at f762deab-e62c-1d53-12fb-b1489567a159 [Follower]" leader-address= leader-id= -2025-10-27T04:03:51.837Z [WARN] server-7667158a-0153-bc75-6844-7a6be5378237: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:51.837Z [INFO] server-7667158a-0153-bc75-6844-7a6be5378237: entering candidate state: node="Node at 7667158a-0153-bc75-6844-7a6be5378237 [Candidate]" term=2 -2025-10-27T04:03:51.837Z [INFO] server-7667158a-0153-bc75-6844-7a6be5378237: election won: term=2 tally=2 -2025-10-27T04:03:51.837Z [INFO] server-7667158a-0153-bc75-6844-7a6be5378237: entering leader state: leader="Node at 7667158a-0153-bc75-6844-7a6be5378237 [Leader]" -2025-10-27T04:03:51.837Z [INFO] server-7667158a-0153-bc75-6844-7a6be5378237: added peer, starting replication: peer=server-237a718e-ed12-92a3-cbcf-87e682693368 -2025-10-27T04:03:51.837Z [INFO] server-7667158a-0153-bc75-6844-7a6be5378237: added peer, starting replication: peer=server-f762deab-e62c-1d53-12fb-b1489567a159 -2025-10-27T04:03:51.837Z [INFO] server-7667158a-0153-bc75-6844-7a6be5378237: pipelining replication: peer="{Voter server-f762deab-e62c-1d53-12fb-b1489567a159 f762deab-e62c-1d53-12fb-b1489567a159}" -2025-10-27T04:03:51.837Z [INFO] server-7667158a-0153-bc75-6844-7a6be5378237: pipelining replication: peer="{Voter server-237a718e-ed12-92a3-cbcf-87e682693368 237a718e-ed12-92a3-cbcf-87e682693368}" -2025-10-27T04:03:51.942Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:51.837741194 +0000 UTC m=+106.832613592 (1 nodes), 55.821461ms from start of poll, 56.014103ms from cluster start. Timeout at 2025-10-27 04:03:51.942744074 +0000 UTC m=+106.937616512, 105.00292ms after stability -2025-10-27T04:03:51.942Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:03:52.048Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:03:51.942853592 +0000 UTC m=+106.937725990 (2 nodes), 1.56µs from start of poll, 161.126501ms from cluster start. Timeout at 2025-10-27 04:03:52.047884272 +0000 UTC m=+107.042756750, 105.03076ms after stability -2025-10-27T04:03:52.350Z [INFO] server-7667158a-0153-bc75-6844-7a6be5378237: aborting pipeline replication: peer="{Voter server-f762deab-e62c-1d53-12fb-b1489567a159 f762deab-e62c-1d53-12fb-b1489567a159}" -2025-10-27T04:03:52.350Z [INFO] server-7667158a-0153-bc75-6844-7a6be5378237: aborting pipeline replication: peer="{Voter server-237a718e-ed12-92a3-cbcf-87e682693368 237a718e-ed12-92a3-cbcf-87e682693368}" ---- PASS: TestRaft_LeadershipTransferLeaderReplicationTimeout (0.57s) +2026-11-29T10:29:53.583Z [INFO] server-ddcd0353-47a5-d5f8-e958-496dd3b3b350: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ddcd0353-47a5-d5f8-e958-496dd3b3b350 Address:ddcd0353-47a5-d5f8-e958-496dd3b3b350} {Suffrage:Voter ID:server-dc78f283-dfec-feb5-8de3-c02483cdc2e8 Address:dc78f283-dfec-feb5-8de3-c02483cdc2e8} {Suffrage:Voter ID:server-4a018ade-9d07-c0d7-573c-39180d0f951e Address:4a018ade-9d07-c0d7-573c-39180d0f951e}]" +2026-11-29T10:29:53.583Z [INFO] server-dc78f283-dfec-feb5-8de3-c02483cdc2e8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ddcd0353-47a5-d5f8-e958-496dd3b3b350 Address:ddcd0353-47a5-d5f8-e958-496dd3b3b350} {Suffrage:Voter ID:server-dc78f283-dfec-feb5-8de3-c02483cdc2e8 Address:dc78f283-dfec-feb5-8de3-c02483cdc2e8} {Suffrage:Voter ID:server-4a018ade-9d07-c0d7-573c-39180d0f951e Address:4a018ade-9d07-c0d7-573c-39180d0f951e}]" +2026-11-29T10:29:53.583Z [INFO] server-ddcd0353-47a5-d5f8-e958-496dd3b3b350: entering follower state: follower="Node at ddcd0353-47a5-d5f8-e958-496dd3b3b350 [Follower]" leader-address= leader-id= +2026-11-29T10:29:53.583Z [INFO] server-4a018ade-9d07-c0d7-573c-39180d0f951e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ddcd0353-47a5-d5f8-e958-496dd3b3b350 Address:ddcd0353-47a5-d5f8-e958-496dd3b3b350} {Suffrage:Voter ID:server-dc78f283-dfec-feb5-8de3-c02483cdc2e8 Address:dc78f283-dfec-feb5-8de3-c02483cdc2e8} {Suffrage:Voter ID:server-4a018ade-9d07-c0d7-573c-39180d0f951e Address:4a018ade-9d07-c0d7-573c-39180d0f951e}]" +2026-11-29T10:29:53.583Z [INFO] server-dc78f283-dfec-feb5-8de3-c02483cdc2e8: entering follower state: follower="Node at dc78f283-dfec-feb5-8de3-c02483cdc2e8 [Follower]" leader-address= leader-id= +2026-11-29T10:29:53.583Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:53.583Z [INFO] server-4a018ade-9d07-c0d7-573c-39180d0f951e: entering follower state: follower="Node at 4a018ade-9d07-c0d7-573c-39180d0f951e [Follower]" leader-address= leader-id= +2026-11-29T10:29:53.639Z [WARN] server-dc78f283-dfec-feb5-8de3-c02483cdc2e8: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:53.639Z [INFO] server-dc78f283-dfec-feb5-8de3-c02483cdc2e8: entering candidate state: node="Node at dc78f283-dfec-feb5-8de3-c02483cdc2e8 [Candidate]" term=2 +2026-11-29T10:29:53.639Z [INFO] server-dc78f283-dfec-feb5-8de3-c02483cdc2e8: election won: term=2 tally=2 +2026-11-29T10:29:53.639Z [INFO] server-dc78f283-dfec-feb5-8de3-c02483cdc2e8: entering leader state: leader="Node at dc78f283-dfec-feb5-8de3-c02483cdc2e8 [Leader]" +2026-11-29T10:29:53.639Z [INFO] server-dc78f283-dfec-feb5-8de3-c02483cdc2e8: added peer, starting replication: peer=server-ddcd0353-47a5-d5f8-e958-496dd3b3b350 +2026-11-29T10:29:53.639Z [INFO] server-dc78f283-dfec-feb5-8de3-c02483cdc2e8: added peer, starting replication: peer=server-4a018ade-9d07-c0d7-573c-39180d0f951e +2026-11-29T10:29:53.639Z [INFO] server-dc78f283-dfec-feb5-8de3-c02483cdc2e8: pipelining replication: peer="{Voter server-4a018ade-9d07-c0d7-573c-39180d0f951e 4a018ade-9d07-c0d7-573c-39180d0f951e}" +2026-11-29T10:29:53.639Z [INFO] server-dc78f283-dfec-feb5-8de3-c02483cdc2e8: pipelining replication: peer="{Voter server-ddcd0353-47a5-d5f8-e958-496dd3b3b350 ddcd0353-47a5-d5f8-e958-496dd3b3b350}" +2026-11-29T10:29:53.745Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:53.639924859 +0000 UTC m=+104.374978961 (1 nodes), 56.45412ms from start of poll, 56.672ms from cluster start. Timeout at 2026-11-29 10:29:53.744926459 +0000 UTC m=+104.479980601, 105.00164ms after stability +2026-11-29T10:29:53.745Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:53.850Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:53.745412779 +0000 UTC m=+104.480466841 (2 nodes), 1.76µs from start of poll, 162.15988ms from cluster start. Timeout at 2026-11-29 10:29:53.850435499 +0000 UTC m=+104.585489601, 105.02276ms after stability +2026-11-29T10:29:54.136Z [INFO] server-dc78f283-dfec-feb5-8de3-c02483cdc2e8: aborting pipeline replication: peer="{Voter server-4a018ade-9d07-c0d7-573c-39180d0f951e 4a018ade-9d07-c0d7-573c-39180d0f951e}" +2026-11-29T10:29:54.136Z [INFO] server-dc78f283-dfec-feb5-8de3-c02483cdc2e8: aborting pipeline replication: peer="{Voter server-ddcd0353-47a5-d5f8-e958-496dd3b3b350 ddcd0353-47a5-d5f8-e958-496dd3b3b350}" +--- PASS: TestRaft_LeadershipTransferLeaderReplicationTimeout (0.55s) === RUN TestRaft_LeadershipTransferIgnoresNonvoters -2025-10-27T04:03:52.350Z [INFO] server-41c8857e-a918-8987-89c9-eefbad8aa686: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-41c8857e-a918-8987-89c9-eefbad8aa686 Address:41c8857e-a918-8987-89c9-eefbad8aa686} {Suffrage:Voter ID:server-4463f3ba-2a99-93f1-8612-907745219eb9 Address:4463f3ba-2a99-93f1-8612-907745219eb9}]" -2025-10-27T04:03:52.350Z [INFO] server-4463f3ba-2a99-93f1-8612-907745219eb9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-41c8857e-a918-8987-89c9-eefbad8aa686 Address:41c8857e-a918-8987-89c9-eefbad8aa686} {Suffrage:Voter ID:server-4463f3ba-2a99-93f1-8612-907745219eb9 Address:4463f3ba-2a99-93f1-8612-907745219eb9}]" -2025-10-27T04:03:52.350Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:03:52.350Z [INFO] server-41c8857e-a918-8987-89c9-eefbad8aa686: entering follower state: follower="Node at 41c8857e-a918-8987-89c9-eefbad8aa686 [Follower]" leader-address= leader-id= -2025-10-27T04:03:52.350Z [INFO] server-4463f3ba-2a99-93f1-8612-907745219eb9: entering follower state: follower="Node at 4463f3ba-2a99-93f1-8612-907745219eb9 [Follower]" leader-address= leader-id= -2025-10-27T04:03:52.401Z [WARN] server-4463f3ba-2a99-93f1-8612-907745219eb9: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:52.401Z [INFO] server-4463f3ba-2a99-93f1-8612-907745219eb9: entering candidate state: node="Node at 4463f3ba-2a99-93f1-8612-907745219eb9 [Candidate]" term=2 -2025-10-27T04:03:52.401Z [INFO] server-4463f3ba-2a99-93f1-8612-907745219eb9: election won: term=2 tally=2 -2025-10-27T04:03:52.401Z [INFO] server-4463f3ba-2a99-93f1-8612-907745219eb9: entering leader state: leader="Node at 4463f3ba-2a99-93f1-8612-907745219eb9 [Leader]" -2025-10-27T04:03:52.401Z [INFO] server-4463f3ba-2a99-93f1-8612-907745219eb9: added peer, starting replication: peer=server-41c8857e-a918-8987-89c9-eefbad8aa686 -2025-10-27T04:03:52.401Z [INFO] server-4463f3ba-2a99-93f1-8612-907745219eb9: pipelining replication: peer="{Voter server-41c8857e-a918-8987-89c9-eefbad8aa686 41c8857e-a918-8987-89c9-eefbad8aa686}" -2025-10-27T04:03:52.507Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:03:52.401599926 +0000 UTC m=+107.396472324 (1 nodes), 50.768947ms from start of poll, 50.924348ms from cluster start. Timeout at 2025-10-27 04:03:52.506602566 +0000 UTC m=+107.501475044, 105.00272ms after stability -2025-10-27T04:03:52.507Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:52.612Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:52.507709011 +0000 UTC m=+107.502581489 (1 nodes), 760ns from start of poll, 157.033513ms from cluster start. Timeout at 2025-10-27 04:03:52.612792572 +0000 UTC m=+107.607665010, 105.083521ms after stability -2025-10-27T04:03:52.613Z [INFO] server-4463f3ba-2a99-93f1-8612-907745219eb9: updating configuration: command=DemoteVoter server-id=server-41c8857e-a918-8987-89c9-eefbad8aa686 server-addr= servers="[{Suffrage:Nonvoter ID:server-41c8857e-a918-8987-89c9-eefbad8aa686 Address:41c8857e-a918-8987-89c9-eefbad8aa686} {Suffrage:Voter ID:server-4463f3ba-2a99-93f1-8612-907745219eb9 Address:4463f3ba-2a99-93f1-8612-907745219eb9}]" -2025-10-27T04:03:52.613Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:52.719Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:52.613115251 +0000 UTC m=+107.607987729 (1 nodes), 600ns from start of poll, 262.439753ms from cluster start. Timeout at 2025-10-27 04:03:52.718202251 +0000 UTC m=+107.713074729, 105.087ms after stability -2025-10-27T04:03:52.719Z [INFO] server-4463f3ba-2a99-93f1-8612-907745219eb9: aborting pipeline replication: peer="{Voter server-41c8857e-a918-8987-89c9-eefbad8aa686 41c8857e-a918-8987-89c9-eefbad8aa686}" ---- PASS: TestRaft_LeadershipTransferIgnoresNonvoters (0.37s) +2026-11-29T10:29:54.137Z [INFO] server-61d851a2-fbe2-5f01-e129-28051fa7233c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-61d851a2-fbe2-5f01-e129-28051fa7233c Address:61d851a2-fbe2-5f01-e129-28051fa7233c} {Suffrage:Voter ID:server-2df782e9-904d-1257-6875-5df377013054 Address:2df782e9-904d-1257-6875-5df377013054}]" +2026-11-29T10:29:54.137Z [INFO] server-2df782e9-904d-1257-6875-5df377013054: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-61d851a2-fbe2-5f01-e129-28051fa7233c Address:61d851a2-fbe2-5f01-e129-28051fa7233c} {Suffrage:Voter ID:server-2df782e9-904d-1257-6875-5df377013054 Address:2df782e9-904d-1257-6875-5df377013054}]" +2026-11-29T10:29:54.137Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:54.137Z [INFO] server-61d851a2-fbe2-5f01-e129-28051fa7233c: entering follower state: follower="Node at 61d851a2-fbe2-5f01-e129-28051fa7233c [Follower]" leader-address= leader-id= +2026-11-29T10:29:54.137Z [INFO] server-2df782e9-904d-1257-6875-5df377013054: entering follower state: follower="Node at 2df782e9-904d-1257-6875-5df377013054 [Follower]" leader-address= leader-id= +2026-11-29T10:29:54.192Z [WARN] server-2df782e9-904d-1257-6875-5df377013054: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:54.192Z [INFO] server-2df782e9-904d-1257-6875-5df377013054: entering candidate state: node="Node at 2df782e9-904d-1257-6875-5df377013054 [Candidate]" term=2 +2026-11-29T10:29:54.192Z [INFO] server-2df782e9-904d-1257-6875-5df377013054: election won: term=2 tally=2 +2026-11-29T10:29:54.192Z [INFO] server-2df782e9-904d-1257-6875-5df377013054: entering leader state: leader="Node at 2df782e9-904d-1257-6875-5df377013054 [Leader]" +2026-11-29T10:29:54.192Z [INFO] server-2df782e9-904d-1257-6875-5df377013054: added peer, starting replication: peer=server-61d851a2-fbe2-5f01-e129-28051fa7233c +2026-11-29T10:29:54.192Z [INFO] server-2df782e9-904d-1257-6875-5df377013054: pipelining replication: peer="{Voter server-61d851a2-fbe2-5f01-e129-28051fa7233c 61d851a2-fbe2-5f01-e129-28051fa7233c}" +2026-11-29T10:29:54.298Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:54.192343179 +0000 UTC m=+104.927397281 (1 nodes), 55.07036ms from start of poll, 55.28596ms from cluster start. Timeout at 2026-11-29 10:29:54.297344859 +0000 UTC m=+105.032398961, 105.00168ms after stability +2026-11-29T10:29:54.298Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:54.404Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:54.298527459 +0000 UTC m=+105.033581561 (1 nodes), 480ns from start of poll, 161.47024ms from cluster start. Timeout at 2026-11-29 10:29:54.403638939 +0000 UTC m=+105.138693001, 105.11144ms after stability +2026-11-29T10:29:54.404Z [INFO] server-2df782e9-904d-1257-6875-5df377013054: updating configuration: command=DemoteVoter server-id=server-61d851a2-fbe2-5f01-e129-28051fa7233c server-addr= servers="[{Suffrage:Nonvoter ID:server-61d851a2-fbe2-5f01-e129-28051fa7233c Address:61d851a2-fbe2-5f01-e129-28051fa7233c} {Suffrage:Voter ID:server-2df782e9-904d-1257-6875-5df377013054 Address:2df782e9-904d-1257-6875-5df377013054}]" +2026-11-29T10:29:54.404Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:54.512Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:54.404148819 +0000 UTC m=+105.139202881 (1 nodes), 440ns from start of poll, 267.09156ms from cluster start. Timeout at 2026-11-29 10:29:54.509221019 +0000 UTC m=+105.244275121, 105.07224ms after stability +2026-11-29T10:29:54.516Z [INFO] server-2df782e9-904d-1257-6875-5df377013054: aborting pipeline replication: peer="{Voter server-61d851a2-fbe2-5f01-e129-28051fa7233c 61d851a2-fbe2-5f01-e129-28051fa7233c}" +--- PASS: TestRaft_LeadershipTransferIgnoresNonvoters (0.38s) === RUN TestRaft_LeadershipTransferStopRightAway --- PASS: TestRaft_LeadershipTransferStopRightAway (0.00s) === RUN TestRaft_GetConfigurationNoBootstrap -2025-10-27T04:03:52.719Z [INFO] server-dfe8f483-2830-c9e9-5a26-cfc2e7a678f2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dfe8f483-2830-c9e9-5a26-cfc2e7a678f2 Address:dfe8f483-2830-c9e9-5a26-cfc2e7a678f2} {Suffrage:Voter ID:server-b313e384-3f30-78eb-1811-8d2e5244f8fc Address:b313e384-3f30-78eb-1811-8d2e5244f8fc}]" -2025-10-27T04:03:52.719Z [INFO] server-b313e384-3f30-78eb-1811-8d2e5244f8fc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dfe8f483-2830-c9e9-5a26-cfc2e7a678f2 Address:dfe8f483-2830-c9e9-5a26-cfc2e7a678f2} {Suffrage:Voter ID:server-b313e384-3f30-78eb-1811-8d2e5244f8fc Address:b313e384-3f30-78eb-1811-8d2e5244f8fc}]" -2025-10-27T04:03:52.719Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:03:52.719Z [INFO] server-dfe8f483-2830-c9e9-5a26-cfc2e7a678f2: entering follower state: follower="Node at dfe8f483-2830-c9e9-5a26-cfc2e7a678f2 [Follower]" leader-address= leader-id= -2025-10-27T04:03:52.719Z [INFO] server-b313e384-3f30-78eb-1811-8d2e5244f8fc: entering follower state: follower="Node at b313e384-3f30-78eb-1811-8d2e5244f8fc [Follower]" leader-address= leader-id= -2025-10-27T04:03:52.793Z [WARN] server-b313e384-3f30-78eb-1811-8d2e5244f8fc: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:52.793Z [INFO] server-b313e384-3f30-78eb-1811-8d2e5244f8fc: entering candidate state: node="Node at b313e384-3f30-78eb-1811-8d2e5244f8fc [Candidate]" term=2 -2025-10-27T04:03:52.793Z [INFO] server-b313e384-3f30-78eb-1811-8d2e5244f8fc: election won: term=2 tally=2 -2025-10-27T04:03:52.793Z [INFO] server-b313e384-3f30-78eb-1811-8d2e5244f8fc: entering leader state: leader="Node at b313e384-3f30-78eb-1811-8d2e5244f8fc [Leader]" -2025-10-27T04:03:52.793Z [INFO] server-b313e384-3f30-78eb-1811-8d2e5244f8fc: added peer, starting replication: peer=server-dfe8f483-2830-c9e9-5a26-cfc2e7a678f2 -2025-10-27T04:03:52.794Z [INFO] server-b313e384-3f30-78eb-1811-8d2e5244f8fc: pipelining replication: peer="{Voter server-dfe8f483-2830-c9e9-5a26-cfc2e7a678f2 dfe8f483-2830-c9e9-5a26-cfc2e7a678f2}" -2025-10-27T04:03:52.899Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:03:52.793978367 +0000 UTC m=+107.788850845 (1 nodes), 74.207467ms from start of poll, 74.341868ms from cluster start. Timeout at 2025-10-27 04:03:52.898981887 +0000 UTC m=+107.893854365, 105.00352ms after stability -2025-10-27T04:03:52.899Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:53.004Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:52.899380887 +0000 UTC m=+107.894253325 (1 nodes), 560ns from start of poll, 179.744348ms from cluster start. Timeout at 2025-10-27 04:03:53.004410967 +0000 UTC m=+107.999283445, 105.03012ms after stability -2025-10-27T04:03:53.016Z [INFO] server-b313e384-3f30-78eb-1811-8d2e5244f8fc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dfe8f483-2830-c9e9-5a26-cfc2e7a678f2 Address:dfe8f483-2830-c9e9-5a26-cfc2e7a678f2} {Suffrage:Voter ID:server-b313e384-3f30-78eb-1811-8d2e5244f8fc Address:b313e384-3f30-78eb-1811-8d2e5244f8fc}]" -2025-10-27T04:03:53.016Z [INFO] server-b313e384-3f30-78eb-1811-8d2e5244f8fc: aborting pipeline replication: peer="{Voter server-dfe8f483-2830-c9e9-5a26-cfc2e7a678f2 dfe8f483-2830-c9e9-5a26-cfc2e7a678f2}" +2026-11-29T10:29:54.517Z [INFO] server-74c90b29-344e-6d34-8171-28038e790278: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-74c90b29-344e-6d34-8171-28038e790278 Address:74c90b29-344e-6d34-8171-28038e790278} {Suffrage:Voter ID:server-e9e138bb-c836-87cf-09ec-e3cf2154599e Address:e9e138bb-c836-87cf-09ec-e3cf2154599e}]" +2026-11-29T10:29:54.517Z [INFO] server-e9e138bb-c836-87cf-09ec-e3cf2154599e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-74c90b29-344e-6d34-8171-28038e790278 Address:74c90b29-344e-6d34-8171-28038e790278} {Suffrage:Voter ID:server-e9e138bb-c836-87cf-09ec-e3cf2154599e Address:e9e138bb-c836-87cf-09ec-e3cf2154599e}]" +2026-11-29T10:29:54.517Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:54.517Z [INFO] server-74c90b29-344e-6d34-8171-28038e790278: entering follower state: follower="Node at 74c90b29-344e-6d34-8171-28038e790278 [Follower]" leader-address= leader-id= +2026-11-29T10:29:54.517Z [INFO] server-e9e138bb-c836-87cf-09ec-e3cf2154599e: entering follower state: follower="Node at e9e138bb-c836-87cf-09ec-e3cf2154599e [Follower]" leader-address= leader-id= +2026-11-29T10:29:54.584Z [WARN] server-74c90b29-344e-6d34-8171-28038e790278: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:54.584Z [INFO] server-74c90b29-344e-6d34-8171-28038e790278: entering candidate state: node="Node at 74c90b29-344e-6d34-8171-28038e790278 [Candidate]" term=2 +2026-11-29T10:29:54.584Z [INFO] server-74c90b29-344e-6d34-8171-28038e790278: election won: term=2 tally=2 +2026-11-29T10:29:54.584Z [INFO] server-74c90b29-344e-6d34-8171-28038e790278: entering leader state: leader="Node at 74c90b29-344e-6d34-8171-28038e790278 [Leader]" +2026-11-29T10:29:54.584Z [INFO] server-74c90b29-344e-6d34-8171-28038e790278: added peer, starting replication: peer=server-e9e138bb-c836-87cf-09ec-e3cf2154599e +2026-11-29T10:29:54.584Z [INFO] server-74c90b29-344e-6d34-8171-28038e790278: pipelining replication: peer="{Voter server-e9e138bb-c836-87cf-09ec-e3cf2154599e e9e138bb-c836-87cf-09ec-e3cf2154599e}" +2026-11-29T10:29:54.690Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:54.584726979 +0000 UTC m=+105.319781081 (1 nodes), 67.55568ms from start of poll, 67.71036ms from cluster start. Timeout at 2026-11-29 10:29:54.689728619 +0000 UTC m=+105.424782721, 105.00164ms after stability +2026-11-29T10:29:54.690Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:54.796Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:54.690816339 +0000 UTC m=+105.425870441 (1 nodes), 1.48µs from start of poll, 173.79972ms from cluster start. Timeout at 2026-11-29 10:29:54.795845739 +0000 UTC m=+105.530899841, 105.0294ms after stability +2026-11-29T10:29:54.808Z [INFO] server-e9e138bb-c836-87cf-09ec-e3cf2154599e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-74c90b29-344e-6d34-8171-28038e790278 Address:74c90b29-344e-6d34-8171-28038e790278} {Suffrage:Voter ID:server-e9e138bb-c836-87cf-09ec-e3cf2154599e Address:e9e138bb-c836-87cf-09ec-e3cf2154599e}]" +2026-11-29T10:29:54.814Z [INFO] server-74c90b29-344e-6d34-8171-28038e790278: aborting pipeline replication: peer="{Voter server-e9e138bb-c836-87cf-09ec-e3cf2154599e e9e138bb-c836-87cf-09ec-e3cf2154599e}" --- PASS: TestRaft_GetConfigurationNoBootstrap (0.30s) === RUN TestRaft_LogStoreIsMonotonic -2025-10-27T04:03:53.017Z [INFO] server-152b8f43-d20d-7fc8-02f7-b4d5e9dd64ea: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-152b8f43-d20d-7fc8-02f7-b4d5e9dd64ea Address:152b8f43-d20d-7fc8-02f7-b4d5e9dd64ea}]" -2025-10-27T04:03:53.017Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:53.017Z [INFO] server-152b8f43-d20d-7fc8-02f7-b4d5e9dd64ea: entering follower state: follower="Node at 152b8f43-d20d-7fc8-02f7-b4d5e9dd64ea [Follower]" leader-address= leader-id= -2025-10-27T04:03:53.107Z [WARN] server-152b8f43-d20d-7fc8-02f7-b4d5e9dd64ea: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:53.107Z [INFO] server-152b8f43-d20d-7fc8-02f7-b4d5e9dd64ea: entering candidate state: node="Node at 152b8f43-d20d-7fc8-02f7-b4d5e9dd64ea [Candidate]" term=2 -2025-10-27T04:03:53.107Z [INFO] server-152b8f43-d20d-7fc8-02f7-b4d5e9dd64ea: election won: term=2 tally=1 -2025-10-27T04:03:53.107Z [INFO] server-152b8f43-d20d-7fc8-02f7-b4d5e9dd64ea: entering leader state: leader="Node at 152b8f43-d20d-7fc8-02f7-b4d5e9dd64ea [Leader]" -2025-10-27T04:03:53.213Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:53.107872831 +0000 UTC m=+108.102745309 (1 nodes), 90.77678ms from start of poll, 90.890101ms from cluster start. Timeout at 2025-10-27 04:03:53.212880831 +0000 UTC m=+108.207753269, 105.00796ms after stability ---- PASS: TestRaft_LogStoreIsMonotonic (0.20s) +2026-11-29T10:29:54.815Z [INFO] server-4616c23c-12ae-01ae-a7f5-cf6f70619c16: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4616c23c-12ae-01ae-a7f5-cf6f70619c16 Address:4616c23c-12ae-01ae-a7f5-cf6f70619c16}]" +2026-11-29T10:29:54.815Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:54.815Z [INFO] server-4616c23c-12ae-01ae-a7f5-cf6f70619c16: entering follower state: follower="Node at 4616c23c-12ae-01ae-a7f5-cf6f70619c16 [Follower]" leader-address= leader-id= +2026-11-29T10:29:54.866Z [WARN] server-4616c23c-12ae-01ae-a7f5-cf6f70619c16: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:54.867Z [INFO] server-4616c23c-12ae-01ae-a7f5-cf6f70619c16: entering candidate state: node="Node at 4616c23c-12ae-01ae-a7f5-cf6f70619c16 [Candidate]" term=2 +2026-11-29T10:29:54.867Z [INFO] server-4616c23c-12ae-01ae-a7f5-cf6f70619c16: election won: term=2 tally=1 +2026-11-29T10:29:54.867Z [INFO] server-4616c23c-12ae-01ae-a7f5-cf6f70619c16: entering leader state: leader="Node at 4616c23c-12ae-01ae-a7f5-cf6f70619c16 [Leader]" +2026-11-29T10:29:54.972Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:54.867310019 +0000 UTC m=+105.602364081 (1 nodes), 52.03348ms from start of poll, 52.11448ms from cluster start. Timeout at 2026-11-29 10:29:54.972317539 +0000 UTC m=+105.707371641, 105.00756ms after stability +--- PASS: TestRaft_LogStoreIsMonotonic (0.16s) === RUN TestRaft_CacheLogWithStoreError -2025-10-27T04:03:53.213Z [INFO] server-9bc806c7-121c-59f3-3f63-babd0664d1eb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9bc806c7-121c-59f3-3f63-babd0664d1eb Address:9bc806c7-121c-59f3-3f63-babd0664d1eb} {Suffrage:Voter ID:server-eb49aead-a230-c93d-02f2-550bf799be1c Address:eb49aead-a230-c93d-02f2-550bf799be1c}]" -2025-10-27T04:03:53.213Z [INFO] server-eb49aead-a230-c93d-02f2-550bf799be1c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9bc806c7-121c-59f3-3f63-babd0664d1eb Address:9bc806c7-121c-59f3-3f63-babd0664d1eb} {Suffrage:Voter ID:server-eb49aead-a230-c93d-02f2-550bf799be1c Address:eb49aead-a230-c93d-02f2-550bf799be1c}]" -2025-10-27T04:03:53.213Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:03:53.213Z [INFO] server-9bc806c7-121c-59f3-3f63-babd0664d1eb: entering follower state: follower="Node at 9bc806c7-121c-59f3-3f63-babd0664d1eb [Follower]" leader-address= leader-id= -2025-10-27T04:03:53.213Z [INFO] server-eb49aead-a230-c93d-02f2-550bf799be1c: entering follower state: follower="Node at eb49aead-a230-c93d-02f2-550bf799be1c [Follower]" leader-address= leader-id= -2025-10-27T04:03:53.279Z [WARN] server-eb49aead-a230-c93d-02f2-550bf799be1c: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:53.279Z [INFO] server-eb49aead-a230-c93d-02f2-550bf799be1c: entering candidate state: node="Node at eb49aead-a230-c93d-02f2-550bf799be1c [Candidate]" term=2 -2025-10-27T04:03:53.279Z [INFO] server-eb49aead-a230-c93d-02f2-550bf799be1c: election won: term=2 tally=2 -2025-10-27T04:03:53.279Z [INFO] server-eb49aead-a230-c93d-02f2-550bf799be1c: entering leader state: leader="Node at eb49aead-a230-c93d-02f2-550bf799be1c [Leader]" -2025-10-27T04:03:53.279Z [INFO] server-eb49aead-a230-c93d-02f2-550bf799be1c: added peer, starting replication: peer=server-9bc806c7-121c-59f3-3f63-babd0664d1eb -2025-10-27T04:03:53.279Z [INFO] server-eb49aead-a230-c93d-02f2-550bf799be1c: pipelining replication: peer="{Voter server-9bc806c7-121c-59f3-3f63-babd0664d1eb 9bc806c7-121c-59f3-3f63-babd0664d1eb}" -2025-10-27T04:03:53.385Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:03:53.279654084 +0000 UTC m=+108.274526602 (1 nodes), 66.104171ms from start of poll, 66.241492ms from cluster start. Timeout at 2025-10-27 04:03:53.384665284 +0000 UTC m=+108.379537762, 105.01116ms after stability -2025-10-27T04:03:53.385Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:53.491Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:53.385565928 +0000 UTC m=+108.380438406 (1 nodes), 1.92µs from start of poll, 172.153296ms from cluster start. Timeout at 2025-10-27 04:03:53.490605048 +0000 UTC m=+108.485477526, 105.03912ms after stability -2025-10-27T04:03:53.491Z [ERROR] server-9bc806c7-121c-59f3-3f63-babd0664d1eb: failed to append to logs: error="unable to store logs within log store, err: "some error"" -2025-10-27T04:03:53.491Z [WARN] server-9bc806c7-121c-59f3-3f63-babd0664d1eb: failed to get previous log: previous-index=9 last-index=7 error="log not found" -2025-10-27T04:03:53.491Z [INFO] server-eb49aead-a230-c93d-02f2-550bf799be1c: aborting pipeline replication: peer="{Voter server-9bc806c7-121c-59f3-3f63-babd0664d1eb 9bc806c7-121c-59f3-3f63-babd0664d1eb}" -2025-10-27T04:03:53.497Z [INFO] server-eb49aead-a230-c93d-02f2-550bf799be1c: pipelining replication: peer="{Voter server-9bc806c7-121c-59f3-3f63-babd0664d1eb 9bc806c7-121c-59f3-3f63-babd0664d1eb}" -2025-10-27T04:03:53.497Z [INFO] server-9bc806c7-121c-59f3-3f63-babd0664d1eb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9bc806c7-121c-59f3-3f63-babd0664d1eb Address:9bc806c7-121c-59f3-3f63-babd0664d1eb} {Suffrage:Voter ID:server-eb49aead-a230-c93d-02f2-550bf799be1c Address:eb49aead-a230-c93d-02f2-550bf799be1c}]" -2025-10-27T04:03:53.497Z [INFO] server-eb49aead-a230-c93d-02f2-550bf799be1c: aborting pipeline replication: peer="{Voter server-9bc806c7-121c-59f3-3f63-babd0664d1eb 9bc806c7-121c-59f3-3f63-babd0664d1eb}" ---- PASS: TestRaft_CacheLogWithStoreError (0.28s) +2026-11-29T10:29:54.973Z [INFO] server-df7b27de-fe2e-1631-238d-4243ecb152e3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-df7b27de-fe2e-1631-238d-4243ecb152e3 Address:df7b27de-fe2e-1631-238d-4243ecb152e3} {Suffrage:Voter ID:server-3e0c8b25-822c-88e8-0f0c-65d7f7733506 Address:3e0c8b25-822c-88e8-0f0c-65d7f7733506}]" +2026-11-29T10:29:54.973Z [INFO] server-3e0c8b25-822c-88e8-0f0c-65d7f7733506: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-df7b27de-fe2e-1631-238d-4243ecb152e3 Address:df7b27de-fe2e-1631-238d-4243ecb152e3} {Suffrage:Voter ID:server-3e0c8b25-822c-88e8-0f0c-65d7f7733506 Address:3e0c8b25-822c-88e8-0f0c-65d7f7733506}]" +2026-11-29T10:29:54.973Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:54.973Z [INFO] server-df7b27de-fe2e-1631-238d-4243ecb152e3: entering follower state: follower="Node at df7b27de-fe2e-1631-238d-4243ecb152e3 [Follower]" leader-address= leader-id= +2026-11-29T10:29:54.973Z [INFO] server-3e0c8b25-822c-88e8-0f0c-65d7f7733506: entering follower state: follower="Node at 3e0c8b25-822c-88e8-0f0c-65d7f7733506 [Follower]" leader-address= leader-id= +2026-11-29T10:29:55.042Z [WARN] server-3e0c8b25-822c-88e8-0f0c-65d7f7733506: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:55.042Z [INFO] server-3e0c8b25-822c-88e8-0f0c-65d7f7733506: entering candidate state: node="Node at 3e0c8b25-822c-88e8-0f0c-65d7f7733506 [Candidate]" term=2 +2026-11-29T10:29:55.042Z [INFO] server-3e0c8b25-822c-88e8-0f0c-65d7f7733506: election won: term=2 tally=2 +2026-11-29T10:29:55.042Z [INFO] server-3e0c8b25-822c-88e8-0f0c-65d7f7733506: entering leader state: leader="Node at 3e0c8b25-822c-88e8-0f0c-65d7f7733506 [Leader]" +2026-11-29T10:29:55.042Z [INFO] server-3e0c8b25-822c-88e8-0f0c-65d7f7733506: added peer, starting replication: peer=server-df7b27de-fe2e-1631-238d-4243ecb152e3 +2026-11-29T10:29:55.042Z [INFO] server-3e0c8b25-822c-88e8-0f0c-65d7f7733506: pipelining replication: peer="{Voter server-df7b27de-fe2e-1631-238d-4243ecb152e3 df7b27de-fe2e-1631-238d-4243ecb152e3}" +2026-11-29T10:29:55.148Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:55.042899019 +0000 UTC m=+105.777953081 (1 nodes), 69.79052ms from start of poll, 69.91368ms from cluster start. Timeout at 2026-11-29 10:29:55.147901299 +0000 UTC m=+105.882955401, 105.00232ms after stability +2026-11-29T10:29:55.148Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:55.254Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:55.148919939 +0000 UTC m=+105.883974001 (1 nodes), 480ns from start of poll, 175.9346ms from cluster start. Timeout at 2026-11-29 10:29:55.253943699 +0000 UTC m=+105.988997761, 105.02376ms after stability +2026-11-29T10:29:55.254Z [ERROR] server-df7b27de-fe2e-1631-238d-4243ecb152e3: failed to append to logs: error="unable to store logs within log store, err: "some error"" +2026-11-29T10:29:55.255Z [WARN] server-df7b27de-fe2e-1631-238d-4243ecb152e3: failed to get previous log: previous-index=9 last-index=7 error="log not found" +2026-11-29T10:29:55.255Z [INFO] server-3e0c8b25-822c-88e8-0f0c-65d7f7733506: aborting pipeline replication: peer="{Voter server-df7b27de-fe2e-1631-238d-4243ecb152e3 df7b27de-fe2e-1631-238d-4243ecb152e3}" +2026-11-29T10:29:55.263Z [INFO] server-3e0c8b25-822c-88e8-0f0c-65d7f7733506: pipelining replication: peer="{Voter server-df7b27de-fe2e-1631-238d-4243ecb152e3 df7b27de-fe2e-1631-238d-4243ecb152e3}" +2026-11-29T10:29:55.264Z [INFO] server-df7b27de-fe2e-1631-238d-4243ecb152e3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-df7b27de-fe2e-1631-238d-4243ecb152e3 Address:df7b27de-fe2e-1631-238d-4243ecb152e3} {Suffrage:Voter ID:server-3e0c8b25-822c-88e8-0f0c-65d7f7733506 Address:3e0c8b25-822c-88e8-0f0c-65d7f7733506}]" +2026-11-29T10:29:55.264Z [INFO] server-3e0c8b25-822c-88e8-0f0c-65d7f7733506: aborting pipeline replication: peer="{Voter server-df7b27de-fe2e-1631-238d-4243ecb152e3 df7b27de-fe2e-1631-238d-4243ecb152e3}" +--- PASS: TestRaft_CacheLogWithStoreError (0.29s) === RUN TestRaft_ReloadConfig -2025-10-27T04:03:53.497Z [INFO] server-728191ad-55ab-278d-3b40-84c88438dc9f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-728191ad-55ab-278d-3b40-84c88438dc9f Address:728191ad-55ab-278d-3b40-84c88438dc9f}]" +2026-11-29T10:29:55.264Z [INFO] server-0c28dbcd-a3bd-2cb9-d43a-ec87bda792c2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c28dbcd-a3bd-2cb9-d43a-ec87bda792c2 Address:0c28dbcd-a3bd-2cb9-d43a-ec87bda792c2}]" --- PASS: TestRaft_ReloadConfig (0.00s) === RUN TestRaft_ReloadConfigValidates -2025-10-27T04:03:53.498Z [INFO] server-33fb16b5-1bf1-b77c-347e-27a8ea40403d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-33fb16b5-1bf1-b77c-347e-27a8ea40403d Address:33fb16b5-1bf1-b77c-347e-27a8ea40403d}]" -2025-10-27T04:03:53.498Z [INFO] server-33fb16b5-1bf1-b77c-347e-27a8ea40403d: entering follower state: follower="Node at 33fb16b5-1bf1-b77c-347e-27a8ea40403d [Shutdown]" leader-address= leader-id= +2026-11-29T10:29:55.266Z [INFO] server-4d8595aa-63b7-382b-435d-202536a70c9d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d8595aa-63b7-382b-435d-202536a70c9d Address:4d8595aa-63b7-382b-435d-202536a70c9d}]" +2026-11-29T10:29:55.266Z [INFO] server-4d8595aa-63b7-382b-435d-202536a70c9d: entering follower state: follower="Node at 4d8595aa-63b7-382b-435d-202536a70c9d [Follower]" leader-address= leader-id= --- PASS: TestRaft_ReloadConfigValidates (0.00s) === RUN TestRaft_InstallSnapshot_InvalidPeers -2025-10-27T04:03:53.499Z [ERROR] failed to install snapshot: error="failed to decode peers: codec.decoder: Only encoded map or array can be decoded into a slice. (valueType: 2)" +2026-11-29T10:29:55.266Z [ERROR] failed to install snapshot: error="failed to decode peers: codec.decoder: Only encoded map or array can be decoded into a slice. (valueType: 2)" --- PASS: TestRaft_InstallSnapshot_InvalidPeers (0.00s) === RUN TestRaft_VoteNotGranted_WhenNodeNotInCluster -2025-10-27T04:03:53.501Z [INFO] server-51da218f-720c-62c9-16cf-3f0c9bcad8e2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-51da218f-720c-62c9-16cf-3f0c9bcad8e2 Address:51da218f-720c-62c9-16cf-3f0c9bcad8e2} {Suffrage:Voter ID:server-6670bb43-77eb-995d-b8cb-837df7694d10 Address:6670bb43-77eb-995d-b8cb-837df7694d10} {Suffrage:Voter ID:server-91940e75-2c78-87ff-3a2c-add0460ee0b4 Address:91940e75-2c78-87ff-3a2c-add0460ee0b4}]" -2025-10-27T04:03:53.501Z [INFO] server-51da218f-720c-62c9-16cf-3f0c9bcad8e2: entering follower state: follower="Node at 51da218f-720c-62c9-16cf-3f0c9bcad8e2 [Follower]" leader-address= leader-id= -2025-10-27T04:03:53.501Z [INFO] server-6670bb43-77eb-995d-b8cb-837df7694d10: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-51da218f-720c-62c9-16cf-3f0c9bcad8e2 Address:51da218f-720c-62c9-16cf-3f0c9bcad8e2} {Suffrage:Voter ID:server-6670bb43-77eb-995d-b8cb-837df7694d10 Address:6670bb43-77eb-995d-b8cb-837df7694d10} {Suffrage:Voter ID:server-91940e75-2c78-87ff-3a2c-add0460ee0b4 Address:91940e75-2c78-87ff-3a2c-add0460ee0b4}]" -2025-10-27T04:03:53.501Z [INFO] server-6670bb43-77eb-995d-b8cb-837df7694d10: entering follower state: follower="Node at 6670bb43-77eb-995d-b8cb-837df7694d10 [Follower]" leader-address= leader-id= -2025-10-27T04:03:53.501Z [INFO] server-91940e75-2c78-87ff-3a2c-add0460ee0b4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-51da218f-720c-62c9-16cf-3f0c9bcad8e2 Address:51da218f-720c-62c9-16cf-3f0c9bcad8e2} {Suffrage:Voter ID:server-6670bb43-77eb-995d-b8cb-837df7694d10 Address:6670bb43-77eb-995d-b8cb-837df7694d10} {Suffrage:Voter ID:server-91940e75-2c78-87ff-3a2c-add0460ee0b4 Address:91940e75-2c78-87ff-3a2c-add0460ee0b4}]" -2025-10-27T04:03:53.501Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:53.501Z [INFO] server-91940e75-2c78-87ff-3a2c-add0460ee0b4: entering follower state: follower="Node at 91940e75-2c78-87ff-3a2c-add0460ee0b4 [Follower]" leader-address= leader-id= -2025-10-27T04:03:53.577Z [WARN] server-91940e75-2c78-87ff-3a2c-add0460ee0b4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:53.577Z [INFO] server-91940e75-2c78-87ff-3a2c-add0460ee0b4: entering candidate state: node="Node at 91940e75-2c78-87ff-3a2c-add0460ee0b4 [Candidate]" term=2 -2025-10-27T04:03:53.577Z [INFO] server-91940e75-2c78-87ff-3a2c-add0460ee0b4: election won: term=2 tally=2 -2025-10-27T04:03:53.577Z [INFO] server-91940e75-2c78-87ff-3a2c-add0460ee0b4: entering leader state: leader="Node at 91940e75-2c78-87ff-3a2c-add0460ee0b4 [Leader]" -2025-10-27T04:03:53.578Z [INFO] server-91940e75-2c78-87ff-3a2c-add0460ee0b4: added peer, starting replication: peer=server-51da218f-720c-62c9-16cf-3f0c9bcad8e2 -2025-10-27T04:03:53.578Z [INFO] server-91940e75-2c78-87ff-3a2c-add0460ee0b4: added peer, starting replication: peer=server-6670bb43-77eb-995d-b8cb-837df7694d10 -2025-10-27T04:03:53.578Z [INFO] server-91940e75-2c78-87ff-3a2c-add0460ee0b4: pipelining replication: peer="{Voter server-6670bb43-77eb-995d-b8cb-837df7694d10 6670bb43-77eb-995d-b8cb-837df7694d10}" -2025-10-27T04:03:53.578Z [INFO] server-91940e75-2c78-87ff-3a2c-add0460ee0b4: pipelining replication: peer="{Voter server-51da218f-720c-62c9-16cf-3f0c9bcad8e2 51da218f-720c-62c9-16cf-3f0c9bcad8e2}" -2025-10-27T04:03:53.684Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:53.578359685 +0000 UTC m=+108.573232163 (1 nodes), 76.519483ms from start of poll, 76.969726ms from cluster start. Timeout at 2025-10-27 04:03:53.683363405 +0000 UTC m=+108.678235843, 105.00368ms after stability -2025-10-27T04:03:53.690Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:03:53.796Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:03:53.690482291 +0000 UTC m=+108.685354729 (2 nodes), 880ns from start of poll, 189.092292ms from cluster start. Timeout at 2025-10-27 04:03:53.795515931 +0000 UTC m=+108.790388409, 105.03368ms after stability -2025-10-27T04:03:53.796Z [INFO] server-91940e75-2c78-87ff-3a2c-add0460ee0b4: updating configuration: command=RemoveServer server-id=server-51da218f-720c-62c9-16cf-3f0c9bcad8e2 server-addr= servers="[{Suffrage:Voter ID:server-6670bb43-77eb-995d-b8cb-837df7694d10 Address:6670bb43-77eb-995d-b8cb-837df7694d10} {Suffrage:Voter ID:server-91940e75-2c78-87ff-3a2c-add0460ee0b4 Address:91940e75-2c78-87ff-3a2c-add0460ee0b4}]" -2025-10-27T04:03:53.796Z [INFO] server-91940e75-2c78-87ff-3a2c-add0460ee0b4: removed peer, stopping replication: peer=server-51da218f-720c-62c9-16cf-3f0c9bcad8e2 last-index=3 -2025-10-27T04:03:53.796Z [INFO] server-91940e75-2c78-87ff-3a2c-add0460ee0b4: aborting pipeline replication: peer="{Voter server-51da218f-720c-62c9-16cf-3f0c9bcad8e2 51da218f-720c-62c9-16cf-3f0c9bcad8e2}" -2025-10-27T04:03:53.903Z [ERROR] server-91940e75-2c78-87ff-3a2c-add0460ee0b4: failed to pipeline appendEntries: peer="{Voter server-6670bb43-77eb-995d-b8cb-837df7694d10 6670bb43-77eb-995d-b8cb-837df7694d10}" error="append pipeline closed" -2025-10-27T04:03:53.903Z [INFO] server-91940e75-2c78-87ff-3a2c-add0460ee0b4: aborting pipeline replication: peer="{Voter server-6670bb43-77eb-995d-b8cb-837df7694d10 6670bb43-77eb-995d-b8cb-837df7694d10}" -2025-10-27T04:03:53.907Z [ERROR] server-91940e75-2c78-87ff-3a2c-add0460ee0b4: failed to heartbeat to: peer=6670bb43-77eb-995d-b8cb-837df7694d10 backoff time=10ms error="failed to connect to peer: 6670bb43-77eb-995d-b8cb-837df7694d10" -2025-10-27T04:03:53.911Z [ERROR] server-91940e75-2c78-87ff-3a2c-add0460ee0b4: failed to appendEntries to: peer="{Voter server-6670bb43-77eb-995d-b8cb-837df7694d10 6670bb43-77eb-995d-b8cb-837df7694d10}" error="failed to connect to peer: 6670bb43-77eb-995d-b8cb-837df7694d10" -2025-10-27T04:03:53.926Z [ERROR] server-91940e75-2c78-87ff-3a2c-add0460ee0b4: failed to heartbeat to: peer=6670bb43-77eb-995d-b8cb-837df7694d10 backoff time=10ms error="failed to connect to peer: 6670bb43-77eb-995d-b8cb-837df7694d10" -2025-10-27T04:03:53.930Z [WARN] server-51da218f-720c-62c9-16cf-3f0c9bcad8e2: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election -2025-10-27T04:03:53.931Z [ERROR] server-91940e75-2c78-87ff-3a2c-add0460ee0b4: failed to appendEntries to: peer="{Voter server-6670bb43-77eb-995d-b8cb-837df7694d10 6670bb43-77eb-995d-b8cb-837df7694d10}" error="failed to connect to peer: 6670bb43-77eb-995d-b8cb-837df7694d10" -2025-10-27T04:03:53.947Z [ERROR] server-91940e75-2c78-87ff-3a2c-add0460ee0b4: failed to heartbeat to: peer=6670bb43-77eb-995d-b8cb-837df7694d10 backoff time=10ms error="failed to connect to peer: 6670bb43-77eb-995d-b8cb-837df7694d10" -2025-10-27T04:03:53.948Z [WARN] server-91940e75-2c78-87ff-3a2c-add0460ee0b4: failed to contact: server-id=server-6670bb43-77eb-995d-b8cb-837df7694d10 time=50.431264ms -2025-10-27T04:03:53.948Z [WARN] server-91940e75-2c78-87ff-3a2c-add0460ee0b4: failed to contact quorum of nodes, stepping down -2025-10-27T04:03:53.948Z [INFO] server-91940e75-2c78-87ff-3a2c-add0460ee0b4: entering follower state: follower="Node at 91940e75-2c78-87ff-3a2c-add0460ee0b4 [Follower]" leader-address= leader-id= -2025-10-27T04:03:53.951Z [ERROR] server-91940e75-2c78-87ff-3a2c-add0460ee0b4: failed to appendEntries to: peer="{Voter server-6670bb43-77eb-995d-b8cb-837df7694d10 6670bb43-77eb-995d-b8cb-837df7694d10}" error="failed to connect to peer: 6670bb43-77eb-995d-b8cb-837df7694d10" -2025-10-27T04:03:54.009Z [WARN] server-91940e75-2c78-87ff-3a2c-add0460ee0b4: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:54.009Z [INFO] server-91940e75-2c78-87ff-3a2c-add0460ee0b4: entering candidate state: node="Node at 91940e75-2c78-87ff-3a2c-add0460ee0b4 [Candidate]" term=3 -2025-10-27T04:03:54.009Z [ERROR] server-91940e75-2c78-87ff-3a2c-add0460ee0b4: failed to make requestVote RPC: target="{Voter server-6670bb43-77eb-995d-b8cb-837df7694d10 6670bb43-77eb-995d-b8cb-837df7694d10}" error="failed to connect to peer: 6670bb43-77eb-995d-b8cb-837df7694d10" term=3 -2025-10-27T04:03:54.023Z [WARN] server-6670bb43-77eb-995d-b8cb-837df7694d10: heartbeat timeout reached, starting election: last-leader-addr=91940e75-2c78-87ff-3a2c-add0460ee0b4 last-leader-id=server-91940e75-2c78-87ff-3a2c-add0460ee0b4 -2025-10-27T04:03:54.023Z [INFO] server-6670bb43-77eb-995d-b8cb-837df7694d10: entering candidate state: node="Node at 6670bb43-77eb-995d-b8cb-837df7694d10 [Candidate]" term=3 -2025-10-27T04:03:54.023Z [ERROR] server-6670bb43-77eb-995d-b8cb-837df7694d10: failed to make requestVote RPC: target="{Voter server-91940e75-2c78-87ff-3a2c-add0460ee0b4 91940e75-2c78-87ff-3a2c-add0460ee0b4}" error="failed to connect to peer: 91940e75-2c78-87ff-3a2c-add0460ee0b4" term=3 -2025-10-27T04:03:54.024Z [WARN] server-6670bb43-77eb-995d-b8cb-837df7694d10: rejecting vote request since node is not in configuration: from=51da218f-720c-62c9-16cf-3f0c9bcad8e2 ---- PASS: TestRaft_VoteNotGranted_WhenNodeNotInCluster (0.52s) +2026-11-29T10:29:55.266Z [INFO] server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e Address:3db14fbf-c107-ebb8-4aca-b1844b6ffb8e} {Suffrage:Voter ID:server-5ec89fb6-a37c-2ec9-7d3a-5fd53ab41a22 Address:5ec89fb6-a37c-2ec9-7d3a-5fd53ab41a22} {Suffrage:Voter ID:server-c19da5a8-53a8-1168-5b3f-e3ce06e1e413 Address:c19da5a8-53a8-1168-5b3f-e3ce06e1e413}]" +2026-11-29T10:29:55.266Z [INFO] server-5ec89fb6-a37c-2ec9-7d3a-5fd53ab41a22: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e Address:3db14fbf-c107-ebb8-4aca-b1844b6ffb8e} {Suffrage:Voter ID:server-5ec89fb6-a37c-2ec9-7d3a-5fd53ab41a22 Address:5ec89fb6-a37c-2ec9-7d3a-5fd53ab41a22} {Suffrage:Voter ID:server-c19da5a8-53a8-1168-5b3f-e3ce06e1e413 Address:c19da5a8-53a8-1168-5b3f-e3ce06e1e413}]" +2026-11-29T10:29:55.266Z [INFO] server-c19da5a8-53a8-1168-5b3f-e3ce06e1e413: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e Address:3db14fbf-c107-ebb8-4aca-b1844b6ffb8e} {Suffrage:Voter ID:server-5ec89fb6-a37c-2ec9-7d3a-5fd53ab41a22 Address:5ec89fb6-a37c-2ec9-7d3a-5fd53ab41a22} {Suffrage:Voter ID:server-c19da5a8-53a8-1168-5b3f-e3ce06e1e413 Address:c19da5a8-53a8-1168-5b3f-e3ce06e1e413}]" +2026-11-29T10:29:55.266Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:55.266Z [INFO] server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e: entering follower state: follower="Node at 3db14fbf-c107-ebb8-4aca-b1844b6ffb8e [Follower]" leader-address= leader-id= +2026-11-29T10:29:55.267Z [INFO] server-5ec89fb6-a37c-2ec9-7d3a-5fd53ab41a22: entering follower state: follower="Node at 5ec89fb6-a37c-2ec9-7d3a-5fd53ab41a22 [Follower]" leader-address= leader-id= +2026-11-29T10:29:55.267Z [INFO] server-c19da5a8-53a8-1168-5b3f-e3ce06e1e413: entering follower state: follower="Node at c19da5a8-53a8-1168-5b3f-e3ce06e1e413 [Follower]" leader-address= leader-id= +2026-11-29T10:29:55.331Z [WARN] server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:55.331Z [INFO] server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e: entering candidate state: node="Node at 3db14fbf-c107-ebb8-4aca-b1844b6ffb8e [Candidate]" term=2 +2026-11-29T10:29:55.331Z [INFO] server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e: election won: term=2 tally=2 +2026-11-29T10:29:55.331Z [INFO] server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e: entering leader state: leader="Node at 3db14fbf-c107-ebb8-4aca-b1844b6ffb8e [Leader]" +2026-11-29T10:29:55.331Z [INFO] server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e: added peer, starting replication: peer=server-5ec89fb6-a37c-2ec9-7d3a-5fd53ab41a22 +2026-11-29T10:29:55.331Z [INFO] server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e: added peer, starting replication: peer=server-c19da5a8-53a8-1168-5b3f-e3ce06e1e413 +2026-11-29T10:29:55.331Z [INFO] server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e: pipelining replication: peer="{Voter server-c19da5a8-53a8-1168-5b3f-e3ce06e1e413 c19da5a8-53a8-1168-5b3f-e3ce06e1e413}" +2026-11-29T10:29:55.332Z [INFO] server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e: pipelining replication: peer="{Voter server-5ec89fb6-a37c-2ec9-7d3a-5fd53ab41a22 5ec89fb6-a37c-2ec9-7d3a-5fd53ab41a22}" +2026-11-29T10:29:55.437Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:55.332014939 +0000 UTC m=+106.067069041 (1 nodes), 65.04256ms from start of poll, 65.22008ms from cluster start. Timeout at 2026-11-29 10:29:55.437022059 +0000 UTC m=+106.172076161, 105.00712ms after stability +2026-11-29T10:29:55.443Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:55.550Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:55.444856419 +0000 UTC m=+106.179910521 (2 nodes), 2.24µs from start of poll, 178.06156ms from cluster start. Timeout at 2026-11-29 10:29:55.549905179 +0000 UTC m=+106.284959281, 105.04876ms after stability +2026-11-29T10:29:55.550Z [INFO] server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e: updating configuration: command=RemoveServer server-id=server-5ec89fb6-a37c-2ec9-7d3a-5fd53ab41a22 server-addr= servers="[{Suffrage:Voter ID:server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e Address:3db14fbf-c107-ebb8-4aca-b1844b6ffb8e} {Suffrage:Voter ID:server-c19da5a8-53a8-1168-5b3f-e3ce06e1e413 Address:c19da5a8-53a8-1168-5b3f-e3ce06e1e413}]" +2026-11-29T10:29:55.550Z [INFO] server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e: removed peer, stopping replication: peer=server-5ec89fb6-a37c-2ec9-7d3a-5fd53ab41a22 last-index=3 +2026-11-29T10:29:55.550Z [INFO] server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e: aborting pipeline replication: peer="{Voter server-5ec89fb6-a37c-2ec9-7d3a-5fd53ab41a22 5ec89fb6-a37c-2ec9-7d3a-5fd53ab41a22}" +2026-11-29T10:29:55.608Z [WARN] server-5ec89fb6-a37c-2ec9-7d3a-5fd53ab41a22: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +2026-11-29T10:29:55.655Z [ERROR] server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e: failed to heartbeat to: peer=c19da5a8-53a8-1168-5b3f-e3ce06e1e413 backoff time=10ms error="failed to connect to peer: c19da5a8-53a8-1168-5b3f-e3ce06e1e413" +2026-11-29T10:29:55.659Z [ERROR] server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e: failed to pipeline appendEntries: peer="{Voter server-c19da5a8-53a8-1168-5b3f-e3ce06e1e413 c19da5a8-53a8-1168-5b3f-e3ce06e1e413}" error="append pipeline closed" +2026-11-29T10:29:55.659Z [INFO] server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e: aborting pipeline replication: peer="{Voter server-c19da5a8-53a8-1168-5b3f-e3ce06e1e413 c19da5a8-53a8-1168-5b3f-e3ce06e1e413}" +2026-11-29T10:29:55.667Z [ERROR] server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e: failed to appendEntries to: peer="{Voter server-c19da5a8-53a8-1168-5b3f-e3ce06e1e413 c19da5a8-53a8-1168-5b3f-e3ce06e1e413}" error="failed to connect to peer: c19da5a8-53a8-1168-5b3f-e3ce06e1e413" +2026-11-29T10:29:55.676Z [ERROR] server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e: failed to heartbeat to: peer=c19da5a8-53a8-1168-5b3f-e3ce06e1e413 backoff time=10ms error="failed to connect to peer: c19da5a8-53a8-1168-5b3f-e3ce06e1e413" +2026-11-29T10:29:55.684Z [ERROR] server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e: failed to appendEntries to: peer="{Voter server-c19da5a8-53a8-1168-5b3f-e3ce06e1e413 c19da5a8-53a8-1168-5b3f-e3ce06e1e413}" error="failed to connect to peer: c19da5a8-53a8-1168-5b3f-e3ce06e1e413" +2026-11-29T10:29:55.699Z [ERROR] server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e: failed to heartbeat to: peer=c19da5a8-53a8-1168-5b3f-e3ce06e1e413 backoff time=10ms error="failed to connect to peer: c19da5a8-53a8-1168-5b3f-e3ce06e1e413" +2026-11-29T10:29:55.701Z [WARN] server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e: failed to contact: server-id=server-c19da5a8-53a8-1168-5b3f-e3ce06e1e413 time=51.00224ms +2026-11-29T10:29:55.701Z [WARN] server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e: failed to contact quorum of nodes, stepping down +2026-11-29T10:29:55.701Z [INFO] server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e: entering follower state: follower="Node at 3db14fbf-c107-ebb8-4aca-b1844b6ffb8e [Follower]" leader-address= leader-id= +2026-11-29T10:29:55.704Z [ERROR] server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e: failed to appendEntries to: peer="{Voter server-c19da5a8-53a8-1168-5b3f-e3ce06e1e413 c19da5a8-53a8-1168-5b3f-e3ce06e1e413}" error="failed to connect to peer: c19da5a8-53a8-1168-5b3f-e3ce06e1e413" +2026-11-29T10:29:55.743Z [WARN] server-c19da5a8-53a8-1168-5b3f-e3ce06e1e413: heartbeat timeout reached, starting election: last-leader-addr=3db14fbf-c107-ebb8-4aca-b1844b6ffb8e last-leader-id=server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e +2026-11-29T10:29:55.743Z [INFO] server-c19da5a8-53a8-1168-5b3f-e3ce06e1e413: entering candidate state: node="Node at c19da5a8-53a8-1168-5b3f-e3ce06e1e413 [Candidate]" term=3 +2026-11-29T10:29:55.743Z [ERROR] server-c19da5a8-53a8-1168-5b3f-e3ce06e1e413: failed to make requestVote RPC: target="{Voter server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e 3db14fbf-c107-ebb8-4aca-b1844b6ffb8e}" error="failed to connect to peer: 3db14fbf-c107-ebb8-4aca-b1844b6ffb8e" term=3 +2026-11-29T10:29:55.758Z [WARN] server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:55.758Z [INFO] server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e: entering candidate state: node="Node at 3db14fbf-c107-ebb8-4aca-b1844b6ffb8e [Candidate]" term=3 +2026-11-29T10:29:55.758Z [ERROR] server-3db14fbf-c107-ebb8-4aca-b1844b6ffb8e: failed to make requestVote RPC: target="{Voter server-c19da5a8-53a8-1168-5b3f-e3ce06e1e413 c19da5a8-53a8-1168-5b3f-e3ce06e1e413}" error="failed to connect to peer: c19da5a8-53a8-1168-5b3f-e3ce06e1e413" term=3 +2026-11-29T10:29:55.761Z [WARN] server-c19da5a8-53a8-1168-5b3f-e3ce06e1e413: rejecting vote request since node is not in configuration: from=5ec89fb6-a37c-2ec9-7d3a-5fd53ab41a22 +--- PASS: TestRaft_VoteNotGranted_WhenNodeNotInCluster (0.50s) === RUN TestRaft_ClusterCanRegainStability_WhenNonVoterWithHigherTermJoin -2025-10-27T04:03:54.025Z [INFO] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-53c5210a-3d2f-ce5e-bec6-75e607d95553 Address:53c5210a-3d2f-ce5e-bec6-75e607d95553} {Suffrage:Voter ID:server-24d56434-ace1-1167-83c6-d6928dea4983 Address:24d56434-ace1-1167-83c6-d6928dea4983} {Suffrage:Voter ID:server-2df62e77-9658-6d7d-7c21-5e94f57526d3 Address:2df62e77-9658-6d7d-7c21-5e94f57526d3}]" -2025-10-27T04:03:54.025Z [INFO] server-24d56434-ace1-1167-83c6-d6928dea4983: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-53c5210a-3d2f-ce5e-bec6-75e607d95553 Address:53c5210a-3d2f-ce5e-bec6-75e607d95553} {Suffrage:Voter ID:server-24d56434-ace1-1167-83c6-d6928dea4983 Address:24d56434-ace1-1167-83c6-d6928dea4983} {Suffrage:Voter ID:server-2df62e77-9658-6d7d-7c21-5e94f57526d3 Address:2df62e77-9658-6d7d-7c21-5e94f57526d3}]" -2025-10-27T04:03:54.025Z [INFO] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: entering follower state: follower="Node at 53c5210a-3d2f-ce5e-bec6-75e607d95553 [Follower]" leader-address= leader-id= -2025-10-27T04:03:54.025Z [INFO] server-24d56434-ace1-1167-83c6-d6928dea4983: entering follower state: follower="Node at 24d56434-ace1-1167-83c6-d6928dea4983 [Follower]" leader-address= leader-id= -2025-10-27T04:03:54.026Z [INFO] server-2df62e77-9658-6d7d-7c21-5e94f57526d3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-53c5210a-3d2f-ce5e-bec6-75e607d95553 Address:53c5210a-3d2f-ce5e-bec6-75e607d95553} {Suffrage:Voter ID:server-24d56434-ace1-1167-83c6-d6928dea4983 Address:24d56434-ace1-1167-83c6-d6928dea4983} {Suffrage:Voter ID:server-2df62e77-9658-6d7d-7c21-5e94f57526d3 Address:2df62e77-9658-6d7d-7c21-5e94f57526d3}]" -2025-10-27T04:03:54.026Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:54.026Z [INFO] server-2df62e77-9658-6d7d-7c21-5e94f57526d3: entering follower state: follower="Node at 2df62e77-9658-6d7d-7c21-5e94f57526d3 [Follower]" leader-address= leader-id= -2025-10-27T04:03:54.080Z [WARN] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:54.080Z [INFO] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: entering candidate state: node="Node at 53c5210a-3d2f-ce5e-bec6-75e607d95553 [Candidate]" term=2 -2025-10-27T04:03:54.080Z [INFO] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: election won: term=2 tally=2 -2025-10-27T04:03:54.080Z [INFO] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: entering leader state: leader="Node at 53c5210a-3d2f-ce5e-bec6-75e607d95553 [Leader]" -2025-10-27T04:03:54.080Z [INFO] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: added peer, starting replication: peer=server-24d56434-ace1-1167-83c6-d6928dea4983 -2025-10-27T04:03:54.080Z [INFO] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: added peer, starting replication: peer=server-2df62e77-9658-6d7d-7c21-5e94f57526d3 -2025-10-27T04:03:54.080Z [INFO] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: pipelining replication: peer="{Voter server-24d56434-ace1-1167-83c6-d6928dea4983 24d56434-ace1-1167-83c6-d6928dea4983}" -2025-10-27T04:03:54.080Z [INFO] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: pipelining replication: peer="{Voter server-2df62e77-9658-6d7d-7c21-5e94f57526d3 2df62e77-9658-6d7d-7c21-5e94f57526d3}" -2025-10-27T04:03:54.185Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:54.080334914 +0000 UTC m=+109.075207432 (1 nodes), 54.25049ms from start of poll, 54.671413ms from cluster start. Timeout at 2025-10-27 04:03:54.185336674 +0000 UTC m=+109.180209152, 105.00172ms after stability -2025-10-27T04:03:54.190Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:03:54.295Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:03:54.190501787 +0000 UTC m=+109.185374225 (2 nodes), 1.96µs from start of poll, 164.838206ms from cluster start. Timeout at 2025-10-27 04:03:54.295517507 +0000 UTC m=+109.290389945, 105.01572ms after stability -2025-10-27T04:03:54.296Z [ERROR] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: failed to heartbeat to: peer=24d56434-ace1-1167-83c6-d6928dea4983 backoff time=10ms error="failed to connect to peer: 24d56434-ace1-1167-83c6-d6928dea4983" -2025-10-27T04:03:54.297Z [ERROR] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: failed to pipeline appendEntries: peer="{Voter server-24d56434-ace1-1167-83c6-d6928dea4983 24d56434-ace1-1167-83c6-d6928dea4983}" error="append pipeline closed" -2025-10-27T04:03:54.297Z [INFO] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: aborting pipeline replication: peer="{Voter server-24d56434-ace1-1167-83c6-d6928dea4983 24d56434-ace1-1167-83c6-d6928dea4983}" -2025-10-27T04:03:54.305Z [ERROR] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: failed to appendEntries to: peer="{Voter server-24d56434-ace1-1167-83c6-d6928dea4983 24d56434-ace1-1167-83c6-d6928dea4983}" error="failed to connect to peer: 24d56434-ace1-1167-83c6-d6928dea4983" -2025-10-27T04:03:54.317Z [ERROR] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: failed to heartbeat to: peer=24d56434-ace1-1167-83c6-d6928dea4983 backoff time=10ms error="failed to connect to peer: 24d56434-ace1-1167-83c6-d6928dea4983" -2025-10-27T04:03:54.323Z [ERROR] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: failed to appendEntries to: peer="{Voter server-24d56434-ace1-1167-83c6-d6928dea4983 24d56434-ace1-1167-83c6-d6928dea4983}" error="failed to connect to peer: 24d56434-ace1-1167-83c6-d6928dea4983" -2025-10-27T04:03:54.334Z [ERROR] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: failed to heartbeat to: peer=24d56434-ace1-1167-83c6-d6928dea4983 backoff time=10ms error="failed to connect to peer: 24d56434-ace1-1167-83c6-d6928dea4983" -2025-10-27T04:03:54.340Z [WARN] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: failed to contact: server-id=server-24d56434-ace1-1167-83c6-d6928dea4983 time=50.032902ms -2025-10-27T04:03:54.341Z [ERROR] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: failed to appendEntries to: peer="{Voter server-24d56434-ace1-1167-83c6-d6928dea4983 24d56434-ace1-1167-83c6-d6928dea4983}" error="failed to connect to peer: 24d56434-ace1-1167-83c6-d6928dea4983" -2025-10-27T04:03:54.352Z [ERROR] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: failed to heartbeat to: peer=24d56434-ace1-1167-83c6-d6928dea4983 backoff time=20ms error="failed to connect to peer: 24d56434-ace1-1167-83c6-d6928dea4983" -2025-10-27T04:03:54.363Z [WARN] server-24d56434-ace1-1167-83c6-d6928dea4983: heartbeat timeout reached, starting election: last-leader-addr=53c5210a-3d2f-ce5e-bec6-75e607d95553 last-leader-id=server-53c5210a-3d2f-ce5e-bec6-75e607d95553 -2025-10-27T04:03:54.363Z [INFO] server-24d56434-ace1-1167-83c6-d6928dea4983: entering candidate state: node="Node at 24d56434-ace1-1167-83c6-d6928dea4983 [Candidate]" term=3 -2025-10-27T04:03:54.363Z [ERROR] server-24d56434-ace1-1167-83c6-d6928dea4983: failed to make requestVote RPC: target="{Voter server-2df62e77-9658-6d7d-7c21-5e94f57526d3 2df62e77-9658-6d7d-7c21-5e94f57526d3}" error="failed to connect to peer: 2df62e77-9658-6d7d-7c21-5e94f57526d3" term=3 -2025-10-27T04:03:54.363Z [ERROR] server-24d56434-ace1-1167-83c6-d6928dea4983: failed to make requestVote RPC: target="{Voter server-53c5210a-3d2f-ce5e-bec6-75e607d95553 53c5210a-3d2f-ce5e-bec6-75e607d95553}" error="failed to connect to peer: 53c5210a-3d2f-ce5e-bec6-75e607d95553" term=3 -2025-10-27T04:03:54.371Z [ERROR] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: failed to appendEntries to: peer="{Voter server-24d56434-ace1-1167-83c6-d6928dea4983 24d56434-ace1-1167-83c6-d6928dea4983}" error="failed to connect to peer: 24d56434-ace1-1167-83c6-d6928dea4983" -2025-10-27T04:03:54.382Z [ERROR] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: failed to heartbeat to: peer=24d56434-ace1-1167-83c6-d6928dea4983 backoff time=25ms error="failed to connect to peer: 24d56434-ace1-1167-83c6-d6928dea4983" -2025-10-27T04:03:54.391Z [WARN] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: failed to contact: server-id=server-24d56434-ace1-1167-83c6-d6928dea4983 time=100.779968ms -2025-10-27T04:03:54.401Z [INFO] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: updating configuration: command=RemoveServer server-id=server-24d56434-ace1-1167-83c6-d6928dea4983 server-addr= servers="[{Suffrage:Voter ID:server-53c5210a-3d2f-ce5e-bec6-75e607d95553 Address:53c5210a-3d2f-ce5e-bec6-75e607d95553} {Suffrage:Voter ID:server-2df62e77-9658-6d7d-7c21-5e94f57526d3 Address:2df62e77-9658-6d7d-7c21-5e94f57526d3}]" -2025-10-27T04:03:54.401Z [INFO] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: removed peer, stopping replication: peer=server-24d56434-ace1-1167-83c6-d6928dea4983 last-index=3 -2025-10-27T04:03:54.401Z [INFO] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: updating configuration: command=AddNonvoter server-id=server-24d56434-ace1-1167-83c6-d6928dea4983 server-addr=24d56434-ace1-1167-83c6-d6928dea4983 servers="[{Suffrage:Voter ID:server-53c5210a-3d2f-ce5e-bec6-75e607d95553 Address:53c5210a-3d2f-ce5e-bec6-75e607d95553} {Suffrage:Voter ID:server-2df62e77-9658-6d7d-7c21-5e94f57526d3 Address:2df62e77-9658-6d7d-7c21-5e94f57526d3} {Suffrage:Nonvoter ID:server-24d56434-ace1-1167-83c6-d6928dea4983 Address:24d56434-ace1-1167-83c6-d6928dea4983}]" -2025-10-27T04:03:54.401Z [INFO] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: added peer, starting replication: peer=server-24d56434-ace1-1167-83c6-d6928dea4983 -2025-10-27T04:03:54.401Z [ERROR] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: failed to appendEntries to: peer="{Nonvoter server-24d56434-ace1-1167-83c6-d6928dea4983 24d56434-ace1-1167-83c6-d6928dea4983}" error="failed to connect to peer: 24d56434-ace1-1167-83c6-d6928dea4983" -2025-10-27T04:03:54.417Z [WARN] server-24d56434-ace1-1167-83c6-d6928dea4983: Election timeout reached, restarting election -2025-10-27T04:03:54.417Z [INFO] server-24d56434-ace1-1167-83c6-d6928dea4983: entering candidate state: node="Node at 24d56434-ace1-1167-83c6-d6928dea4983 [Candidate]" term=23 -2025-10-27T04:03:54.417Z [WARN] server-2df62e77-9658-6d7d-7c21-5e94f57526d3: rejecting vote request since we have a leader: from=24d56434-ace1-1167-83c6-d6928dea4983 leader=53c5210a-3d2f-ce5e-bec6-75e607d95553 leader-id=server-53c5210a-3d2f-ce5e-bec6-75e607d95553 -2025-10-27T04:03:54.417Z [WARN] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: rejecting vote request since we have a leader: from=24d56434-ace1-1167-83c6-d6928dea4983 leader=53c5210a-3d2f-ce5e-bec6-75e607d95553 leader-id=server-53c5210a-3d2f-ce5e-bec6-75e607d95553 -2025-10-27T04:03:54.420Z [ERROR] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: peer has newer term, stopping replication: peer="{Voter server-24d56434-ace1-1167-83c6-d6928dea4983 24d56434-ace1-1167-83c6-d6928dea4983}" -2025-10-27T04:03:54.420Z [ERROR] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: peer has newer term, stopping replication: peer="{Nonvoter server-24d56434-ace1-1167-83c6-d6928dea4983 24d56434-ace1-1167-83c6-d6928dea4983}" -2025-10-27T04:03:54.421Z [INFO] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: entering follower state: follower="Node at 53c5210a-3d2f-ce5e-bec6-75e607d95553 [Follower]" leader-address= leader-id= -2025-10-27T04:03:54.421Z [INFO] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: aborting pipeline replication: peer="{Voter server-2df62e77-9658-6d7d-7c21-5e94f57526d3 2df62e77-9658-6d7d-7c21-5e94f57526d3}" -2025-10-27T04:03:54.472Z [WARN] server-2df62e77-9658-6d7d-7c21-5e94f57526d3: heartbeat timeout reached, starting election: last-leader-addr=53c5210a-3d2f-ce5e-bec6-75e607d95553 last-leader-id=server-53c5210a-3d2f-ce5e-bec6-75e607d95553 -2025-10-27T04:03:54.472Z [INFO] server-2df62e77-9658-6d7d-7c21-5e94f57526d3: entering candidate state: node="Node at 2df62e77-9658-6d7d-7c21-5e94f57526d3 [Candidate]" term=3 -2025-10-27T04:03:54.472Z [INFO] server-2df62e77-9658-6d7d-7c21-5e94f57526d3: election won: term=3 tally=2 -2025-10-27T04:03:54.472Z [INFO] server-2df62e77-9658-6d7d-7c21-5e94f57526d3: entering leader state: leader="Node at 2df62e77-9658-6d7d-7c21-5e94f57526d3 [Leader]" -2025-10-27T04:03:54.472Z [INFO] server-2df62e77-9658-6d7d-7c21-5e94f57526d3: added peer, starting replication: peer=server-53c5210a-3d2f-ce5e-bec6-75e607d95553 -2025-10-27T04:03:54.472Z [INFO] server-2df62e77-9658-6d7d-7c21-5e94f57526d3: added peer, starting replication: peer=server-24d56434-ace1-1167-83c6-d6928dea4983 -2025-10-27T04:03:54.472Z [ERROR] server-2df62e77-9658-6d7d-7c21-5e94f57526d3: peer has newer term, stopping replication: peer="{Nonvoter server-24d56434-ace1-1167-83c6-d6928dea4983 24d56434-ace1-1167-83c6-d6928dea4983}" -2025-10-27T04:03:54.472Z [INFO] server-2df62e77-9658-6d7d-7c21-5e94f57526d3: pipelining replication: peer="{Voter server-53c5210a-3d2f-ce5e-bec6-75e607d95553 53c5210a-3d2f-ce5e-bec6-75e607d95553}" -2025-10-27T04:03:54.472Z [INFO] server-2df62e77-9658-6d7d-7c21-5e94f57526d3: aborting pipeline replication: peer="{Voter server-53c5210a-3d2f-ce5e-bec6-75e607d95553 53c5210a-3d2f-ce5e-bec6-75e607d95553}" -2025-10-27T04:03:54.472Z [INFO] server-2df62e77-9658-6d7d-7c21-5e94f57526d3: entering follower state: follower="Node at 2df62e77-9658-6d7d-7c21-5e94f57526d3 [Follower]" leader-address= leader-id= -2025-10-27T04:03:54.510Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:54.514Z [WARN] server-24d56434-ace1-1167-83c6-d6928dea4983: Election timeout reached, restarting election -2025-10-27T04:03:54.514Z [INFO] server-24d56434-ace1-1167-83c6-d6928dea4983: entering candidate state: node="Node at 24d56434-ace1-1167-83c6-d6928dea4983 [Candidate]" term=24 -2025-10-27T04:03:54.514Z [WARN] server-2df62e77-9658-6d7d-7c21-5e94f57526d3: rejecting vote request since node is not a voter: from=24d56434-ace1-1167-83c6-d6928dea4983 -2025-10-27T04:03:54.514Z [WARN] server-53c5210a-3d2f-ce5e-bec6-75e607d95553: rejecting vote request since we have a leader: from=24d56434-ace1-1167-83c6-d6928dea4983 leader=2df62e77-9658-6d7d-7c21-5e94f57526d3 leader-id=server-2df62e77-9658-6d7d-7c21-5e94f57526d3 -2025-10-27T04:03:54.563Z [WARN] server-2df62e77-9658-6d7d-7c21-5e94f57526d3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:54.563Z [INFO] server-2df62e77-9658-6d7d-7c21-5e94f57526d3: entering candidate state: node="Node at 2df62e77-9658-6d7d-7c21-5e94f57526d3 [Candidate]" term=25 -2025-10-27T04:03:54.564Z [INFO] server-2df62e77-9658-6d7d-7c21-5e94f57526d3: election won: term=25 tally=2 -2025-10-27T04:03:54.564Z [INFO] server-2df62e77-9658-6d7d-7c21-5e94f57526d3: entering leader state: leader="Node at 2df62e77-9658-6d7d-7c21-5e94f57526d3 [Leader]" -2025-10-27T04:03:54.564Z [INFO] server-2df62e77-9658-6d7d-7c21-5e94f57526d3: added peer, starting replication: peer=server-53c5210a-3d2f-ce5e-bec6-75e607d95553 -2025-10-27T04:03:54.564Z [INFO] server-2df62e77-9658-6d7d-7c21-5e94f57526d3: added peer, starting replication: peer=server-24d56434-ace1-1167-83c6-d6928dea4983 -2025-10-27T04:03:54.564Z [WARN] server-24d56434-ace1-1167-83c6-d6928dea4983: failed to get previous log: previous-index=5 last-index=2 error="log not found" -2025-10-27T04:03:54.564Z [INFO] server-24d56434-ace1-1167-83c6-d6928dea4983: entering follower state: follower="Node at 24d56434-ace1-1167-83c6-d6928dea4983 [Follower]" leader-address=2df62e77-9658-6d7d-7c21-5e94f57526d3 leader-id=server-2df62e77-9658-6d7d-7c21-5e94f57526d3 -2025-10-27T04:03:54.564Z [WARN] server-2df62e77-9658-6d7d-7c21-5e94f57526d3: appendEntries rejected, sending older logs: peer="{Nonvoter server-24d56434-ace1-1167-83c6-d6928dea4983 24d56434-ace1-1167-83c6-d6928dea4983}" next=3 -2025-10-27T04:03:54.564Z [INFO] server-2df62e77-9658-6d7d-7c21-5e94f57526d3: pipelining replication: peer="{Nonvoter server-24d56434-ace1-1167-83c6-d6928dea4983 24d56434-ace1-1167-83c6-d6928dea4983}" -2025-10-27T04:03:54.564Z [INFO] server-2df62e77-9658-6d7d-7c21-5e94f57526d3: pipelining replication: peer="{Voter server-53c5210a-3d2f-ce5e-bec6-75e607d95553 53c5210a-3d2f-ce5e-bec6-75e607d95553}" -2025-10-27T04:03:54.670Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:54.564402541 +0000 UTC m=+109.559274979 (1 nodes), 54.090969ms from start of poll, 538.73896ms from cluster start. Timeout at 2025-10-27 04:03:54.669414621 +0000 UTC m=+109.664287099, 105.01212ms after stability -2025-10-27T04:03:54.693Z [INFO] server-2df62e77-9658-6d7d-7c21-5e94f57526d3: updating configuration: command=RemoveServer server-id=server-24d56434-ace1-1167-83c6-d6928dea4983 server-addr= servers="[{Suffrage:Voter ID:server-53c5210a-3d2f-ce5e-bec6-75e607d95553 Address:53c5210a-3d2f-ce5e-bec6-75e607d95553} {Suffrage:Voter ID:server-2df62e77-9658-6d7d-7c21-5e94f57526d3 Address:2df62e77-9658-6d7d-7c21-5e94f57526d3}]" -2025-10-27T04:03:54.693Z [INFO] server-2df62e77-9658-6d7d-7c21-5e94f57526d3: removed peer, stopping replication: peer=server-24d56434-ace1-1167-83c6-d6928dea4983 last-index=107 -2025-10-27T04:03:54.693Z [INFO] server-2df62e77-9658-6d7d-7c21-5e94f57526d3: aborting pipeline replication: peer="{Nonvoter server-24d56434-ace1-1167-83c6-d6928dea4983 24d56434-ace1-1167-83c6-d6928dea4983}" -2025-10-27T04:03:54.694Z [INFO] server-2df62e77-9658-6d7d-7c21-5e94f57526d3: updating configuration: command=AddVoter server-id=server-24d56434-ace1-1167-83c6-d6928dea4983 server-addr=24d56434-ace1-1167-83c6-d6928dea4983 servers="[{Suffrage:Voter ID:server-53c5210a-3d2f-ce5e-bec6-75e607d95553 Address:53c5210a-3d2f-ce5e-bec6-75e607d95553} {Suffrage:Voter ID:server-2df62e77-9658-6d7d-7c21-5e94f57526d3 Address:2df62e77-9658-6d7d-7c21-5e94f57526d3} {Suffrage:Voter ID:server-24d56434-ace1-1167-83c6-d6928dea4983 Address:24d56434-ace1-1167-83c6-d6928dea4983}]" -2025-10-27T04:03:54.694Z [INFO] server-2df62e77-9658-6d7d-7c21-5e94f57526d3: added peer, starting replication: peer=server-24d56434-ace1-1167-83c6-d6928dea4983 -2025-10-27T04:03:54.696Z [WARN] server-24d56434-ace1-1167-83c6-d6928dea4983: failed to get previous log: previous-index=108 last-index=107 error="log not found" -2025-10-27T04:03:54.696Z [WARN] server-2df62e77-9658-6d7d-7c21-5e94f57526d3: appendEntries rejected, sending older logs: peer="{Voter server-24d56434-ace1-1167-83c6-d6928dea4983 24d56434-ace1-1167-83c6-d6928dea4983}" next=108 -2025-10-27T04:03:54.696Z [INFO] server-2df62e77-9658-6d7d-7c21-5e94f57526d3: pipelining replication: peer="{Voter server-24d56434-ace1-1167-83c6-d6928dea4983 24d56434-ace1-1167-83c6-d6928dea4983}" -2025-10-27T04:03:55.771Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:55.877Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:55.771753229 +0000 UTC m=+110.766625707 (1 nodes), 2.68µs from start of poll, 1.746089688s from cluster start. Timeout at 2025-10-27 04:03:55.876777349 +0000 UTC m=+110.871649867, 105.02416ms after stability -2025-10-27T04:03:55.877Z [INFO] server-2df62e77-9658-6d7d-7c21-5e94f57526d3: aborting pipeline replication: peer="{Voter server-24d56434-ace1-1167-83c6-d6928dea4983 24d56434-ace1-1167-83c6-d6928dea4983}" -2025-10-27T04:03:55.877Z [INFO] server-2df62e77-9658-6d7d-7c21-5e94f57526d3: aborting pipeline replication: peer="{Voter server-53c5210a-3d2f-ce5e-bec6-75e607d95553 53c5210a-3d2f-ce5e-bec6-75e607d95553}" ---- PASS: TestRaft_ClusterCanRegainStability_WhenNonVoterWithHigherTermJoin (1.85s) +2026-11-29T10:29:55.762Z [INFO] server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e Address:f7d1cc3f-5164-55d6-d197-a1492ac3f84e} {Suffrage:Voter ID:server-23d4edaa-9310-7c26-d5b8-13dc0011d29a Address:23d4edaa-9310-7c26-d5b8-13dc0011d29a} {Suffrage:Voter ID:server-ac763943-c574-6590-8a2b-ed28eebe37d9 Address:ac763943-c574-6590-8a2b-ed28eebe37d9}]" +2026-11-29T10:29:55.762Z [INFO] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e Address:f7d1cc3f-5164-55d6-d197-a1492ac3f84e} {Suffrage:Voter ID:server-23d4edaa-9310-7c26-d5b8-13dc0011d29a Address:23d4edaa-9310-7c26-d5b8-13dc0011d29a} {Suffrage:Voter ID:server-ac763943-c574-6590-8a2b-ed28eebe37d9 Address:ac763943-c574-6590-8a2b-ed28eebe37d9}]" +2026-11-29T10:29:55.762Z [INFO] server-ac763943-c574-6590-8a2b-ed28eebe37d9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e Address:f7d1cc3f-5164-55d6-d197-a1492ac3f84e} {Suffrage:Voter ID:server-23d4edaa-9310-7c26-d5b8-13dc0011d29a Address:23d4edaa-9310-7c26-d5b8-13dc0011d29a} {Suffrage:Voter ID:server-ac763943-c574-6590-8a2b-ed28eebe37d9 Address:ac763943-c574-6590-8a2b-ed28eebe37d9}]" +2026-11-29T10:29:55.762Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:55.762Z [INFO] server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e: entering follower state: follower="Node at f7d1cc3f-5164-55d6-d197-a1492ac3f84e [Follower]" leader-address= leader-id= +2026-11-29T10:29:55.762Z [INFO] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: entering follower state: follower="Node at 23d4edaa-9310-7c26-d5b8-13dc0011d29a [Follower]" leader-address= leader-id= +2026-11-29T10:29:55.762Z [INFO] server-ac763943-c574-6590-8a2b-ed28eebe37d9: entering follower state: follower="Node at ac763943-c574-6590-8a2b-ed28eebe37d9 [Follower]" leader-address= leader-id= +2026-11-29T10:29:55.826Z [WARN] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:55.826Z [INFO] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: entering candidate state: node="Node at 23d4edaa-9310-7c26-d5b8-13dc0011d29a [Candidate]" term=2 +2026-11-29T10:29:55.826Z [INFO] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: election won: term=2 tally=2 +2026-11-29T10:29:55.826Z [INFO] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: entering leader state: leader="Node at 23d4edaa-9310-7c26-d5b8-13dc0011d29a [Leader]" +2026-11-29T10:29:55.826Z [INFO] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: added peer, starting replication: peer=server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e +2026-11-29T10:29:55.826Z [INFO] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: added peer, starting replication: peer=server-ac763943-c574-6590-8a2b-ed28eebe37d9 +2026-11-29T10:29:55.826Z [INFO] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: pipelining replication: peer="{Voter server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e f7d1cc3f-5164-55d6-d197-a1492ac3f84e}" +2026-11-29T10:29:55.827Z [INFO] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: pipelining replication: peer="{Voter server-ac763943-c574-6590-8a2b-ed28eebe37d9 ac763943-c574-6590-8a2b-ed28eebe37d9}" +2026-11-29T10:29:55.932Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:55.826779779 +0000 UTC m=+106.561833881 (1 nodes), 64.30836ms from start of poll, 64.4822ms from cluster start. Timeout at 2026-11-29 10:29:55.931780779 +0000 UTC m=+106.666834881, 105.001ms after stability +2026-11-29T10:29:55.939Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:56.045Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:55.939517819 +0000 UTC m=+106.674571881 (2 nodes), 960ns from start of poll, 177.2202ms from cluster start. Timeout at 2026-11-29 10:29:56.044535699 +0000 UTC m=+106.779589801, 105.01792ms after stability +2026-11-29T10:29:56.045Z [ERROR] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: failed to pipeline appendEntries: peer="{Voter server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e f7d1cc3f-5164-55d6-d197-a1492ac3f84e}" error="append pipeline closed" +2026-11-29T10:29:56.045Z [INFO] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: aborting pipeline replication: peer="{Voter server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e f7d1cc3f-5164-55d6-d197-a1492ac3f84e}" +2026-11-29T10:29:56.048Z [ERROR] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: failed to heartbeat to: peer=f7d1cc3f-5164-55d6-d197-a1492ac3f84e backoff time=10ms error="failed to connect to peer: f7d1cc3f-5164-55d6-d197-a1492ac3f84e" +2026-11-29T10:29:56.056Z [ERROR] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: failed to appendEntries to: peer="{Voter server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e f7d1cc3f-5164-55d6-d197-a1492ac3f84e}" error="failed to connect to peer: f7d1cc3f-5164-55d6-d197-a1492ac3f84e" +2026-11-29T10:29:56.066Z [ERROR] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: failed to heartbeat to: peer=f7d1cc3f-5164-55d6-d197-a1492ac3f84e backoff time=10ms error="failed to connect to peer: f7d1cc3f-5164-55d6-d197-a1492ac3f84e" +2026-11-29T10:29:56.076Z [ERROR] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: failed to appendEntries to: peer="{Voter server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e f7d1cc3f-5164-55d6-d197-a1492ac3f84e}" error="failed to connect to peer: f7d1cc3f-5164-55d6-d197-a1492ac3f84e" +2026-11-29T10:29:56.085Z [ERROR] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: failed to heartbeat to: peer=f7d1cc3f-5164-55d6-d197-a1492ac3f84e backoff time=10ms error="failed to connect to peer: f7d1cc3f-5164-55d6-d197-a1492ac3f84e" +2026-11-29T10:29:56.092Z [WARN] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: failed to contact: server-id=server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e time=51.49388ms +2026-11-29T10:29:56.094Z [ERROR] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: failed to appendEntries to: peer="{Voter server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e f7d1cc3f-5164-55d6-d197-a1492ac3f84e}" error="failed to connect to peer: f7d1cc3f-5164-55d6-d197-a1492ac3f84e" +2026-11-29T10:29:56.104Z [ERROR] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: failed to heartbeat to: peer=f7d1cc3f-5164-55d6-d197-a1492ac3f84e backoff time=20ms error="failed to connect to peer: f7d1cc3f-5164-55d6-d197-a1492ac3f84e" +2026-11-29T10:29:56.120Z [ERROR] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: failed to appendEntries to: peer="{Voter server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e f7d1cc3f-5164-55d6-d197-a1492ac3f84e}" error="failed to connect to peer: f7d1cc3f-5164-55d6-d197-a1492ac3f84e" +2026-11-29T10:29:56.132Z [ERROR] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: failed to heartbeat to: peer=f7d1cc3f-5164-55d6-d197-a1492ac3f84e backoff time=25ms error="failed to connect to peer: f7d1cc3f-5164-55d6-d197-a1492ac3f84e" +2026-11-29T10:29:56.132Z [WARN] server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e: heartbeat timeout reached, starting election: last-leader-addr=23d4edaa-9310-7c26-d5b8-13dc0011d29a last-leader-id=server-23d4edaa-9310-7c26-d5b8-13dc0011d29a +2026-11-29T10:29:56.132Z [INFO] server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e: entering candidate state: node="Node at f7d1cc3f-5164-55d6-d197-a1492ac3f84e [Candidate]" term=3 +2026-11-29T10:29:56.133Z [ERROR] server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e: failed to make requestVote RPC: target="{Voter server-ac763943-c574-6590-8a2b-ed28eebe37d9 ac763943-c574-6590-8a2b-ed28eebe37d9}" error="failed to connect to peer: ac763943-c574-6590-8a2b-ed28eebe37d9" term=3 +2026-11-29T10:29:56.133Z [ERROR] server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e: failed to make requestVote RPC: target="{Voter server-23d4edaa-9310-7c26-d5b8-13dc0011d29a 23d4edaa-9310-7c26-d5b8-13dc0011d29a}" error="failed to connect to peer: 23d4edaa-9310-7c26-d5b8-13dc0011d29a" term=3 +2026-11-29T10:29:56.141Z [WARN] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: failed to contact: server-id=server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e time=101.0356ms +2026-11-29T10:29:56.151Z [INFO] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: updating configuration: command=RemoveServer server-id=server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e server-addr= servers="[{Suffrage:Voter ID:server-23d4edaa-9310-7c26-d5b8-13dc0011d29a Address:23d4edaa-9310-7c26-d5b8-13dc0011d29a} {Suffrage:Voter ID:server-ac763943-c574-6590-8a2b-ed28eebe37d9 Address:ac763943-c574-6590-8a2b-ed28eebe37d9}]" +2026-11-29T10:29:56.151Z [INFO] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: removed peer, stopping replication: peer=server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e last-index=3 +2026-11-29T10:29:56.151Z [INFO] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: updating configuration: command=AddNonvoter server-id=server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e server-addr=f7d1cc3f-5164-55d6-d197-a1492ac3f84e servers="[{Suffrage:Voter ID:server-23d4edaa-9310-7c26-d5b8-13dc0011d29a Address:23d4edaa-9310-7c26-d5b8-13dc0011d29a} {Suffrage:Voter ID:server-ac763943-c574-6590-8a2b-ed28eebe37d9 Address:ac763943-c574-6590-8a2b-ed28eebe37d9} {Suffrage:Nonvoter ID:server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e Address:f7d1cc3f-5164-55d6-d197-a1492ac3f84e}]" +2026-11-29T10:29:56.151Z [INFO] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: added peer, starting replication: peer=server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e +2026-11-29T10:29:56.151Z [ERROR] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: failed to appendEntries to: peer="{Nonvoter server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e f7d1cc3f-5164-55d6-d197-a1492ac3f84e}" error="failed to connect to peer: f7d1cc3f-5164-55d6-d197-a1492ac3f84e" +2026-11-29T10:29:56.169Z [ERROR] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: peer has newer term, stopping replication: peer="{Nonvoter server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e f7d1cc3f-5164-55d6-d197-a1492ac3f84e}" +2026-11-29T10:29:56.169Z [ERROR] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: peer has newer term, stopping replication: peer="{Voter server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e f7d1cc3f-5164-55d6-d197-a1492ac3f84e}" +2026-11-29T10:29:56.169Z [INFO] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: entering follower state: follower="Node at 23d4edaa-9310-7c26-d5b8-13dc0011d29a [Follower]" leader-address= leader-id= +2026-11-29T10:29:56.169Z [INFO] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: aborting pipeline replication: peer="{Voter server-ac763943-c574-6590-8a2b-ed28eebe37d9 ac763943-c574-6590-8a2b-ed28eebe37d9}" +2026-11-29T10:29:56.216Z [WARN] server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e: Election timeout reached, restarting election +2026-11-29T10:29:56.216Z [INFO] server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e: entering candidate state: node="Node at f7d1cc3f-5164-55d6-d197-a1492ac3f84e [Candidate]" term=23 +2026-11-29T10:29:56.216Z [WARN] server-ac763943-c574-6590-8a2b-ed28eebe37d9: rejecting vote request since we have a leader: from=f7d1cc3f-5164-55d6-d197-a1492ac3f84e leader=23d4edaa-9310-7c26-d5b8-13dc0011d29a leader-id=server-23d4edaa-9310-7c26-d5b8-13dc0011d29a +2026-11-29T10:29:56.216Z [WARN] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: rejecting vote request since node is not a voter: from=f7d1cc3f-5164-55d6-d197-a1492ac3f84e +2026-11-29T10:29:56.238Z [WARN] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:56.238Z [INFO] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: entering candidate state: node="Node at 23d4edaa-9310-7c26-d5b8-13dc0011d29a [Candidate]" term=24 +2026-11-29T10:29:56.238Z [INFO] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: election won: term=24 tally=2 +2026-11-29T10:29:56.238Z [INFO] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: entering leader state: leader="Node at 23d4edaa-9310-7c26-d5b8-13dc0011d29a [Leader]" +2026-11-29T10:29:56.238Z [INFO] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: added peer, starting replication: peer=server-ac763943-c574-6590-8a2b-ed28eebe37d9 +2026-11-29T10:29:56.238Z [INFO] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: added peer, starting replication: peer=server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e +2026-11-29T10:29:56.238Z [WARN] server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e: failed to get previous log: previous-index=4 last-index=2 error="log not found" +2026-11-29T10:29:56.238Z [INFO] server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e: entering follower state: follower="Node at f7d1cc3f-5164-55d6-d197-a1492ac3f84e [Follower]" leader-address=23d4edaa-9310-7c26-d5b8-13dc0011d29a leader-id=server-23d4edaa-9310-7c26-d5b8-13dc0011d29a +2026-11-29T10:29:56.238Z [WARN] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: appendEntries rejected, sending older logs: peer="{Nonvoter server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e f7d1cc3f-5164-55d6-d197-a1492ac3f84e}" next=3 +2026-11-29T10:29:56.238Z [INFO] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: pipelining replication: peer="{Nonvoter server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e f7d1cc3f-5164-55d6-d197-a1492ac3f84e}" +2026-11-29T10:29:56.238Z [INFO] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: pipelining replication: peer="{Voter server-ac763943-c574-6590-8a2b-ed28eebe37d9 ac763943-c574-6590-8a2b-ed28eebe37d9}" +2026-11-29T10:29:56.256Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:56.362Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:56.256749939 +0000 UTC m=+106.991804041 (1 nodes), 34.52µs from start of poll, 494.45236ms from cluster start. Timeout at 2026-11-29 10:29:56.361777179 +0000 UTC m=+107.096831281, 105.02724ms after stability +2026-11-29T10:29:56.391Z [INFO] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: updating configuration: command=RemoveServer server-id=server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e server-addr= servers="[{Suffrage:Voter ID:server-23d4edaa-9310-7c26-d5b8-13dc0011d29a Address:23d4edaa-9310-7c26-d5b8-13dc0011d29a} {Suffrage:Voter ID:server-ac763943-c574-6590-8a2b-ed28eebe37d9 Address:ac763943-c574-6590-8a2b-ed28eebe37d9}]" +2026-11-29T10:29:56.391Z [INFO] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: removed peer, stopping replication: peer=server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e last-index=106 +2026-11-29T10:29:56.391Z [INFO] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: aborting pipeline replication: peer="{Nonvoter server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e f7d1cc3f-5164-55d6-d197-a1492ac3f84e}" +2026-11-29T10:29:56.391Z [INFO] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: updating configuration: command=AddVoter server-id=server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e server-addr=f7d1cc3f-5164-55d6-d197-a1492ac3f84e servers="[{Suffrage:Voter ID:server-23d4edaa-9310-7c26-d5b8-13dc0011d29a Address:23d4edaa-9310-7c26-d5b8-13dc0011d29a} {Suffrage:Voter ID:server-ac763943-c574-6590-8a2b-ed28eebe37d9 Address:ac763943-c574-6590-8a2b-ed28eebe37d9} {Suffrage:Voter ID:server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e Address:f7d1cc3f-5164-55d6-d197-a1492ac3f84e}]" +2026-11-29T10:29:56.391Z [INFO] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: added peer, starting replication: peer=server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e +2026-11-29T10:29:56.391Z [WARN] server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e: failed to get previous log: previous-index=107 last-index=106 error="log not found" +2026-11-29T10:29:56.391Z [WARN] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: appendEntries rejected, sending older logs: peer="{Voter server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e f7d1cc3f-5164-55d6-d197-a1492ac3f84e}" next=107 +2026-11-29T10:29:56.391Z [INFO] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: pipelining replication: peer="{Voter server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e f7d1cc3f-5164-55d6-d197-a1492ac3f84e}" +2026-11-29T10:29:57.458Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:57.567Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:57.459225979 +0000 UTC m=+108.194280081 (1 nodes), 1.8µs from start of poll, 1.6969284s from cluster start. Timeout at 2026-11-29 10:29:57.564245779 +0000 UTC m=+108.299299881, 105.0198ms after stability +2026-11-29T10:29:57.567Z [INFO] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: aborting pipeline replication: peer="{Voter server-f7d1cc3f-5164-55d6-d197-a1492ac3f84e f7d1cc3f-5164-55d6-d197-a1492ac3f84e}" +2026-11-29T10:29:57.567Z [INFO] server-23d4edaa-9310-7c26-d5b8-13dc0011d29a: aborting pipeline replication: peer="{Voter server-ac763943-c574-6590-8a2b-ed28eebe37d9 ac763943-c574-6590-8a2b-ed28eebe37d9}" +--- PASS: TestRaft_ClusterCanRegainStability_WhenNonVoterWithHigherTermJoin (1.81s) === RUN TestRaft_FollowerRemovalNoElection -2025-10-27T04:03:55.878Z [INFO] server-47970d8f-9e81-5138-6237-1d802a80459e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-47970d8f-9e81-5138-6237-1d802a80459e Address:47970d8f-9e81-5138-6237-1d802a80459e} {Suffrage:Voter ID:server-475d7637-e61b-154e-15a7-aa205c97bb79 Address:475d7637-e61b-154e-15a7-aa205c97bb79} {Suffrage:Voter ID:server-eaf9f8fa-57b0-a8d4-bfbf-722e3746c80a Address:eaf9f8fa-57b0-a8d4-bfbf-722e3746c80a}]" -2025-10-27T04:03:55.878Z [INFO] server-475d7637-e61b-154e-15a7-aa205c97bb79: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-47970d8f-9e81-5138-6237-1d802a80459e Address:47970d8f-9e81-5138-6237-1d802a80459e} {Suffrage:Voter ID:server-475d7637-e61b-154e-15a7-aa205c97bb79 Address:475d7637-e61b-154e-15a7-aa205c97bb79} {Suffrage:Voter ID:server-eaf9f8fa-57b0-a8d4-bfbf-722e3746c80a Address:eaf9f8fa-57b0-a8d4-bfbf-722e3746c80a}]" -2025-10-27T04:03:55.878Z [INFO] server-47970d8f-9e81-5138-6237-1d802a80459e: entering follower state: follower="Node at 47970d8f-9e81-5138-6237-1d802a80459e [Follower]" leader-address= leader-id= -2025-10-27T04:03:55.878Z [INFO] server-eaf9f8fa-57b0-a8d4-bfbf-722e3746c80a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-47970d8f-9e81-5138-6237-1d802a80459e Address:47970d8f-9e81-5138-6237-1d802a80459e} {Suffrage:Voter ID:server-475d7637-e61b-154e-15a7-aa205c97bb79 Address:475d7637-e61b-154e-15a7-aa205c97bb79} {Suffrage:Voter ID:server-eaf9f8fa-57b0-a8d4-bfbf-722e3746c80a Address:eaf9f8fa-57b0-a8d4-bfbf-722e3746c80a}]" -2025-10-27T04:03:55.878Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:55.878Z [INFO] server-475d7637-e61b-154e-15a7-aa205c97bb79: entering follower state: follower="Node at 475d7637-e61b-154e-15a7-aa205c97bb79 [Follower]" leader-address= leader-id= -2025-10-27T04:03:55.878Z [INFO] server-eaf9f8fa-57b0-a8d4-bfbf-722e3746c80a: entering follower state: follower="Node at eaf9f8fa-57b0-a8d4-bfbf-722e3746c80a [Follower]" leader-address= leader-id= -2025-10-27T04:03:56.046Z [WARN] server-47970d8f-9e81-5138-6237-1d802a80459e: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:56.046Z [INFO] server-47970d8f-9e81-5138-6237-1d802a80459e: entering candidate state: node="Node at 47970d8f-9e81-5138-6237-1d802a80459e [Candidate]" term=2 -2025-10-27T04:03:56.046Z [INFO] server-47970d8f-9e81-5138-6237-1d802a80459e: election won: term=2 tally=2 -2025-10-27T04:03:56.046Z [INFO] server-47970d8f-9e81-5138-6237-1d802a80459e: entering leader state: leader="Node at 47970d8f-9e81-5138-6237-1d802a80459e [Leader]" -2025-10-27T04:03:56.046Z [INFO] server-47970d8f-9e81-5138-6237-1d802a80459e: added peer, starting replication: peer=server-475d7637-e61b-154e-15a7-aa205c97bb79 -2025-10-27T04:03:56.046Z [INFO] server-47970d8f-9e81-5138-6237-1d802a80459e: added peer, starting replication: peer=server-eaf9f8fa-57b0-a8d4-bfbf-722e3746c80a -2025-10-27T04:03:56.046Z [INFO] server-47970d8f-9e81-5138-6237-1d802a80459e: pipelining replication: peer="{Voter server-eaf9f8fa-57b0-a8d4-bfbf-722e3746c80a eaf9f8fa-57b0-a8d4-bfbf-722e3746c80a}" -2025-10-27T04:03:56.047Z [INFO] server-47970d8f-9e81-5138-6237-1d802a80459e: pipelining replication: peer="{Voter server-475d7637-e61b-154e-15a7-aa205c97bb79 475d7637-e61b-154e-15a7-aa205c97bb79}" -2025-10-27T04:03:56.252Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:56.046948109 +0000 UTC m=+111.041820627 (1 nodes), 168.507191ms from start of poll, 168.759033ms from cluster start. Timeout at 2025-10-27 04:03:56.251951549 +0000 UTC m=+111.246824027, 205.0034ms after stability -2025-10-27T04:03:56.252Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:56.457Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:56.252128591 +0000 UTC m=+111.247001069 (1 nodes), 2.36µs from start of poll, 373.939475ms from cluster start. Timeout at 2025-10-27 04:03:56.457151031 +0000 UTC m=+111.452023509, 205.02244ms after stability -2025-10-27T04:03:56.462Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:03:56.668Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:03:56.463002271 +0000 UTC m=+111.457874789 (2 nodes), 3.56µs from start of poll, 584.813195ms from cluster start. Timeout at 2025-10-27 04:03:56.668035711 +0000 UTC m=+111.662908109, 205.03332ms after stability -2025-10-27T04:03:56.668Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:56.873Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:56.668373794 +0000 UTC m=+111.663246192 (1 nodes), 880ns from start of poll, 790.184598ms from cluster start. Timeout at 2025-10-27 04:03:56.873404394 +0000 UTC m=+111.868276912, 205.03072ms after stability - raft_test.go:2961: [INFO] restarting Node at 475d7637-e61b-154e-15a7-aa205c97bb79 [Follower] -2025-10-27T04:03:56.873Z [INFO] server-475d7637-e61b-154e-15a7-aa205c97bb79: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-47970d8f-9e81-5138-6237-1d802a80459e Address:47970d8f-9e81-5138-6237-1d802a80459e} {Suffrage:Voter ID:server-475d7637-e61b-154e-15a7-aa205c97bb79 Address:475d7637-e61b-154e-15a7-aa205c97bb79} {Suffrage:Voter ID:server-eaf9f8fa-57b0-a8d4-bfbf-722e3746c80a Address:eaf9f8fa-57b0-a8d4-bfbf-722e3746c80a}]" -2025-10-27T04:03:56.873Z [INFO] server-475d7637-e61b-154e-15a7-aa205c97bb79: entering follower state: follower="Node at 475d7637-e61b-154e-15a7-aa205c97bb79 [Follower]" leader-address= leader-id= -2025-10-27T04:03:57.124Z [INFO] server-47970d8f-9e81-5138-6237-1d802a80459e: aborting pipeline replication: peer="{Voter server-eaf9f8fa-57b0-a8d4-bfbf-722e3746c80a eaf9f8fa-57b0-a8d4-bfbf-722e3746c80a}" -2025-10-27T04:03:57.497Z [ERROR] server-47970d8f-9e81-5138-6237-1d802a80459e: failed to pipeline appendEntries: peer="{Voter server-475d7637-e61b-154e-15a7-aa205c97bb79 475d7637-e61b-154e-15a7-aa205c97bb79}" error="command enqueue timeout" -2025-10-27T04:03:57.497Z [INFO] server-47970d8f-9e81-5138-6237-1d802a80459e: aborting pipeline replication: peer="{Voter server-475d7637-e61b-154e-15a7-aa205c97bb79 475d7637-e61b-154e-15a7-aa205c97bb79}" -2025-10-27T04:03:57.641Z [ERROR] server-47970d8f-9e81-5138-6237-1d802a80459e: failed to heartbeat to: peer=475d7637-e61b-154e-15a7-aa205c97bb79 backoff time=10ms error="command timed out" ---- PASS: TestRaft_FollowerRemovalNoElection (1.76s) +2026-11-29T10:29:57.567Z [INFO] server-aa242acc-e0e5-577c-0a24-34f338cb52b2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-aa242acc-e0e5-577c-0a24-34f338cb52b2 Address:aa242acc-e0e5-577c-0a24-34f338cb52b2} {Suffrage:Voter ID:server-df089129-ac9b-e555-ad2a-5539af046bc4 Address:df089129-ac9b-e555-ad2a-5539af046bc4} {Suffrage:Voter ID:server-9061cf0c-c2c8-ece6-999c-653b6704f655 Address:9061cf0c-c2c8-ece6-999c-653b6704f655}]" +2026-11-29T10:29:57.567Z [INFO] server-df089129-ac9b-e555-ad2a-5539af046bc4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-aa242acc-e0e5-577c-0a24-34f338cb52b2 Address:aa242acc-e0e5-577c-0a24-34f338cb52b2} {Suffrage:Voter ID:server-df089129-ac9b-e555-ad2a-5539af046bc4 Address:df089129-ac9b-e555-ad2a-5539af046bc4} {Suffrage:Voter ID:server-9061cf0c-c2c8-ece6-999c-653b6704f655 Address:9061cf0c-c2c8-ece6-999c-653b6704f655}]" +2026-11-29T10:29:57.567Z [INFO] server-9061cf0c-c2c8-ece6-999c-653b6704f655: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-aa242acc-e0e5-577c-0a24-34f338cb52b2 Address:aa242acc-e0e5-577c-0a24-34f338cb52b2} {Suffrage:Voter ID:server-df089129-ac9b-e555-ad2a-5539af046bc4 Address:df089129-ac9b-e555-ad2a-5539af046bc4} {Suffrage:Voter ID:server-9061cf0c-c2c8-ece6-999c-653b6704f655 Address:9061cf0c-c2c8-ece6-999c-653b6704f655}]" +2026-11-29T10:29:57.567Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:57.567Z [INFO] server-aa242acc-e0e5-577c-0a24-34f338cb52b2: entering follower state: follower="Node at aa242acc-e0e5-577c-0a24-34f338cb52b2 [Follower]" leader-address= leader-id= +2026-11-29T10:29:57.567Z [INFO] server-df089129-ac9b-e555-ad2a-5539af046bc4: entering follower state: follower="Node at df089129-ac9b-e555-ad2a-5539af046bc4 [Follower]" leader-address= leader-id= +2026-11-29T10:29:57.567Z [INFO] server-9061cf0c-c2c8-ece6-999c-653b6704f655: entering follower state: follower="Node at 9061cf0c-c2c8-ece6-999c-653b6704f655 [Follower]" leader-address= leader-id= +2026-11-29T10:29:57.731Z [WARN] server-aa242acc-e0e5-577c-0a24-34f338cb52b2: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:57.732Z [INFO] server-aa242acc-e0e5-577c-0a24-34f338cb52b2: entering candidate state: node="Node at aa242acc-e0e5-577c-0a24-34f338cb52b2 [Candidate]" term=2 +2026-11-29T10:29:57.733Z [INFO] server-aa242acc-e0e5-577c-0a24-34f338cb52b2: election won: term=2 tally=2 +2026-11-29T10:29:57.733Z [INFO] server-aa242acc-e0e5-577c-0a24-34f338cb52b2: entering leader state: leader="Node at aa242acc-e0e5-577c-0a24-34f338cb52b2 [Leader]" +2026-11-29T10:29:57.733Z [INFO] server-aa242acc-e0e5-577c-0a24-34f338cb52b2: added peer, starting replication: peer=server-df089129-ac9b-e555-ad2a-5539af046bc4 +2026-11-29T10:29:57.733Z [INFO] server-aa242acc-e0e5-577c-0a24-34f338cb52b2: added peer, starting replication: peer=server-9061cf0c-c2c8-ece6-999c-653b6704f655 +2026-11-29T10:29:57.733Z [INFO] server-aa242acc-e0e5-577c-0a24-34f338cb52b2: pipelining replication: peer="{Voter server-df089129-ac9b-e555-ad2a-5539af046bc4 df089129-ac9b-e555-ad2a-5539af046bc4}" +2026-11-29T10:29:57.733Z [INFO] server-aa242acc-e0e5-577c-0a24-34f338cb52b2: pipelining replication: peer="{Voter server-9061cf0c-c2c8-ece6-999c-653b6704f655 9061cf0c-c2c8-ece6-999c-653b6704f655}" +2026-11-29T10:29:57.938Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:57.733292139 +0000 UTC m=+108.468346241 (1 nodes), 165.57112ms from start of poll, 165.77284ms from cluster start. Timeout at 2026-11-29 10:29:57.938295179 +0000 UTC m=+108.673349281, 205.00304ms after stability +2026-11-29T10:29:57.938Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:58.143Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:57.938479179 +0000 UTC m=+108.673533241 (1 nodes), 1.64µs from start of poll, 370.95984ms from cluster start. Timeout at 2026-11-29 10:29:58.143498459 +0000 UTC m=+108.878552561, 205.01932ms after stability +2026-11-29T10:29:58.149Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:58.355Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:58.149834219 +0000 UTC m=+108.884888321 (2 nodes), 1.88µs from start of poll, 582.31492ms from cluster start. Timeout at 2026-11-29 10:29:58.354857499 +0000 UTC m=+109.089911561, 205.02324ms after stability +2026-11-29T10:29:58.355Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:58.561Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:58.355680779 +0000 UTC m=+109.090734881 (1 nodes), 2.12µs from start of poll, 788.16148ms from cluster start. Timeout at 2026-11-29 10:29:58.560715619 +0000 UTC m=+109.295769721, 205.03484ms after stability + raft_test.go:2961: [INFO] restarting Node at df089129-ac9b-e555-ad2a-5539af046bc4 [Follower] +2026-11-29T10:29:58.561Z [INFO] server-df089129-ac9b-e555-ad2a-5539af046bc4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-aa242acc-e0e5-577c-0a24-34f338cb52b2 Address:aa242acc-e0e5-577c-0a24-34f338cb52b2} {Suffrage:Voter ID:server-df089129-ac9b-e555-ad2a-5539af046bc4 Address:df089129-ac9b-e555-ad2a-5539af046bc4} {Suffrage:Voter ID:server-9061cf0c-c2c8-ece6-999c-653b6704f655 Address:9061cf0c-c2c8-ece6-999c-653b6704f655}]" +2026-11-29T10:29:58.561Z [INFO] server-df089129-ac9b-e555-ad2a-5539af046bc4: entering follower state: follower="Node at df089129-ac9b-e555-ad2a-5539af046bc4 [Follower]" leader-address= leader-id= +2026-11-29T10:29:58.814Z [INFO] server-aa242acc-e0e5-577c-0a24-34f338cb52b2: aborting pipeline replication: peer="{Voter server-9061cf0c-c2c8-ece6-999c-653b6704f655 9061cf0c-c2c8-ece6-999c-653b6704f655}" +2026-11-29T10:29:59.207Z [ERROR] server-aa242acc-e0e5-577c-0a24-34f338cb52b2: failed to pipeline appendEntries: peer="{Voter server-df089129-ac9b-e555-ad2a-5539af046bc4 df089129-ac9b-e555-ad2a-5539af046bc4}" error="command enqueue timeout" +2026-11-29T10:29:59.207Z [INFO] server-aa242acc-e0e5-577c-0a24-34f338cb52b2: aborting pipeline replication: peer="{Voter server-df089129-ac9b-e555-ad2a-5539af046bc4 df089129-ac9b-e555-ad2a-5539af046bc4}" +2026-11-29T10:29:59.335Z [ERROR] server-aa242acc-e0e5-577c-0a24-34f338cb52b2: failed to heartbeat to: peer=df089129-ac9b-e555-ad2a-5539af046bc4 backoff time=10ms error="command timed out" +--- PASS: TestRaft_FollowerRemovalNoElection (1.77s) === RUN TestRaft_VoteWithNoIDNoAddr -2025-10-27T04:03:57.642Z [INFO] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2ae5b62c-d563-57ee-147e-265c1295d1ae Address:2ae5b62c-d563-57ee-147e-265c1295d1ae} {Suffrage:Voter ID:server-f8e91ef2-78da-bd9b-4462-c724509f678f Address:f8e91ef2-78da-bd9b-4462-c724509f678f} {Suffrage:Voter ID:server-e75b7061-6928-bd5b-2cdf-e9ab945b058f Address:e75b7061-6928-bd5b-2cdf-e9ab945b058f}]" -2025-10-27T04:03:57.643Z [INFO] server-f8e91ef2-78da-bd9b-4462-c724509f678f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2ae5b62c-d563-57ee-147e-265c1295d1ae Address:2ae5b62c-d563-57ee-147e-265c1295d1ae} {Suffrage:Voter ID:server-f8e91ef2-78da-bd9b-4462-c724509f678f Address:f8e91ef2-78da-bd9b-4462-c724509f678f} {Suffrage:Voter ID:server-e75b7061-6928-bd5b-2cdf-e9ab945b058f Address:e75b7061-6928-bd5b-2cdf-e9ab945b058f}]" -2025-10-27T04:03:57.643Z [INFO] server-e75b7061-6928-bd5b-2cdf-e9ab945b058f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2ae5b62c-d563-57ee-147e-265c1295d1ae Address:2ae5b62c-d563-57ee-147e-265c1295d1ae} {Suffrage:Voter ID:server-f8e91ef2-78da-bd9b-4462-c724509f678f Address:f8e91ef2-78da-bd9b-4462-c724509f678f} {Suffrage:Voter ID:server-e75b7061-6928-bd5b-2cdf-e9ab945b058f Address:e75b7061-6928-bd5b-2cdf-e9ab945b058f}]" -2025-10-27T04:03:57.643Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:57.643Z [INFO] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: entering follower state: follower="Node at 2ae5b62c-d563-57ee-147e-265c1295d1ae [Follower]" leader-address= leader-id= -2025-10-27T04:03:57.643Z [INFO] server-f8e91ef2-78da-bd9b-4462-c724509f678f: entering follower state: follower="Node at f8e91ef2-78da-bd9b-4462-c724509f678f [Follower]" leader-address= leader-id= -2025-10-27T04:03:57.643Z [INFO] server-e75b7061-6928-bd5b-2cdf-e9ab945b058f: entering follower state: follower="Node at e75b7061-6928-bd5b-2cdf-e9ab945b058f [Follower]" leader-address= leader-id= -2025-10-27T04:03:57.699Z [WARN] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= -2025-10-27T04:03:57.699Z [INFO] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: entering candidate state: node="Node at 2ae5b62c-d563-57ee-147e-265c1295d1ae [Candidate]" term=2 -2025-10-27T04:03:57.699Z [INFO] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: election won: term=2 tally=2 -2025-10-27T04:03:57.699Z [INFO] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: entering leader state: leader="Node at 2ae5b62c-d563-57ee-147e-265c1295d1ae [Leader]" -2025-10-27T04:03:57.699Z [INFO] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: added peer, starting replication: peer=server-f8e91ef2-78da-bd9b-4462-c724509f678f -2025-10-27T04:03:57.699Z [INFO] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: added peer, starting replication: peer=server-e75b7061-6928-bd5b-2cdf-e9ab945b058f -2025-10-27T04:03:57.699Z [INFO] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: pipelining replication: peer="{Voter server-f8e91ef2-78da-bd9b-4462-c724509f678f f8e91ef2-78da-bd9b-4462-c724509f678f}" -2025-10-27T04:03:57.699Z [INFO] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: pipelining replication: peer="{Voter server-e75b7061-6928-bd5b-2cdf-e9ab945b058f e75b7061-6928-bd5b-2cdf-e9ab945b058f}" -2025-10-27T04:03:57.806Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:57.69969916 +0000 UTC m=+112.694571598 (1 nodes), 56.531347ms from start of poll, 56.97071ms from cluster start. Timeout at 2025-10-27 04:03:57.80470172 +0000 UTC m=+112.799574158, 105.00256ms after stability -2025-10-27T04:03:57.808Z [INFO] cluster: starting stability test: raft-state=Leader -2025-10-27T04:03:57.913Z [INFO] cluster: stable state for Leader reached at 2025-10-27 04:03:57.808314381 +0000 UTC m=+112.803186859 (1 nodes), 1.12µs from start of poll, 165.585971ms from cluster start. Timeout at 2025-10-27 04:03:57.913349942 +0000 UTC m=+112.908222420, 105.035561ms after stability -2025-10-27T04:03:57.919Z [INFO] cluster: starting stability test: raft-state=Follower -2025-10-27T04:03:58.024Z [INFO] cluster: stable state for Follower reached at 2025-10-27 04:03:57.919059938 +0000 UTC m=+112.913932416 (2 nodes), 3.84µs from start of poll, 276.331528ms from cluster start. Timeout at 2025-10-27 04:03:58.024081218 +0000 UTC m=+113.018953696, 105.02128ms after stability -2025-10-27T04:03:58.026Z [ERROR] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: failed to pipeline appendEntries: peer="{Voter server-f8e91ef2-78da-bd9b-4462-c724509f678f f8e91ef2-78da-bd9b-4462-c724509f678f}" error="append pipeline closed" -2025-10-27T04:03:58.026Z [INFO] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: aborting pipeline replication: peer="{Voter server-f8e91ef2-78da-bd9b-4462-c724509f678f f8e91ef2-78da-bd9b-4462-c724509f678f}" -2025-10-27T04:03:58.026Z [ERROR] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: failed to heartbeat to: peer=f8e91ef2-78da-bd9b-4462-c724509f678f backoff time=10ms error="failed to connect to peer: f8e91ef2-78da-bd9b-4462-c724509f678f" -2025-10-27T04:03:58.030Z [ERROR] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: failed to pipeline appendEntries: peer="{Voter server-e75b7061-6928-bd5b-2cdf-e9ab945b058f e75b7061-6928-bd5b-2cdf-e9ab945b058f}" error="append pipeline closed" -2025-10-27T04:03:58.030Z [INFO] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: aborting pipeline replication: peer="{Voter server-e75b7061-6928-bd5b-2cdf-e9ab945b058f e75b7061-6928-bd5b-2cdf-e9ab945b058f}" -2025-10-27T04:03:58.030Z [ERROR] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: failed to heartbeat to: peer=e75b7061-6928-bd5b-2cdf-e9ab945b058f backoff time=10ms error="failed to connect to peer: e75b7061-6928-bd5b-2cdf-e9ab945b058f" -2025-10-27T04:03:58.037Z [ERROR] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: failed to appendEntries to: peer="{Voter server-f8e91ef2-78da-bd9b-4462-c724509f678f f8e91ef2-78da-bd9b-4462-c724509f678f}" error="failed to connect to peer: f8e91ef2-78da-bd9b-4462-c724509f678f" -2025-10-27T04:03:58.039Z [ERROR] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: failed to appendEntries to: peer="{Voter server-e75b7061-6928-bd5b-2cdf-e9ab945b058f e75b7061-6928-bd5b-2cdf-e9ab945b058f}" error="failed to connect to peer: e75b7061-6928-bd5b-2cdf-e9ab945b058f" -2025-10-27T04:03:58.046Z [ERROR] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: failed to heartbeat to: peer=f8e91ef2-78da-bd9b-4462-c724509f678f backoff time=10ms error="failed to connect to peer: f8e91ef2-78da-bd9b-4462-c724509f678f" -2025-10-27T04:03:58.050Z [ERROR] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: failed to heartbeat to: peer=e75b7061-6928-bd5b-2cdf-e9ab945b058f backoff time=10ms error="failed to connect to peer: e75b7061-6928-bd5b-2cdf-e9ab945b058f" -2025-10-27T04:03:58.055Z [ERROR] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: failed to appendEntries to: peer="{Voter server-f8e91ef2-78da-bd9b-4462-c724509f678f f8e91ef2-78da-bd9b-4462-c724509f678f}" error="failed to connect to peer: f8e91ef2-78da-bd9b-4462-c724509f678f" -2025-10-27T04:03:58.059Z [ERROR] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: failed to appendEntries to: peer="{Voter server-e75b7061-6928-bd5b-2cdf-e9ab945b058f e75b7061-6928-bd5b-2cdf-e9ab945b058f}" error="failed to connect to peer: e75b7061-6928-bd5b-2cdf-e9ab945b058f" -2025-10-27T04:03:58.063Z [ERROR] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: failed to heartbeat to: peer=f8e91ef2-78da-bd9b-4462-c724509f678f backoff time=10ms error="failed to connect to peer: f8e91ef2-78da-bd9b-4462-c724509f678f" -2025-10-27T04:03:58.067Z [ERROR] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: failed to heartbeat to: peer=e75b7061-6928-bd5b-2cdf-e9ab945b058f backoff time=10ms error="failed to connect to peer: e75b7061-6928-bd5b-2cdf-e9ab945b058f" -2025-10-27T04:03:58.070Z [WARN] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: failed to contact: server-id=server-f8e91ef2-78da-bd9b-4462-c724509f678f time=50.065502ms -2025-10-27T04:03:58.072Z [ERROR] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: failed to appendEntries to: peer="{Voter server-f8e91ef2-78da-bd9b-4462-c724509f678f f8e91ef2-78da-bd9b-4462-c724509f678f}" error="failed to connect to peer: f8e91ef2-78da-bd9b-4462-c724509f678f" -2025-10-27T04:03:58.078Z [ERROR] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: failed to appendEntries to: peer="{Voter server-e75b7061-6928-bd5b-2cdf-e9ab945b058f e75b7061-6928-bd5b-2cdf-e9ab945b058f}" error="failed to connect to peer: e75b7061-6928-bd5b-2cdf-e9ab945b058f" -2025-10-27T04:03:58.079Z [ERROR] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: failed to heartbeat to: peer=f8e91ef2-78da-bd9b-4462-c724509f678f backoff time=20ms error="failed to connect to peer: f8e91ef2-78da-bd9b-4462-c724509f678f" -2025-10-27T04:03:58.082Z [WARN] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: failed to contact: server-id=server-f8e91ef2-78da-bd9b-4462-c724509f678f time=61.313659ms -2025-10-27T04:03:58.082Z [WARN] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: failed to contact: server-id=server-e75b7061-6928-bd5b-2cdf-e9ab945b058f time=57.582833ms -2025-10-27T04:03:58.082Z [WARN] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: failed to contact quorum of nodes, stepping down -2025-10-27T04:03:58.082Z [INFO] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: entering follower state: follower="Node at 2ae5b62c-d563-57ee-147e-265c1295d1ae [Follower]" leader-address= leader-id= -2025-10-27T04:03:58.098Z [WARN] server-f8e91ef2-78da-bd9b-4462-c724509f678f: heartbeat timeout reached, starting election: last-leader-addr=2ae5b62c-d563-57ee-147e-265c1295d1ae last-leader-id=server-2ae5b62c-d563-57ee-147e-265c1295d1ae -2025-10-27T04:03:58.098Z [INFO] server-f8e91ef2-78da-bd9b-4462-c724509f678f: entering candidate state: node="Node at f8e91ef2-78da-bd9b-4462-c724509f678f [Candidate]" term=3 -2025-10-27T04:03:58.098Z [ERROR] server-2ae5b62c-d563-57ee-147e-265c1295d1ae: failed to appendEntries to: peer="{Voter server-f8e91ef2-78da-bd9b-4462-c724509f678f f8e91ef2-78da-bd9b-4462-c724509f678f}" error="failed to connect to peer: f8e91ef2-78da-bd9b-4462-c724509f678f" -2025-10-27T04:03:58.098Z [WARN] server-e75b7061-6928-bd5b-2cdf-e9ab945b058f: rejecting vote request since we have a leader: from=f8e91ef2-78da-bd9b-4462-c724509f678f leader=2ae5b62c-d563-57ee-147e-265c1295d1ae leader-id=server-2ae5b62c-d563-57ee-147e-265c1295d1ae -2025-10-27T04:03:58.098Z [ERROR] server-f8e91ef2-78da-bd9b-4462-c724509f678f: failed to make requestVote RPC: target="{Voter server-2ae5b62c-d563-57ee-147e-265c1295d1ae 2ae5b62c-d563-57ee-147e-265c1295d1ae}" error="failed to connect to peer: 2ae5b62c-d563-57ee-147e-265c1295d1ae" term=3 -2025-10-27T04:03:58.130Z [INFO] server-f8e91ef2-78da-bd9b-4462-c724509f678f: entering follower state: follower="Node at f8e91ef2-78da-bd9b-4462-c724509f678f [Follower]" leader-address= leader-id= +2026-11-29T10:29:59.336Z [INFO] server-fd4f5d91-497f-3214-9074-98b4abb9d8d3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fd4f5d91-497f-3214-9074-98b4abb9d8d3 Address:fd4f5d91-497f-3214-9074-98b4abb9d8d3} {Suffrage:Voter ID:server-08155706-0657-6915-7fca-8b77cae297bf Address:08155706-0657-6915-7fca-8b77cae297bf} {Suffrage:Voter ID:server-080718e2-9d62-52dd-273f-08d12bdf55d2 Address:080718e2-9d62-52dd-273f-08d12bdf55d2}]" +2026-11-29T10:29:59.336Z [INFO] server-08155706-0657-6915-7fca-8b77cae297bf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fd4f5d91-497f-3214-9074-98b4abb9d8d3 Address:fd4f5d91-497f-3214-9074-98b4abb9d8d3} {Suffrage:Voter ID:server-08155706-0657-6915-7fca-8b77cae297bf Address:08155706-0657-6915-7fca-8b77cae297bf} {Suffrage:Voter ID:server-080718e2-9d62-52dd-273f-08d12bdf55d2 Address:080718e2-9d62-52dd-273f-08d12bdf55d2}]" +2026-11-29T10:29:59.336Z [INFO] server-080718e2-9d62-52dd-273f-08d12bdf55d2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fd4f5d91-497f-3214-9074-98b4abb9d8d3 Address:fd4f5d91-497f-3214-9074-98b4abb9d8d3} {Suffrage:Voter ID:server-08155706-0657-6915-7fca-8b77cae297bf Address:08155706-0657-6915-7fca-8b77cae297bf} {Suffrage:Voter ID:server-080718e2-9d62-52dd-273f-08d12bdf55d2 Address:080718e2-9d62-52dd-273f-08d12bdf55d2}]" +2026-11-29T10:29:59.336Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:59.336Z [INFO] server-fd4f5d91-497f-3214-9074-98b4abb9d8d3: entering follower state: follower="Node at fd4f5d91-497f-3214-9074-98b4abb9d8d3 [Follower]" leader-address= leader-id= +2026-11-29T10:29:59.336Z [INFO] server-08155706-0657-6915-7fca-8b77cae297bf: entering follower state: follower="Node at 08155706-0657-6915-7fca-8b77cae297bf [Follower]" leader-address= leader-id= +2026-11-29T10:29:59.336Z [INFO] server-080718e2-9d62-52dd-273f-08d12bdf55d2: entering follower state: follower="Node at 080718e2-9d62-52dd-273f-08d12bdf55d2 [Follower]" leader-address= leader-id= +2026-11-29T10:29:59.396Z [WARN] server-fd4f5d91-497f-3214-9074-98b4abb9d8d3: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:59.396Z [INFO] server-fd4f5d91-497f-3214-9074-98b4abb9d8d3: entering candidate state: node="Node at fd4f5d91-497f-3214-9074-98b4abb9d8d3 [Candidate]" term=2 +2026-11-29T10:29:59.396Z [INFO] server-fd4f5d91-497f-3214-9074-98b4abb9d8d3: election won: term=2 tally=2 +2026-11-29T10:29:59.396Z [INFO] server-fd4f5d91-497f-3214-9074-98b4abb9d8d3: entering leader state: leader="Node at fd4f5d91-497f-3214-9074-98b4abb9d8d3 [Leader]" +2026-11-29T10:29:59.396Z [INFO] server-fd4f5d91-497f-3214-9074-98b4abb9d8d3: added peer, starting replication: peer=server-08155706-0657-6915-7fca-8b77cae297bf +2026-11-29T10:29:59.396Z [INFO] server-fd4f5d91-497f-3214-9074-98b4abb9d8d3: added peer, starting replication: peer=server-080718e2-9d62-52dd-273f-08d12bdf55d2 +2026-11-29T10:29:59.396Z [INFO] server-fd4f5d91-497f-3214-9074-98b4abb9d8d3: pipelining replication: peer="{Voter server-080718e2-9d62-52dd-273f-08d12bdf55d2 080718e2-9d62-52dd-273f-08d12bdf55d2}" +2026-11-29T10:29:59.397Z [INFO] server-fd4f5d91-497f-3214-9074-98b4abb9d8d3: pipelining replication: peer="{Voter server-08155706-0657-6915-7fca-8b77cae297bf 08155706-0657-6915-7fca-8b77cae297bf}" +2026-11-29T10:29:59.397Z [WARN] server-08155706-0657-6915-7fca-8b77cae297bf: rejecting vote request since our last term is greater: candidate=fd4f5d91-497f-3214-9074-98b4abb9d8d3 last-term=2 last-candidate-term=1 +2026-11-29T10:29:59.502Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:59.397388859 +0000 UTC m=+110.132442961 (1 nodes), 61.19636ms from start of poll, 61.38896ms from cluster start. Timeout at 2026-11-29 10:29:59.502389779 +0000 UTC m=+110.237443841, 105.00088ms after stability +2026-11-29T10:29:59.502Z [INFO] cluster: starting stability test: raft-state=Leader +2026-11-29T10:29:59.609Z [INFO] cluster: stable state for Leader reached at 2026-11-29 10:29:59.502536979 +0000 UTC m=+110.237591081 (1 nodes), 1.88µs from start of poll, 166.53708ms from cluster start. Timeout at 2026-11-29 10:29:59.607576819 +0000 UTC m=+110.342630921, 105.03984ms after stability +2026-11-29T10:29:59.614Z [INFO] cluster: starting stability test: raft-state=Follower +2026-11-29T10:29:59.719Z [INFO] cluster: stable state for Follower reached at 2026-11-29 10:29:59.614850099 +0000 UTC m=+110.349904161 (2 nodes), 2.28µs from start of poll, 278.85016ms from cluster start. Timeout at 2026-11-29 10:29:59.719885939 +0000 UTC m=+110.454940041, 105.03588ms after stability +2026-11-29T10:29:59.720Z [ERROR] server-fd4f5d91-497f-3214-9074-98b4abb9d8d3: failed to pipeline appendEntries: peer="{Voter server-08155706-0657-6915-7fca-8b77cae297bf 08155706-0657-6915-7fca-8b77cae297bf}" error="append pipeline closed" +2026-11-29T10:29:59.720Z [INFO] server-fd4f5d91-497f-3214-9074-98b4abb9d8d3: aborting pipeline replication: peer="{Voter server-08155706-0657-6915-7fca-8b77cae297bf 08155706-0657-6915-7fca-8b77cae297bf}" +2026-11-29T10:29:59.724Z [ERROR] server-fd4f5d91-497f-3214-9074-98b4abb9d8d3: failed to heartbeat to: peer=080718e2-9d62-52dd-273f-08d12bdf55d2 backoff time=10ms error="failed to connect to peer: 080718e2-9d62-52dd-273f-08d12bdf55d2" +2026-11-29T10:29:59.727Z [ERROR] server-fd4f5d91-497f-3214-9074-98b4abb9d8d3: failed to appendEntries to: peer="{Voter server-08155706-0657-6915-7fca-8b77cae297bf 08155706-0657-6915-7fca-8b77cae297bf}" error="failed to connect to peer: 08155706-0657-6915-7fca-8b77cae297bf" +2026-11-29T10:29:59.728Z [ERROR] server-fd4f5d91-497f-3214-9074-98b4abb9d8d3: failed to pipeline appendEntries: peer="{Voter server-080718e2-9d62-52dd-273f-08d12bdf55d2 080718e2-9d62-52dd-273f-08d12bdf55d2}" error="append pipeline closed" +2026-11-29T10:29:59.728Z [INFO] server-fd4f5d91-497f-3214-9074-98b4abb9d8d3: aborting pipeline replication: peer="{Voter server-080718e2-9d62-52dd-273f-08d12bdf55d2 080718e2-9d62-52dd-273f-08d12bdf55d2}" +2026-11-29T10:29:59.728Z [ERROR] server-fd4f5d91-497f-3214-9074-98b4abb9d8d3: failed to heartbeat to: peer=08155706-0657-6915-7fca-8b77cae297bf backoff time=10ms error="failed to connect to peer: 08155706-0657-6915-7fca-8b77cae297bf" +2026-11-29T10:29:59.736Z [ERROR] server-fd4f5d91-497f-3214-9074-98b4abb9d8d3: failed to appendEntries to: peer="{Voter server-080718e2-9d62-52dd-273f-08d12bdf55d2 080718e2-9d62-52dd-273f-08d12bdf55d2}" error="failed to connect to peer: 080718e2-9d62-52dd-273f-08d12bdf55d2" +2026-11-29T10:29:59.742Z [ERROR] server-fd4f5d91-497f-3214-9074-98b4abb9d8d3: failed to heartbeat to: peer=080718e2-9d62-52dd-273f-08d12bdf55d2 backoff time=10ms error="failed to connect to peer: 080718e2-9d62-52dd-273f-08d12bdf55d2" +2026-11-29T10:29:59.743Z [ERROR] server-fd4f5d91-497f-3214-9074-98b4abb9d8d3: failed to appendEntries to: peer="{Voter server-08155706-0657-6915-7fca-8b77cae297bf 08155706-0657-6915-7fca-8b77cae297bf}" error="failed to connect to peer: 08155706-0657-6915-7fca-8b77cae297bf" +2026-11-29T10:29:59.748Z [ERROR] server-fd4f5d91-497f-3214-9074-98b4abb9d8d3: failed to heartbeat to: peer=08155706-0657-6915-7fca-8b77cae297bf backoff time=10ms error="failed to connect to peer: 08155706-0657-6915-7fca-8b77cae297bf" +2026-11-29T10:29:59.753Z [ERROR] server-fd4f5d91-497f-3214-9074-98b4abb9d8d3: failed to appendEntries to: peer="{Voter server-080718e2-9d62-52dd-273f-08d12bdf55d2 080718e2-9d62-52dd-273f-08d12bdf55d2}" error="failed to connect to peer: 080718e2-9d62-52dd-273f-08d12bdf55d2" +2026-11-29T10:29:59.759Z [ERROR] server-fd4f5d91-497f-3214-9074-98b4abb9d8d3: failed to heartbeat to: peer=080718e2-9d62-52dd-273f-08d12bdf55d2 backoff time=10ms error="failed to connect to peer: 080718e2-9d62-52dd-273f-08d12bdf55d2" +2026-11-29T10:29:59.762Z [ERROR] server-fd4f5d91-497f-3214-9074-98b4abb9d8d3: failed to appendEntries to: peer="{Voter server-08155706-0657-6915-7fca-8b77cae297bf 08155706-0657-6915-7fca-8b77cae297bf}" error="failed to connect to peer: 08155706-0657-6915-7fca-8b77cae297bf" +2026-11-29T10:29:59.768Z [ERROR] server-fd4f5d91-497f-3214-9074-98b4abb9d8d3: failed to heartbeat to: peer=08155706-0657-6915-7fca-8b77cae297bf backoff time=10ms error="failed to connect to peer: 08155706-0657-6915-7fca-8b77cae297bf" +2026-11-29T10:29:59.769Z [WARN] server-fd4f5d91-497f-3214-9074-98b4abb9d8d3: failed to contact: server-id=server-08155706-0657-6915-7fca-8b77cae297bf time=50.81456ms +2026-11-29T10:29:59.769Z [WARN] server-fd4f5d91-497f-3214-9074-98b4abb9d8d3: failed to contact: server-id=server-080718e2-9d62-52dd-273f-08d12bdf55d2 time=50.91548ms +2026-11-29T10:29:59.769Z [WARN] server-fd4f5d91-497f-3214-9074-98b4abb9d8d3: failed to contact quorum of nodes, stepping down +2026-11-29T10:29:59.769Z [ERROR] server-fd4f5d91-497f-3214-9074-98b4abb9d8d3: failed to appendEntries to: peer="{Voter server-080718e2-9d62-52dd-273f-08d12bdf55d2 080718e2-9d62-52dd-273f-08d12bdf55d2}" error="failed to connect to peer: 080718e2-9d62-52dd-273f-08d12bdf55d2" +2026-11-29T10:29:59.769Z [INFO] server-fd4f5d91-497f-3214-9074-98b4abb9d8d3: entering follower state: follower="Node at fd4f5d91-497f-3214-9074-98b4abb9d8d3 [Follower]" leader-address= leader-id= +2026-11-29T10:29:59.788Z [WARN] server-08155706-0657-6915-7fca-8b77cae297bf: heartbeat timeout reached, starting election: last-leader-addr=fd4f5d91-497f-3214-9074-98b4abb9d8d3 last-leader-id=server-fd4f5d91-497f-3214-9074-98b4abb9d8d3 +2026-11-29T10:29:59.788Z [INFO] server-08155706-0657-6915-7fca-8b77cae297bf: entering candidate state: node="Node at 08155706-0657-6915-7fca-8b77cae297bf [Candidate]" term=3 +2026-11-29T10:29:59.788Z [WARN] server-080718e2-9d62-52dd-273f-08d12bdf55d2: rejecting vote request since we have a leader: from=08155706-0657-6915-7fca-8b77cae297bf leader=fd4f5d91-497f-3214-9074-98b4abb9d8d3 leader-id=server-fd4f5d91-497f-3214-9074-98b4abb9d8d3 +2026-11-29T10:29:59.788Z [ERROR] server-08155706-0657-6915-7fca-8b77cae297bf: failed to make requestVote RPC: target="{Voter server-fd4f5d91-497f-3214-9074-98b4abb9d8d3 fd4f5d91-497f-3214-9074-98b4abb9d8d3}" error="failed to connect to peer: fd4f5d91-497f-3214-9074-98b4abb9d8d3" term=3 +2026-11-29T10:29:59.815Z [WARN] server-080718e2-9d62-52dd-273f-08d12bdf55d2: heartbeat timeout reached, starting election: last-leader-addr=fd4f5d91-497f-3214-9074-98b4abb9d8d3 last-leader-id=server-fd4f5d91-497f-3214-9074-98b4abb9d8d3 +2026-11-29T10:29:59.815Z [INFO] server-080718e2-9d62-52dd-273f-08d12bdf55d2: entering candidate state: node="Node at 080718e2-9d62-52dd-273f-08d12bdf55d2 [Candidate]" term=3 +2026-11-29T10:29:59.815Z [INFO] server-08155706-0657-6915-7fca-8b77cae297bf: duplicate requestVote for same term: term=3 +2026-11-29T10:29:59.815Z [ERROR] server-080718e2-9d62-52dd-273f-08d12bdf55d2: failed to make requestVote RPC: target="{Voter server-fd4f5d91-497f-3214-9074-98b4abb9d8d3 fd4f5d91-497f-3214-9074-98b4abb9d8d3}" error="failed to connect to peer: fd4f5d91-497f-3214-9074-98b4abb9d8d3" term=3 +2026-11-29T10:29:59.826Z [INFO] server-08155706-0657-6915-7fca-8b77cae297bf: entering follower state: follower="Node at 08155706-0657-6915-7fca-8b77cae297bf [Follower]" leader-address= leader-id= --- PASS: TestRaft_VoteWithNoIDNoAddr (0.49s) === RUN TestRaft_runFollower_State_Transition === RUN TestRaft_runFollower_State_Transition/NonVoter -2025-10-27T04:03:58.131Z [INFO] 127.0.0.1:45703 :: starting node: addr=127.0.0.1:45703 -2025-10-27T04:03:58.131Z [INFO] 127.0.0.1:45703 :: initial configuration: index=0 servers=[] -2025-10-27T04:03:58.131Z [INFO] 127.0.0.1:45703 :: entering follower state: follower="Node at 127.0.0.1:45703 [Follower]" leader-address= leader-id= -2025-10-27T04:03:58.219Z [WARN] 127.0.0.1:45703 :: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +2026-11-29T10:29:59.826Z [INFO] 127.0.0.1:37145 :: starting node: addr=127.0.0.1:37145 +2026-11-29T10:29:59.826Z [INFO] 127.0.0.1:37145 :: initial configuration: index=0 servers=[] +2026-11-29T10:29:59.826Z [INFO] 127.0.0.1:37145 :: entering follower state: follower="Node at 127.0.0.1:37145 [Follower]" leader-address= leader-id= +2026-11-29T10:29:59.901Z [WARN] 127.0.0.1:37145 :: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election === RUN TestRaft_runFollower_State_Transition/Voter -2025-10-27T04:03:58.282Z [INFO] 127.0.0.1:34421 :: starting node: addr=127.0.0.1:34421 -2025-10-27T04:03:58.282Z [INFO] 127.0.0.1:34421 :: initial configuration: index=0 servers=[] -2025-10-27T04:03:58.282Z [INFO] 127.0.0.1:34421 :: entering follower state: follower="Node at 127.0.0.1:34421 [Follower]" leader-address= leader-id= -2025-10-27T04:03:58.355Z [WARN] 127.0.0.1:34421 :: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:29:59.979Z [INFO] 127.0.0.1:44665 :: starting node: addr=127.0.0.1:44665 +2026-11-29T10:29:59.979Z [INFO] 127.0.0.1:44665 :: initial configuration: index=0 servers=[] +2026-11-29T10:29:59.979Z [INFO] 127.0.0.1:44665 :: entering follower state: follower="Node at 127.0.0.1:44665 [Follower]" leader-address= leader-id= +2026-11-29T10:30:00.042Z [WARN] 127.0.0.1:44665 :: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= === RUN TestRaft_runFollower_State_Transition/Not_in_Config -2025-10-27T04:03:58.433Z [INFO] 127.0.0.1:35039 :: starting node: addr=127.0.0.1:35039 -2025-10-27T04:03:58.433Z [INFO] 127.0.0.1:35039 :: initial configuration: index=0 servers=[] -2025-10-27T04:03:58.433Z [INFO] 127.0.0.1:35039 :: entering follower state: follower="Node at 127.0.0.1:35039 [Follower]" leader-address= leader-id= -2025-10-27T04:03:58.494Z [WARN] 127.0.0.1:35039 :: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election ---- PASS: TestRaft_runFollower_State_Transition (0.45s) +2026-11-29T10:30:00.131Z [INFO] 127.0.0.1:34861 :: starting node: addr=127.0.0.1:34861 +2026-11-29T10:30:00.131Z [INFO] 127.0.0.1:34861 :: initial configuration: index=0 servers=[] +2026-11-29T10:30:00.131Z [INFO] 127.0.0.1:34861 :: entering follower state: follower="Node at 127.0.0.1:34861 [Follower]" leader-address= leader-id= +2026-11-29T10:30:00.219Z [WARN] 127.0.0.1:34861 :: heartbeat timeout reached, not part of a stable configuration or a non-voter, not triggering a leader election +--- PASS: TestRaft_runFollower_State_Transition (0.46s) --- PASS: TestRaft_runFollower_State_Transition/NonVoter (0.15s) --- PASS: TestRaft_runFollower_State_Transition/Voter (0.15s) --- PASS: TestRaft_runFollower_State_Transition/Not_in_Config (0.15s) === RUN TestRaft_runFollower_ReloadTimeoutConfigs -2025-10-27T04:03:58.584Z [INFO] 127.0.0.1:35583 :: starting node: addr=127.0.0.1:35583 -2025-10-27T04:03:58.584Z [INFO] 127.0.0.1:35583 :: initial configuration: index=0 servers=[] -2025-10-27T04:03:58.584Z [INFO] 127.0.0.1:35583 :: entering follower state: follower="Node at 127.0.0.1:35583 [Follower]" leader-address= leader-id= -2025-10-27T04:03:58.584Z [WARN] 127.0.0.1:35583 :: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= +2026-11-29T10:30:00.282Z [INFO] 127.0.0.1:42269 :: starting node: addr=127.0.0.1:42269 +2026-11-29T10:30:00.282Z [INFO] 127.0.0.1:42269 :: initial configuration: index=0 servers=[] +2026-11-29T10:30:00.282Z [INFO] 127.0.0.1:42269 :: entering follower state: follower="Node at 127.0.0.1:42269 [Follower]" leader-address= leader-id= +2026-11-29T10:30:00.282Z [WARN] 127.0.0.1:42269 :: heartbeat timeout reached, starting election: last-leader-addr= last-leader-id= --- PASS: TestRaft_runFollower_ReloadTimeoutConfigs (0.15s) === RUN TestSaturationMetric === RUN TestSaturationMetric/without_smoothing @@ -4260,7 +4274,7 @@ === RUN TestOverrideNotifyBool --- PASS: TestOverrideNotifyBool (0.00s) PASS -ok github.com/hashicorp/raft 113.744s +ok github.com/hashicorp/raft 111.182s ? github.com/hashicorp/raft/bench [no test files] make[1]: Leaving directory '/build/reproducible-path/golang-github-hashicorp-raft-1.5.0' create-stamp debian/debhelper-build-stamp @@ -4289,12 +4303,14 @@ dpkg-buildpackage: info: binary-only upload (no source included) dpkg-genchanges: info: including full source code in upload I: copying local configuration +I: user script /srv/workspace/pbuilder/3049399/tmp/hooks/B01_cleanup starting +I: user script /srv/workspace/pbuilder/3049399/tmp/hooks/B01_cleanup finished I: unmounting dev/ptmx filesystem I: unmounting dev/pts filesystem I: unmounting dev/shm filesystem I: unmounting proc filesystem I: unmounting sys filesystem I: cleaning the build env -I: removing directory /srv/workspace/pbuilder/2445373 and its subdirectories -I: Current time: Sun Oct 26 16:04:04 -12 2025 -I: pbuilder-time-stamp: 1761537844 +I: removing directory /srv/workspace/pbuilder/3049399 and its subdirectories +I: Current time: Mon Nov 30 00:30:04 +14 2026 +I: pbuilder-time-stamp: 1795948204